Natural Language Inference (NLI) has been one of the fundamental tasks in Natural Language Processing (NLP). Recognizing Textual Entailment (RTE) between the two pieces of text is a crucial problem. It adds further challenges when it involves two languages, i.e., in the cross-lingual scenario. In this paper, we propose VisioTextual-Attention (VTA) — an effective visual–textual coattention mechanism for multi-modal crosslingual NLI. Through our research, we show that instead of using only linguistic input features, introducing visual features supporting the textual inputs improves the performance of NLI models if an effective cross-modal attention mechanism is carefully constructed. We perform several experiments on a standard cross-lingual textual entailment dataset in Hindi–English language pairs and show that the addition of visual information to the dataset along with our proposed VisioTextual Attention (VTA) enhances performance and surpasses the current state-of-the-art by 4.5%. Through monolingual experiments, we also show that the proposed VTA mechanism surpasses monolingual state-of-the-art by a margin of 2.89%. We argue that our VTA mechanism is model agnostic and can be used with other deep learning-based architectures for grounded cross-lingual NLI.
@article{DE2023100023,
title = {Towards Improvement of Grounded Cross-lingual Natural Language Inference with VisioTextual Attention},
journal = {Natural Language Processing Journal},
volume = {4},
pages = {100023},
year = {2023},
issn = {2949-7191},
doi = {https://doi.org/10.1016/j.nlp.2023.100023},
url = {https://www.sciencedirect.com/science/article/pii/S2949719123000201},
author = {Arkadipta De and Maunendra Sankar Desarkar and Asif Ekbal},
keywords = {Attention mechanism, Textual entailment, Natural Language Inference, Grounded textual entailment, Cross-lingual textual entailment},
abstract = {Natural Language Inference (NLI) has been one of the fundamental tasks in Natural Language Processing (NLP). Recognizing Textual Entailment (RTE) between the two pieces of text is a crucial problem. It adds further challenges when it involves two languages, i.e., in the cross-lingual scenario. In this paper, we propose VisioTextual-Attention (VTA) — an effective visual–textual coattention mechanism for multi-modal cross-lingual NLI. Through our research, we show that instead of using only linguistic input features, introducing visual features supporting the textual inputs improves the performance of NLI models if an effective cross-modal attention mechanism is carefully constructed. We perform several experiments on a standard cross-lingual textual entailment dataset in Hindi–English language pairs and show that the addition of visual information to the dataset along with our proposed VisioTextual Attention (VTA) enhances performance and surpasses the current state-of-the-art by 4.5%. Through monolingual experiments, we also show that the proposed VTA mechanism surpasses monolingual state-of-the-art by a margin of 2.89%. We argue that our VTA mechanism is model agnostic and can be used with other deep learning-based architectures for grounded cross-lingual NLI.}
}