Adding CrowsPairs task for English and French

lm_eval/tasks/__init__.py

@@ -59,6 +59,7 @@
 from . import wino_bias
 from . import e2e_nlg_cleaned
 from . import gem_asset_turk
+from . import crows_pairs_multilingual
 
 ########################################
 # Translation tasks
@@ -317,6 +318,10 @@
     "wino_bias_type1_anti": wino_bias.WinoBiasType1Anti,
     "wino_bias_type2_pro": wino_bias.WinoBiasType2Pro,
     "wino_bias_type2_anti": wino_bias.WinoBiasType2Anti,
+
+    # Crows-Pairs
+    "crows_pairs_english": crows_pairs_multilingual.CrowsPairsEnglish,
+    "crows_pairs_french": crows_pairs_multilingual.CrowsPairsFrench,
 }
 
 

lm_eval/tasks/crows_pairs_multilingual.py

@@ -0,0 +1,65 @@
+"""
+French CrowS-Pairs: Extending a challenge dataset for measuring social bias in masked language models to a language other than English
+https://hal.inria.fr/hal-03629677/file/ACLFinal.pdf
+Measuring social biases in masked language models in English and French.
+https://gitlab.inria.fr/french-crows-pairs/acl-2022-paper-data-and-code/-/tree/main
+"""
+from lm_eval.base import PromptSourceTask
+
+
+_CITATION = """\
+@inproceedings{neveol2022french,
+  title={French CrowS-Pairs: Extending a challenge dataset for measuring social bias in masked language models to a language other than English},
+  author={N{\'e}v{\'e}ol, Aur{\'e}lie and Dupont, Yoann and Bezan{\c{c}}on, Julien and Fort, Kar{\"e}n},
+  booktitle={ACL 2022-60th Annual Meeting of the Association for Computational Linguistics},
+  year={2022}
+"""
+
+
+class CrowsPairsEnglish(PromptSourceTask):
+    VERSION = 0
+    DATASET_PATH = "oskarvanderwal/crows_pairs_multilingual"
+    DATASET_NAME = "english"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        pass
+
+    def validation_docs(self):
+        pass
+
+    def test_docs(self):
+        if self.has_test_docs():
+            return self.dataset["test"]
+
+class CrowsPairsFrench(PromptSourceTask):
+    VERSION = 0
+    DATASET_PATH = "oskarvanderwal/crows_pairs_multilingual"
+    DATASET_NAME = "french"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        pass
+
+    def validation_docs(self):
+        pass
+
+    def test_docs(self):
+        if self.has_test_docs():
+            return self.dataset["test"]