fix and clean sem_eval_2014 template (#697)

* fix and clean sem_eval_2014 template

* some fixes

Co-authored-by: Victor Sanh <[email protected]>

Author: KhalidAlt

promptsource/templates/sem_eval_2014_task_1/templates.yaml

@@ -6,83 +6,93 @@ templates:
     jinja: 'Does the premise: "{{premise}}" agree with the hypothesis: "{{hypothesis}}"
       ? ||| {{answer_choices[entailment_judgment]}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: true
-    name: entailment_basic_3
+    name: premise_agree_hypothesis
     reference: ''
   2aa091cb-02ff-4c8c-964c-4c5e53df8c1b: !Template
     answer_choices: null
     id: 2aa091cb-02ff-4c8c-964c-4c5e53df8c1b
-    jinja: "How related are the two sentences : \"{{hypothesis}}\" and \"{{premise}}\"\
-      \ ? Rate it from 1-5. \n||| {{(((10*relatedness_score)|round)/10)}}"
+    jinja: 'How related are the two sentences : "{{hypothesis}}" and "{{premise}}"
+      ? Rate it from 1-5, where  1 is completely unrelated and 5 is very related.
+
+      ||| {{(((10*relatedness_score)|round)/10)}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Pearson correlation
+      - Spearman correlation
+      - Mean Squared Error
       original_task: true
-    name: relatedness_basic_2
+    name: related_rate
     reference: ''
   75203dd2-5ec3-4e91-b95f-228ad9bd2010: !Template
     answer_choices: neither ||| entailing ||| contradicting
     id: 75203dd2-5ec3-4e91-b95f-228ad9bd2010
     jinja: "Sentence 1: \"{{hypothesis}}\" \nSentence 2: \"{{premise}}\"\nAre the\
-      \ two sentences {{\"entailing\"}} or {{\"contradicting\"}} each other?\n|||\
-      \ {{answer_choices[entailment_judgment]}}"
+      \ two sentences {{answer_choices[1]}} or {{answer_choices[2]}} each other? If\
+      \ none of these options are valid, answer \"{{answer_choices[0]}}\".\n||| {{answer_choices[entailment_judgment]}}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: true
-    name: entailment_basic_2
+    name: entailing_or_contradicting
     reference: ''
   892c58fd-64f5-4059-8fb8-c74bc025ff40: !Template
     answer_choices: Neutral ||| Entailment ||| Contradiction
     id: 892c58fd-64f5-4059-8fb8-c74bc025ff40
     jinja: "Given the following hypothesis: {{hypothesis}}.\nAs well as the premise:\
       \ {{premise}}, \nPredict the Entailment relation between the premise and hypothesis\
-      \ from the labels {{\"Neutral\"}}, {{\"Entailment\"}}, {{ \"Contradiction\"\
-      }} |||\n {{answer_choices[entailment_judgment]}}\n"
+      \ from the labels {{answer_choices[0]}}, {{answer_choices[1]}}, {{answer_choices[2]}}\
+      \ |||\n {{answer_choices[entailment_judgment]}}\n"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: true
-    name: entailment_basic_1
+    name: entailment_relation
     reference: ''
   91a6b1db-be59-41bd-9eea-73bb7a4e7350: !Template
-    answer_choices: neither entails nor contradicts ||| entails ||| contradicts
+    answer_choices: Neutral ||| Entailment ||| Contradiction
     id: 91a6b1db-be59-41bd-9eea-73bb7a4e7350
     jinja: 'Given the hypothesis: {{hypothesis}} and the premise: {{premise}}. Out
-      of the options, {{"neither entails nor contradicts"}}, {{"entails"}} and {{
-      "contradicts"}}, the hypothesis ||| {{answer_choices[entailment_judgment]}}
-      the premise.
-
-      '
+      of the options, {{answer_choices[0]}}, {{answer_choices[1]}} and {{answer_choices[2]}}
+      what is the entailment label? ||| {{answer_choices[entailment_judgment]}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: true
-    name: entailment_localization_1
+    name: entailment_label
     reference: ''
   a58fe8b4-f185-46a9-8fca-6dc66d0812be: !Template
     answer_choices: null
     id: a58fe8b4-f185-46a9-8fca-6dc66d0812be
     jinja: "Given the following hypothesis: {{hypothesis}}.\nAs well as the premise:\
-      \ {{premise}}, \nGive a score on how related the hypothesis and premise was,\
+      \ {{premise}}, \nGive a score on how related the hypothesis and premise were,\
       \ from the scale 1 to 5, where  1 is completely unrelated and 5 is very related:\
       \ |||   {{(((10*relatedness_score)|round)/10)}}\n\n"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Pearson correlation
+      - Spearman correlation
+      - Mean Squared Error
       original_task: true
-    name: relatedness_basic_1
+    name: related_score
     reference: ''
   d9380ec0-18b3-48b2-99eb-9f9cb47ab7c7: !Template
     answer_choices: unclear ||| yes ||| no
     id: d9380ec0-18b3-48b2-99eb-9f9cb47ab7c7
     jinja: Does {{premise}} imply that {{hypothesis}}?  Please answer yes, no, or
       unclear. ||| {{answer_choices[entailment_judgment]}}
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: true
-    name: entailment_basic_4
+    name: premise_imply_hypothesis
     reference: ''