Merge answer_choices and metadata. (#548)

* Combine answer_choices and answer_choices_key in Python code.

* Remove span squad and sequence accuracy from app choices.

* Test merge on community dataset.

* Fix path parsing for community datasets.

* Forgot to put the slash back in.

* Just skip it.

* Actually skipping.

* Test just the a datasets.

* Update all the rest.

* some missing traces of `answer_choices_key`

* Fix handling of answer_choices when not set.

Co-authored-by: VictorSanh <[email protected]>

Author: stephenbach

promptsource/app.py

@@ -348,8 +348,6 @@ def get_infos(d_name):
                 st.markdown("##### Metrics")
                 st.text(", ".join(template.metadata.metrics) if template.metadata.metrics else None)
                 st.markdown("##### Answer Choices")
-                st.text(", ".join(template.answer_choices) if template.answer_choices is not None else None)
-                st.markdown("##### Answer Choices Key")
                 if template.get_answer_choices_expr() is not None:
                     show_jinja(template.get_answer_choices_expr())
                 else:
@@ -491,11 +489,9 @@ def get_infos(d_name):
                         metrics_choices = [
                             "BLEU",
                             "ROUGE",
-                            "Span Squad",
                             "Squad",
                             "Trivia QA",
                             "Accuracy",
-                            "Sequence Accuracy",
                             "Pearson Correlation",
                             "Spearman Correlation",
                             "MultiRC",
@@ -518,21 +514,13 @@ def get_infos(d_name):
                         )
 
                         # Answer choices
-                        state.answer_choices = st.text_input(
-                            "Answer Choices",
-                            value=" ||| ".join(template.answer_choices) if template.answer_choices is not None else "",
-                            help="A ||| separated list of possible outputs (or leave blank). "
-                            + "Value is available in Jinja in a list called answer_choices.",
-                        )
-
-                        # Answer choices key
                         if template.get_answer_choices_expr() is not None:
-                            answer_choices_key = template.get_answer_choices_expr()
+                            answer_choices = template.get_answer_choices_expr()
                         else:
-                            answer_choices_key = ""
-                        state.answer_choices_key = st.text_input(
-                            "Answer Choices Key",
-                            value=answer_choices_key,
+                            answer_choices = ""
+                        state.answer_choices = st.text_input(
+                            "Answer Choices",
+                            value=answer_choices,
                             help="A Jinja expression for computing answer choices. "
                             "Separate choices with a triple bar (|||).",
                         )
@@ -553,14 +541,11 @@ def get_infos(d_name):
                             elif updated_template_name == "":
                                 st.error("Need to provide a template name.")
                             else:
-                                # Parses state.answer_choices and state.answer_choices_key
-                                updated_answer_choices = [x.strip() for x in state.answer_choices.split("|||")]
-                                if len(updated_answer_choices) == 0 or len(updated_answer_choices) == 1:
+                                # Parses state.answer_choices
+                                if state.answer_choices == "":
                                     updated_answer_choices = None
-                                if state.answer_choices_key == "":
-                                    updated_answer_choices_key = None
                                 else:
-                                    updated_answer_choices_key = state.answer_choices_key
+                                    updated_answer_choices = state.answer_choices
 
                                 dataset_templates.update_template(
                                     state.template_name,
@@ -569,7 +554,6 @@ def get_infos(d_name):
                                     state.reference,
                                     state.metadata,
                                     updated_answer_choices,
-                                    updated_answer_choices_key,
                                 )
                                 # Update the state as well
                                 state.template_name = updated_template_name

promptsource/seqio_tasks/tasks.py

@@ -118,7 +118,7 @@ def add_task(dataset_name, subset_name, template_name, task_name=None, split_map
     )
 
     # Add rank classification eval task
-    if template.answer_choices or template.answer_choices_key:
+    if template.answer_choices:
         rank_classification_preprocessor = functools.partial(
             t5.data.preprocessors.rank_classification,
             inputs_fn=lambda ex: tf.fill((len(ex["answer_choices"]),), ex["inputs"]),
@@ -225,7 +225,7 @@ def add_task(dataset_name, subset_name, template_name, task_name=None, split_map
         if (dataset_name, subset_name) in d4_eval:
             if template.metadata.original_task:
                 d4_eval_mixture.append(task_name)
-            # TODO use template.metadata.answer_choices or answer_choice_keys here for rank eval
+            # TODO use template.metadata.answer_choices here for rank eval
         if (dataset_name, subset_name) in bias_fairness_eval:
             bias_fairness_eval_mixture.append(task_name)
 
@@ -245,7 +245,7 @@ def add_task(dataset_name, subset_name, template_name, task_name=None, split_map
         template = dataset[template_name]
         if template.metadata.original_task:
             d4_eval_mixture.append(task_name)  # TODO or add to ANLI special mixture
-        # TODO use template.metadata.answer_choices or answer_choice_keys here for rank eval
+        # TODO use template.metadata.answer_choices here for rank eval
 
 
 TASK_BLACKLIST = [

promptsource/templates.py

@@ -58,7 +58,7 @@ class Template(yaml.YAMLObject):
 
     yaml_tag = "!Template"
 
-    def __init__(self, name, jinja, reference, metadata=None, answer_choices=None, answer_choices_key=None):
+    def __init__(self, name, jinja, reference, metadata=None, answer_choices=None):
         """
         Creates a prompt template.
 
@@ -73,23 +73,19 @@ def __init__(self, name, jinja, reference, metadata=None, answer_choices=None, a
         :param jinja: template expressed in Jinja
         :param reference: string describing author or paper reference for template
         :param metadata: a Metadata object with template annotations
-        :param answer_choices: list of strings that enumerates the possible completions
-                               for templates that should be evaluated as ranked
-                               completions. If None, then the template is open-ended.
-                               This list is accessible from within Jinja as the
-                               variable `answer_choices`.
-                               TODO: Merge answer_choices and answer_choices_key
-        :param answer_choices_key: Jinja expression for answer choices, or None if
-                                   no answer choices
-
+        :param answer_choices: Jinja expression for answer choices. Should produce
+                               a ||| delimited string of choices that enumerates
+                               the possible completions for templates that should
+                               be evaluated as ranked completions. If None, then
+                               the template is open-ended. This list is accessible
+                               from within Jinja as the variable `answer_choices`.
         """
         self.id = str(uuid.uuid4())
         self.name = name
         self.jinja = jinja
         self.reference = reference
         self.metadata = metadata if metadata is not None else Template.Metadata()
         self.answer_choices = answer_choices
-        self.answer_choices_key = answer_choices_key
 
     def get_id(self):
         """
@@ -115,36 +111,24 @@ def get_reference(self):
         """
         return self.reference
 
-    def get_answer_choices(self):
-        """
-        Returns a list of strings enumerating the possible completions for
-        this template, or None if the template is open ended.
-
-        :return: List[String]
-        """
-        # TODO: Replace answer_choices with answer_choices_key values
-        return self.answer_choices
-
     def get_answer_choices_expr(self):
         """
         Returns a Jinja expression for computing the answer choices from an example.
 
         :return: String, or None if no answer choices
         """
-        # TODO: Change to return answer_choices
-        return self.answer_choices_key
+        return self.answer_choices
 
     def get_answer_choices_list(self, example):
         """
         Returns a list of answer choices for a given example
 
         :return: list of strings, or None if get_answer_choices_expr is None
         """
-        # TODO: remove when merging answer_choices and answer_choices_key
-        if self.get_answer_choices_expr() is None:
-            return self.get_answer_choices()
+        jinja = self.get_answer_choices_expr()
+        if jinja is None:
+            return None
 
-        jinja = self.answer_choices_key
         rtemplate = env.from_string(jinja)
         protected_example = self._escape_pipe(example)
         rendered_choices = rtemplate.render(**protected_example)
@@ -156,11 +140,10 @@ def get_fixed_answer_choices_list(self):
 
         :return: list of strings, or None if no static list exists
         """
-        # TODO: remove when merging answer_choices and answer_choices_key
-        if self.get_answer_choices_expr() is None:
-            return self.get_answer_choices()
+        jinja = self.get_answer_choices_expr()
+        if jinja is None:
+            return None
 
-        jinja = self.answer_choices_key
         parse = env.parse(jinja)
         variables = meta.find_undeclared_variables(parse)
         if len(variables) == 0:
@@ -450,8 +433,7 @@ def update_template(
         jinja: str,
         reference: str,
         metadata: Template.Metadata,
-        answer_choices: List[str],
-        answer_choices_key: str,
+        answer_choices: str,
     ) -> None:
         """
         Updates a pre-existing template and writes changes
@@ -461,16 +443,14 @@ def update_template(
         :param jinja: new jinja entry
         :param reference: new reference entry
         :param metadata: a Metadata object with template annotations
-        :param answer_choices: new answer_choices list
-        :param answer_choices_key: new answer_choices_key string
+        :param answer_choices: new answer_choices string
         """
         template_id = self.name_to_id_mapping[current_template_name]
         self.templates[template_id].name = new_template_name
         self.templates[template_id].jinja = jinja
         self.templates[template_id].reference = reference
         self.templates[template_id].metadata = metadata
         self.templates[template_id].answer_choices = answer_choices
-        self.templates[template_id].answer_choices_key = answer_choices_key
 
         self.write_to_file()
 
@@ -512,7 +492,6 @@ def get_templates_data_frame():
         "choices_in_prompt": [],
         "metrics": [],
         "answer_choices": [],
-        "answer_choices_key": [],
         "jinja": [],
     }
 
@@ -530,8 +509,7 @@ def get_templates_data_frame():
             data["original_task"].append(template.metadata.original_task)
             data["choices_in_prompt"].append(template.metadata.choices_in_prompt)
             data["metrics"].append(template.metadata.metrics)
-            data["answer_choices"].append(template.get_answer_choices())
-            data["answer_choices_key"].append(template.get_answer_choices_expr())
+            data["answer_choices"].append(template.get_answer_choices_expr())
             data["jinja"].append(template.jinja)
 
     return pd.DataFrame(data)

promptsource/templates/Zaid/coqa_expanded/templates.yaml

@@ -2,7 +2,6 @@ dataset: Zaid/coqa_expanded
 templates:
   12ad4331-d063-4b56-b0f6-76f59c690717: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 12ad4331-d063-4b56-b0f6-76f59c690717
     jinja: "Below is a passage, followed by a series of questions and answers about\
       \ the passage. Answer the last question based on the information contained in\
@@ -18,7 +17,6 @@ templates:
     reference: 'Metric: variant of SQuAD (Section 6.1 of the paper)'
   2f9fb20d-f4c9-4371-9cd4-db47607cb7a3: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 2f9fb20d-f4c9-4371-9cd4-db47607cb7a3
     jinja: "What is the answer to the last question in the dialogue below? If there\
       \ is no answer in the passage, say \"unknown\".\n\nPassage: {{story}}\n\nQ:\
@@ -33,7 +31,6 @@ templates:
     reference: 'Metric: variant of SQuAD (Section 6.1 of the paper)'
   9aff8967-d41c-4d79-8ef4-fc3650773735: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 9aff8967-d41c-4d79-8ef4-fc3650773735
     jinja: "Complete the dialogue based on the information contained in the passage.\
       \ If there is no answer in the passage, say \"unknown\".\n\nPassage: {{story}}\n\
@@ -48,7 +45,6 @@ templates:
     reference: 'Metric: variant of SQuAD (Section 6.1 of the paper)'
   9bc32f2e-eee6-4006-bce3-74a79403d33e: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 9bc32f2e-eee6-4006-bce3-74a79403d33e
     jinja: "Answer the last question based on the information contained in the passage.\
       \ If there is no answer in the passage, say \"unknown\".\n\nPassage: {{story}}\n\
@@ -63,7 +59,6 @@ templates:
     reference: 'Metric: variant of SQuAD (Section 6.1 of the paper)'
   bacb6534-e607-4afc-a412-ccfcd9fe38e2: !Template
     answer_choices: null
-    answer_choices_key: null
     id: bacb6534-e607-4afc-a412-ccfcd9fe38e2
     jinja: 'In the passage below, extract the part which answers the last question.
       If there is no answer in the passage, say "unknown".
@@ -94,7 +89,6 @@ templates:
     reference: ''
   be39974f-aa86-4076-b444-bd3c2732b17b: !Template
     answer_choices: null
-    answer_choices_key: null
     id: be39974f-aa86-4076-b444-bd3c2732b17b
     jinja: "Help me complete the dialogue about this passage. If there is no answer\
       \ in the passage, say \"unknown\".\n\nPassage: {{story}}\n\nQ: {{question}}\
@@ -109,7 +103,6 @@ templates:
     reference: 'Metric: variant of SQuAD (Section 6.1 of the paper)'
   d95440ce-d538-40f8-ae09-664e05852ca8: !Template
     answer_choices: null
-    answer_choices_key: null
     id: d95440ce-d538-40f8-ae09-664e05852ca8
     jinja: "{{story}}\n\nQ: {{question}} \nA: ||| {% if answer[\"answer_start\"] !=\
       \ -1 %}\n{{answer[\"input_text\"]}}\n{% else %}\nunknown\n{% endif %}"

promptsource/templates/Zaid/quac_expanded/templates.yaml

@@ -2,7 +2,6 @@ dataset: Zaid/quac_expanded
 templates:
   01d8c949-89a7-4a44-9a39-6cf2ac3e0a7b: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 01d8c949-89a7-4a44-9a39-6cf2ac3e0a7b
     jinja: "What is the answer to the last question in the dialogue below? If there\
       \ is no answer in the passage, say \"unknown\".\n\nPassage: {{context}}\n\n\
@@ -16,7 +15,6 @@ templates:
     reference: 'Metric: F1'
   1484c6e6-bf42-47ca-9ea7-c3c552a24de1: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 1484c6e6-bf42-47ca-9ea7-c3c552a24de1
     jinja: "{{context}}\n\nQ: {{question}} \nA: ||| {{answer[\"texts\"][0]}}"
     metadata: !TemplateMetadata
@@ -28,7 +26,6 @@ templates:
     reference: 'Brown et al. NeurIPS 2020. Metric: F1'
   2bca0532-01a3-4a64-a228-a57ae0965719: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 2bca0532-01a3-4a64-a228-a57ae0965719
     jinja: "Below is a passage, followed by a series of questions and answers about\
       \ the passage. Answer the last question based on the information contained in\
@@ -43,7 +40,6 @@ templates:
     reference: 'Metric: F1'
   4abd0379-dbc0-4f71-901b-dd0af3581157: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 4abd0379-dbc0-4f71-901b-dd0af3581157
     jinja: "Answer the last question based on the information contained in the passage.\
       \ If there is no answer in the passage, say \"unknown\".\n\nPassage: {{context}}\n\
@@ -57,7 +53,6 @@ templates:
     reference: 'Metric: F1'
   8ebbd098-b40c-4e69-8cbb-0ffecf0fe2a6: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 8ebbd098-b40c-4e69-8cbb-0ffecf0fe2a6
     jinja: "Complete the dialogue based on the information contained in the passage.\
       \ If there is no answer in the passage, say \"unknown\".\n\nPassage: {{context}}\n\
@@ -71,7 +66,6 @@ templates:
     reference: 'Metric: F1'
   e624695b-5d26-47cc-bdb4-ac2bee4ddaea: !Template
     answer_choices: null
-    answer_choices_key: null
     id: e624695b-5d26-47cc-bdb4-ac2bee4ddaea
     jinja: "Help me complete the dialogue about this passage. If there is no answer\
       \ in the passage, say \"unknown\".\n\nPassage: {{context}}\n\nQ: {{question}}\

promptsource/templates/acronym_identification/templates.yaml

@@ -2,7 +2,6 @@ dataset: acronym_identification
 templates:
   64f438f2-9968-459f-82d2-24bad632b358: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 64f438f2-9968-459f-82d2-24bad632b358
     jinja: "{% set random_abbr = '' %}\n{% set _dummy = none %}\n{% set abbr_exp_dict\
       \ = namespace(value = {}) %}\n{% set abbr_string=namespace(value='') %}\n{%\
@@ -48,7 +47,6 @@ templates:
     reference: Given the tokens, find the expansion of an abbreviation in the tokens.
   81babc83-18cd-4eed-a343-8ede56b21df5: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 81babc83-18cd-4eed-a343-8ede56b21df5
     jinja: "Given the BIO encoding as follows:  \"{{\"B-short\"}}\" and \"{{\"I-short\"\
       }}\" represent the beginning and intermediate tokens for abbreviations.\"{{\"\
@@ -66,7 +64,6 @@ templates:
     reference: Given the comma separated tokens, generate BIO encoding for abbreviations.
   8832e5f7-7c45-46da-b85f-71fcb444f264: !Template
     answer_choices: null
-    answer_choices_key: null
     id: 8832e5f7-7c45-46da-b85f-71fcb444f264
     jinja: 'List all the expansions of the acronyms present in the following comma-separated
       tokens. Return {{"No expansions found"}} if the expansions can''t be found.
@@ -122,7 +119,6 @@ templates:
     reference: Given the tokens, list the expansion tokens.
   cae58242-cde9-472d-ae9e-56fc7e79c0d1: !Template
     answer_choices: null
-    answer_choices_key: null
     id: cae58242-cde9-472d-ae9e-56fc7e79c0d1
     jinja: "List all the acryonyms in the following comma-separated tokens: \n\n{{tokens|join(',\
       \ ')}}\n|||\n{% set abbr_string=namespace(value='') %}\n{% set answer_list=namespace(value=[])\
@@ -142,7 +138,6 @@ templates:
     reference: Given the tokens, list the abbreviations.
   e4e42433-0e37-4aa5-bbce-7f336ecac6a3: !Template
     answer_choices: null
-    answer_choices_key: null
     id: e4e42433-0e37-4aa5-bbce-7f336ecac6a3
     jinja: "{% set _dummy = none %}\n{% set abbr_exp_dict = namespace(value = {})\
       \ %}\n{% set abbr_string=namespace(value='') %}\n{% set exp_string=namespace(value='')%}\n\
@@ -185,7 +180,6 @@ templates:
     reference: Given the tokens, find the abbreviation mapping.
   eed32ee4-ebc3-499f-ba61-e91461f56ccb: !Template
     answer_choices: null
-    answer_choices_key: null
     id: eed32ee4-ebc3-499f-ba61-e91461f56ccb
     jinja: "{% set random_exp = '' %}{% set _dummy = none %}{% set exp_abbr_dict =\
       \ namespace(value = {}) %}{% set abbr_string=namespace(value='') %}{% set exp_string=namespace(value='')%}{%\