Skip to content

Commit 57bebba

Browse files
tangxiangruVictorSanh
tangxiangru
and
VictorSanh
authored
Updated the existing prompts (#704)
* Updated the existing prompts * minor fixes + metrics * add metrics + answer choices when relevant and minor changes * add metrics Co-authored-by: Victor Sanh <[email protected]>
1 parent 3de4cab commit 57bebba

File tree

3 files changed

+200
-128
lines changed

3 files changed

+200
-128
lines changed

‎promptsource/templates/billsum/templates.yaml‎

Lines changed: 78 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,104 +1,137 @@
11
dataset: billsum
22
templates:
3-
3ac01292-4a54-4546-b4e6-c225ae114213: !Template
3+
0938c6e4-dbaf-43d8-8d8f-4bc62489ae74: !Template
44
answer_choices: null
5-
id: 3ac01292-4a54-4546-b4e6-c225ae114213
6-
jinja: 'Summarize: {{text}}|||
5+
id: 0938c6e4-dbaf-43d8-8d8f-4bc62489ae74
6+
jinja: 'Given the title: "{{title}}" and the summary of a bill: {{summary}}.
77
8-
Title: {{title}}
8+
Write this bill based on the title and summary.
99
10-
Summary: {{summary}}'
10+
|||
11+
12+
{{text}}'
1113
metadata: !TemplateMetadata
12-
choices_in_prompt: null
13-
metrics: []
14-
original_task: null
15-
name: 'Summarize: (text-> title,summary)'
14+
choices_in_prompt: false
15+
metrics:
16+
- BLEU
17+
- ROUGE
18+
original_task: false
19+
name: 'Write a bill: (title, summary->text)'
1620
reference: ''
1721
3c790ac3-0557-47a9-9b71-1cb435f15629: !Template
1822
answer_choices: null
1923
id: 3c790ac3-0557-47a9-9b71-1cb435f15629
20-
jinja: 'Summarize this bill: {{text}} |||
21-
22-
{{title}}'
24+
jinja: "Given a state bill: {{text}}. \nPlease write the title of this bill in\
25+
\ one sentence.\n|||\n{{title}}"
2326
metadata: !TemplateMetadata
24-
choices_in_prompt: null
25-
metrics: []
26-
original_task: null
27+
choices_in_prompt: false
28+
metrics:
29+
- BLEU
30+
- ROUGE
31+
original_task: false
2732
name: 'Summarize this bill in one sentence: (text-> title)'
2833
reference: ''
2934
438192e5-d67a-4098-9d82-a9fe892f6be2: !Template
3035
answer_choices: null
3136
id: 438192e5-d67a-4098-9d82-a9fe892f6be2
32-
jinja: 'Write a bill: {{summary}} |||
37+
jinja: 'Given a summary of a bill: {{summary}}.
38+
39+
Write this bill.
40+
41+
|||
3342
3443
{{text}}'
3544
metadata: !TemplateMetadata
36-
choices_in_prompt: null
37-
metrics: []
38-
original_task: null
45+
choices_in_prompt: false
46+
metrics:
47+
- BLEU
48+
- ROUGE
49+
original_task: false
3950
name: 'Write a bill: (summary-> text)'
4051
reference: ''
4152
4891a8e7-258c-41e2-80d3-0c1a054acb07: !Template
4253
answer_choices: null
4354
id: 4891a8e7-258c-41e2-80d3-0c1a054acb07
44-
jinja: 'Write a bill: {{title}} |||
55+
jinja: 'Given a title: "{{title}}" of a bill.
56+
57+
Write this bill based on this title.
58+
59+
|||
4560
4661
{{text}}'
4762
metadata: !TemplateMetadata
48-
choices_in_prompt: null
49-
metrics: []
50-
original_task: null
63+
choices_in_prompt: false
64+
metrics:
65+
- BLEU
66+
- ROUGE
67+
original_task: false
5168
name: 'Write a bill: (title-> text)'
5269
reference: ''
5370
550fa161-af4e-4430-9844-ce7dad587733: !Template
5471
answer_choices: null
5572
id: 550fa161-af4e-4430-9844-ce7dad587733
56-
jinja: 'Summarize this bill: {{text}} |||
73+
jinja: 'Given this bill: {{text}}.
74+
75+
Write a summary of this bill.
76+
77+
|||
5778
5879
{{summary}}'
5980
metadata: !TemplateMetadata
60-
choices_in_prompt: null
61-
metrics: []
62-
original_task: null
81+
choices_in_prompt: false
82+
metrics:
83+
- BLEU
84+
- ROUGE
85+
original_task: true
6386
name: 'Summarize this bill: (text-> summary)'
6487
reference: ''
6588
5d2404b9-63ff-406e-977d-eda6afb5c689: !Template
6689
answer_choices: null
6790
id: 5d2404b9-63ff-406e-977d-eda6afb5c689
68-
jinja: '{{summary}}
69-
70-
===
91+
jinja: 'Given a summary: {{summary}}, we want to generate a title from this summary.
7192
72-
Generate title from summary:
93+
|||
7394
74-
|||{{title}}'
95+
{{title}}'
7596
metadata: !TemplateMetadata
76-
choices_in_prompt: null
77-
metrics: []
78-
original_task: null
97+
choices_in_prompt: false
98+
metrics:
99+
- BLEU
100+
- ROUGE
101+
original_task: false
79102
name: Generate title from summary
80103
reference: ''
81104
6a439a80-4924-49e9-b5ae-f661683b399f: !Template
82105
answer_choices: null
83106
id: 6a439a80-4924-49e9-b5ae-f661683b399f
84-
jinja: 'Summarize: {{text}}
107+
jinja: 'Summarize this US bill: {{text}}
108+
109+
|||
85110
86-
|||{{summary}}'
111+
{{summary}}'
87112
metadata: !TemplateMetadata
88-
choices_in_prompt: null
89-
metrics: []
90-
original_task: null
113+
choices_in_prompt: false
114+
metrics:
115+
- BLEU
116+
- ROUGE
117+
original_task: true
91118
name: 'Summarize: (text -> summary )'
92119
reference: ''
93120
ea9f0376-6cec-450c-b258-89f479cb9f6d: !Template
94121
answer_choices: null
95122
id: ea9f0376-6cec-450c-b258-89f479cb9f6d
96-
jinja: 'Summarize: {{summary}}
123+
jinja: 'Given a summary of a bill: {{summary}}.
124+
125+
Please write the title of this summary.
97126
98-
|||{{title}}'
127+
|||
128+
129+
{{title}}'
99130
metadata: !TemplateMetadata
100-
choices_in_prompt: null
101-
metrics: []
102-
original_task: null
131+
choices_in_prompt: false
132+
metrics:
133+
- BLEU
134+
- ROUGE
135+
original_task: false
103136
name: 'Summarize: (summary -> title)'
104137
reference: ''

‎promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml‎

Lines changed: 75 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,102 +1,113 @@
11
dataset: pubmed_qa
22
subset: pqa_labeled
33
templates:
4+
00584766-2415-4d10-ab76-bf86058faa07: !Template
5+
answer_choices: null
6+
id: 00584766-2415-4d10-ab76-bf86058faa07
7+
jinja: "Given a research abstract: {{ context.contexts | join(\", \") }}\nAnd\
8+
\ given keywords: {{context.meshes | join(\", \")}}.\n \nWhat is the question\
9+
\ answered by the above research abstract? \n|||\n{{question}} "
10+
metadata: !TemplateMetadata
11+
choices_in_prompt: false
12+
metrics:
13+
- BLEU
14+
- ROUGE
15+
original_task: false
16+
name: Generate Question Title with meshes
17+
reference: ''
418
00f58886-e04a-4efb-bf41-cfcbd00a5e7d: !Template
519
answer_choices: null
620
id: 00f58886-e04a-4efb-bf41-cfcbd00a5e7d
7-
jinja: "\"{{ context.contexts | join(\", \") }}\"\n\nAnswer the following question.\n\
8-
\nQ: \"{{question}}\" ||| \n{{long_answer}}\n"
21+
jinja: "Given a PubMed abstract: {{ context.contexts | join(\", \") }}\n\nAnswer\
22+
\ this question: \"{{question}}\" \n||| \n{{long_answer}}\n"
923
metadata: !TemplateMetadata
10-
choices_in_prompt: null
11-
metrics: []
12-
original_task: null
24+
choices_in_prompt: false
25+
metrics:
26+
- BLEU
27+
- ROUGE
28+
- Other
29+
original_task: false
1330
name: Question Answering (Long)
1431
reference: Provide a long/verbose answer to the provided question
1532
0b630e04-02a8-46d6-b164-a41cd34042ff: !Template
1633
answer_choices: null
1734
id: 0b630e04-02a8-46d6-b164-a41cd34042ff
18-
jinja: '"{{ context.contexts | join(", ") }}"
19-
20-
21-
What is the main question answered by the above research abstract? |||
22-
23-
{{question}} '
35+
jinja: "Given a research abstract: {{ context.contexts | join(\", \") }}\n\nWhat\
36+
\ is the question answered by the above research abstract? \n|||\n{{question}} "
2437
metadata: !TemplateMetadata
25-
choices_in_prompt: null
26-
metrics: []
27-
original_task: null
38+
choices_in_prompt: false
39+
metrics:
40+
- BLEU
41+
- ROUGE
42+
original_task: false
2843
name: 'Generate Question Title '
2944
reference: Given abstract, generate title (which is in the form of a question)
3045
1e0a77f8-0eb4-40a1-814d-8a111df66e5e: !Template
31-
answer_choices: null
46+
answer_choices: 'Yes ||| No ||| Maybe'
3247
id: 1e0a77f8-0eb4-40a1-814d-8a111df66e5e
33-
jinja: "Q: \"{{ question }}\" \n\nA: \"{{ long_answer }}\"\n\nSummarize the above\
34-
\ answer as: YES, NO or MAYBE ? |||\n{{final_decision}}"
48+
jinja: "Question: \"{{ question }}\" \n\nAnswer: \"{{ long_answer }}\"\n\nSummarize\
49+
\ the above answer as YES, NO, or MAYBE? \n|||\n{{final_decision}}"
3550
metadata: !TemplateMetadata
36-
choices_in_prompt: null
37-
metrics: []
38-
original_task: null
51+
choices_in_prompt: true
52+
metrics:
53+
- Accuracy
54+
original_task: false
3955
name: Long Answer to Final Decision
4056
reference: Given a question, the full text of the relevant answer, summarize a
4157
yes/no/maybe answer
4258
21240f74-530a-47b7-a5d9-a6a13083b72e: !Template
43-
answer_choices: null
59+
answer_choices: '{{context.labels | join("|||")}}'
4460
id: 21240f74-530a-47b7-a5d9-a6a13083b72e
45-
jinja: '{% set n_sections = context.contexts | length %}
46-
47-
{% set choice = range(0, n_sections) | random %}
48-
49-
50-
"{{ context.contexts[choice] }}"
51-
52-
53-
In a research article, the above text would most likely be found in which section: {{
54-
context.labels[:-1] | join(", ") }} or {{ context.labels[-1] }} ? |||
55-
56-
57-
{{ context.labels[choice] }}
58-
59-
'
61+
jinja: "{% set n_sections = context.contexts | length %}\n{% set choice = range(0,\
62+
\ n_sections) | random %}\n\n\"{{ context.contexts[choice] }}\"\n\nThe above\
63+
\ text would most likely be found in which section of a biomedical paper: {{\
64+
\ context.labels[:-1] | join(\", \") }} or {{ context.labels[-1] }} ? \n|||\n\
65+
{{ context.labels[choice] }}\n"
6066
metadata: !TemplateMetadata
61-
choices_in_prompt: null
62-
metrics: []
63-
original_task: null
67+
choices_in_prompt: false
68+
metrics:
69+
- Accuracy
70+
original_task: false
6471
name: Context Section Type
6572
reference: Assign text in an abstract to specific paper section headers
6673
45cb344c-bb36-492a-ace0-7cfc897e127a: !Template
6774
answer_choices: null
6875
id: 45cb344c-bb36-492a-ace0-7cfc897e127a
69-
jinja: '"{{ context.contexts | join(", ") }}"
70-
71-
72-
What are the MeSH terms for this PubMed abstract? |||
73-
74-
{{ context.meshes | join(", ") }}'
76+
jinja: "Given a PubMed abstract:{{ context.contexts | join(\", \") }}\n\nWhat\
77+
\ are the MeSH (Medical Subject Headings) terms for this? \n|||\n{{ context.meshes | join(\", \") }}"
7578
metadata: !TemplateMetadata
76-
choices_in_prompt: null
77-
metrics: []
78-
original_task: null
79+
choices_in_prompt: false
80+
metrics:
81+
- BLEU
82+
- ROUGE
83+
- Other
84+
original_task: false
7985
name: Medical Subject Headings
8086
reference: 'Predict the set of MeSH terms for a given PubMed abstract '
81-
91d481e5-fac6-4532-b013-5ac1235b6e1a: !Template
87+
48ee58bb-6a4a-4667-9d9c-69618408c6ce: !Template
8288
answer_choices: null
89+
id: 48ee58bb-6a4a-4667-9d9c-69618408c6ce
90+
jinja: "Given a research abstract: {{ context.contexts | join(\", \") }}\nAnd\
91+
\ given long answer of a question: {{long_answer}}.\n \nWhat is the question asked by this research paper?\
92+
\ \n|||\n{{question}} "
93+
metadata: !TemplateMetadata
94+
choices_in_prompt: false
95+
metrics:
96+
- BLEU
97+
- ROUGE
98+
original_task: false
99+
name: Generate Question Title with long answer
100+
reference: ''
101+
91d481e5-fac6-4532-b013-5ac1235b6e1a: !Template
102+
answer_choices: 'Yes ||| No ||| Maybe'
83103
id: 91d481e5-fac6-4532-b013-5ac1235b6e1a
84-
jinja: '"{{ context.contexts | join(", ") }}"
85-
86-
87-
Answer the following question as YES, NO, MAYBE
88-
89-
90-
Q: "{{question}}"
91-
92-
93-
A: ||| {{final_decision}}
94-
95-
96-
'
104+
jinja: "Given a PubMed abstract: {{ context.contexts | join(\", \") }}\n\nAnswer\
105+
\ the question: \"{{question}}\" as YES, NO, MAYBE.\n||| \n{{final_decision}}\n\
106+
\n"
97107
metadata: !TemplateMetadata
98-
choices_in_prompt: null
99-
metrics: []
108+
choices_in_prompt: false
109+
metrics:
110+
- Accuracy
100111
original_task: true
101112
name: Question Answering (Short)
102113
reference: Answer the following question using the provided abstract text

0 commit comments

Comments
 (0)