Skip to content

Commit 86854b5

Browse files
craffelcraffel
authored
Add more prompts to LAMBADA (#535)
* Add more prompts to LAMBADA * Fix answer_choices_key * Fix syntax and remove LM style * Add me to the hub users * Add prompts for OpenAI variant of LAMBADA * Fix corrupted templates * Don't check features when there are no features * Remove duplicates * Add craffel/openai_lambada dataset to seqio tasks * Fix splitting and answer_choices_key
1 parent 7147aaf commit 86854b5

File tree

6 files changed

+153
-9
lines changed

6 files changed

+153
-9
lines changed

‎promptsource/app.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ def get_infos(d_name):
229229
"Dataset",
230230
dataset_list,
231231
key="dataset_select",
232-
index=12, # AG_NEWS
232+
index=4, # AG_NEWS
233233
help="Select the dataset to work on.",
234234
)
235235

‎promptsource/seqio_tasks/experiment_D4.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ imdb,,sentiment,cls,,,TRUE,TRUE,,25000,25000,TRUE,,25000,,,,,,imdb,cls/sentiment
8383
rotten_tomatoes,,sentiment,cls,,,TRUE,TRUE,,8530,8530,TRUE,,8530,,,,,,rotten tomatoes,cls/sentiment analysis,,,,Pang and Lee 2005
8484
yelp_review_full,,sentiment,cls,no dev set,,TRUE,TRUE,,650000,500000,TRUE,,500000,,,,,,yelp review full,other/regression,,,,Zhang et al. 2015; (link)
8585
lambada,,story_completion,gen,revisit: story or cloze or coref? trivial cloze prompt; training set is just unlabeled corpora; GPT task,GPT,,,TRUE,0,0,,TRUE,0,accuracy;perplexity;median_rank,,https://arxiv.org/pdf/1606.06031.pdf,,,,,,,,
86+
craffel/openai_lambada,,story_completion,gen,revisit: story or cloze or coref? trivial cloze prompt; training set is just unlabeled corpora; GPT task,GPT,,,TRUE,0,0,,TRUE,0,accuracy;perplexity;median_rank,,https://arxiv.org/pdf/1606.06031.pdf,,,,,,,,
8687
story_cloze,2016,story_completion,cls,todo: custom loading; swag like?,GPT,,,TRUE,,0,,TRUE,0,accuracy,,https://arxiv.org/pdf/1604.01696.pdf,,,,,,,,
8788
hellaswag,,story_completion,cls,,GPT,,,TRUE,39905,0,TRUE,,39905,accuracy,,https://arxiv.org/pdf/1905.07830.pdf,,,hellaswag,qa/multiple-choice qa,,,,Zellers et al. 2019
8889
common_gen,,structure_to_text,gen,,,TRUE,TRUE,,67389,67389,TRUE,,67389,,,,,,common gen,other,,,,Lin et al. 2020b

‎promptsource/templates.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
# These are users whose datasets should be included in the results returned by
2727
# filter_english_datasets (regardless of their metadata)
28-
INCLUDED_USERS = {"Zaid"}
28+
INCLUDED_USERS = {"Zaid", "craffel"}
2929

3030

3131
def highlight(input):

‎promptsource/templates/craffel/openai_lambada/templates.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
dataset: craffel/openai_lambada
2+
templates:
3+
1ee5ddef-fffb-4b73-a2f7-f600ffac63cb: !Template
4+
answer_choices: null
5+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
6+
id: 1ee5ddef-fffb-4b73-a2f7-f600ffac63cb
7+
jinja: '{{ text.split()[:-1] | join('' '') }}...
8+
9+
10+
What comes after the ellipses? ||| {{ text.split()[-1] }}'
11+
metadata: !TemplateMetadata
12+
choices_in_prompt: false
13+
metrics:
14+
- Accuracy
15+
original_task: true
16+
name: ellipses
17+
reference: ''
18+
4f08e9d4-bcff-4bc0-9902-87c497625d17: !Template
19+
answer_choices: null
20+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
21+
id: 4f08e9d4-bcff-4bc0-9902-87c497625d17
22+
jinja: 'Fill in the blank:
23+
24+
25+
{{ text.split()[:-1] | join('' '') }} ____. ||| {{ text.split()[-1]
26+
}}'
27+
metadata: !TemplateMetadata
28+
choices_in_prompt: false
29+
metrics:
30+
- Accuracy
31+
original_task: true
32+
name: GPT-3 style
33+
reference: Brown et al.
34+
507de732-8298-4971-bac3-7d768d511a31: !Template
35+
answer_choices: null
36+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
37+
id: 507de732-8298-4971-bac3-7d768d511a31
38+
jinja: '{{ text.split()[:-1] | join('' '') }} ____.
39+
40+
41+
Fill in the ____: ||| {{ text.split()[-1] }}'
42+
metadata: !TemplateMetadata
43+
choices_in_prompt: false
44+
metrics:
45+
- Accuracy
46+
original_task: true
47+
name: fill in the ____
48+
reference: ''
49+
774b4349-0524-4a34-881b-b344f8f5c34e: !Template
50+
answer_choices: null
51+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
52+
id: 774b4349-0524-4a34-881b-b344f8f5c34e
53+
jinja: 'This story got cut short. What comes next?
54+
55+
56+
{{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
57+
metadata: !TemplateMetadata
58+
choices_in_prompt: false
59+
metrics:
60+
- Accuracy
61+
original_task: true
62+
name: what comes next
63+
reference: ''
64+
ef072a60-252e-4c52-aa8a-4152bb4dd83c: !Template
65+
answer_choices: null
66+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
67+
id: ef072a60-252e-4c52-aa8a-4152bb4dd83c
68+
jinja: 'Please predict the next word after the following chunk of text.
69+
70+
71+
{{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
72+
metadata: !TemplateMetadata
73+
choices_in_prompt: false
74+
metrics:
75+
- Accuracy
76+
original_task: true
77+
name: please next word
78+
reference: ''

‎promptsource/templates/lambada/templates.yaml‎

Lines changed: 67 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,78 @@
11
dataset: lambada
22
templates:
3+
3747e80a-4182-44eb-944b-dee40095bb17: !Template
4+
answer_choices: null
5+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
6+
id: 3747e80a-4182-44eb-944b-dee40095bb17
7+
jinja: 'Please predict the next word after the following chunk of text.
8+
9+
10+
{{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
11+
metadata: !TemplateMetadata
12+
choices_in_prompt: false
13+
metrics:
14+
- Accuracy
15+
original_task: true
16+
name: please next word
17+
reference: ''
18+
506765b8-17c0-4946-bbb0-b28288caacb3: !Template
19+
answer_choices: null
20+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
21+
id: 506765b8-17c0-4946-bbb0-b28288caacb3
22+
jinja: '{{ text.split()[:-1] | join('' '') }} ____.
23+
24+
25+
Fill in the ____: ||| {{ text.split()[-1] }}'
26+
metadata: !TemplateMetadata
27+
choices_in_prompt: false
28+
metrics:
29+
- Accuracy
30+
original_task: true
31+
name: fill in the ____
32+
reference: ''
33+
948664d5-2ea2-4245-b656-9283948dd5cd: !Template
34+
answer_choices: null
35+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
36+
id: 948664d5-2ea2-4245-b656-9283948dd5cd
37+
jinja: '{{ text.split()[:-1] | join('' '') }}...
38+
39+
40+
What comes after the ellipses? ||| {{ text.split()[-1] }}'
41+
metadata: !TemplateMetadata
42+
choices_in_prompt: false
43+
metrics:
44+
- Accuracy
45+
original_task: true
46+
name: ellipses
47+
reference: ''
48+
acfe374c-60ce-4354-b285-e7b0717cffe5: !Template
49+
answer_choices: null
50+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
51+
id: acfe374c-60ce-4354-b285-e7b0717cffe5
52+
jinja: 'This story got cut short. What comes next?
53+
54+
55+
{{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
56+
metadata: !TemplateMetadata
57+
choices_in_prompt: false
58+
metrics:
59+
- Accuracy
60+
original_task: true
61+
name: what comes next
62+
reference: ''
363
d5707bd9-d3cc-4535-b4c1-5c2aee8cb8c7: !Template
464
answer_choices: null
5-
answer_choices_key: null
65+
answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
666
id: d5707bd9-d3cc-4535-b4c1-5c2aee8cb8c7
7-
jinja: '{{ text.split('' '')[:-1] | join('' '') }} ||| {{ text.split('' '')[-1]
67+
jinja: 'Fill in the blank:
68+
69+
70+
{{ text.split()[:-1] | join('' '') }} ____. ||| {{ text.split()[-1]
871
}}'
972
metadata: !TemplateMetadata
1073
choices_in_prompt: false
1174
metrics:
1275
- Accuracy
1376
original_task: true
14-
name: predict the last word
15-
reference: ''
77+
name: GPT-3 style
78+
reference: Brown et al.

‎test/test_templates.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,10 @@ def test_dataset(dataset):
6767
else:
6868
raise e
6969

70-
features = builder_instance.info.features.keys()
71-
features = set([feature.replace("-", "_") for feature in features])
70+
has_features = builder_instance.info.features is not None
71+
if has_features:
72+
features = builder_instance.info.features.keys()
73+
features = set([feature.replace("-", "_") for feature in features])
7274

7375
# Initializes sets for checking uniqueness among templates
7476
template_name_set = set()
@@ -89,7 +91,7 @@ def test_dataset(dataset):
8991

9092
variables = meta.find_undeclared_variables(parse)
9193
for variable in variables:
92-
if variable not in features and variable != "answer_choices":
94+
if has_features and variable not in features and variable != "answer_choices":
9395
raise ValueError(f"Template for dataset {dataset_name}/{subset_name} "
9496
f"with uuid {template.get_id()} has unrecognized variable {variable}.")
9597

0 commit comments

Comments
 (0)