Skip to content

Commit a3a23fe

Browse files
jheerjheer
committed
Add Blade EMNLP'24 paper.
1 parent 0d80bb0 commit a3a23fe

File tree

4 files changed

+272
-0
lines changed

4 files changed

+272
-0
lines changed

‎static/images/figures/blade.png‎

159 KB
Loading

‎static/images/thumbs/blade.png‎

8.64 KB
Loading

‎static/papers/blade.json‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
{
2+
"doi": "10.48550/arXiv.2408.09667",
3+
"web_name": "blade",
4+
"title": "BLADE: Benchmarking Language Model Agents for Data-Driven Science",
5+
"venue": "EMNLP",
6+
"year": 2024,
7+
"note": "",
8+
"start_page": null,
9+
"end_page": null,
10+
"volume": null,
11+
"issue": null,
12+
"editors": "",
13+
"publisher": "",
14+
"location": "",
15+
"pdf": "https://arxiv.org/pdf/2408.09667",
16+
"abstract": "Data-driven scientific discovery requires the iterative integration of scientific domain knowledge, statistical expertise, and an understanding of data semantics to make nuanced analytical decisions, e.g., about which variables, transformations, and statistical models to consider. LM-based agents equipped with planning, memory, and code execution capabilities have the potential to support data-driven science. However, evaluating agents on such open-ended tasks is challenging due to multiple valid approaches, partially correct steps, and different ways to express the same decisions. To address these challenges, we present BLADE, a benchmark to automatically evaluate agents' multifaceted approaches to open-ended research questions. BLADE consists of 12 datasets and research questions drawn from existing scientific literature, with ground truth collected from independent analyses by expert data scientists and researchers. To automatically evaluate agent responses, we developed corresponding computational methods to match different representations of analyses to this ground truth. Though language models possess considerable world knowledge, our evaluation shows that they are often limited to basic analyses. However, agents capable of interacting with the underlying data demonstrate improved, but still non-optimal, diversity in their analytical decision making. Our work enables the evaluation of agents for data-driven science and provides researchers deeper insights into agents' analysis approaches.",
17+
"thumbnail": "images/thumbs/blade.png",
18+
"figure": "images/figures/blade.png",
19+
"caption": "Overview of BLADE. We gathered research questions and datasets from existing research papers, crowd-sourced analysis studies and statistic textbooks as well as analyses from expert annotators (boxes 1-2-3). Given a research question and dataset, LM agents generate a full analysis containing the relevant conceptual variables, a data transform function, and a statistical modeling function (boxes 1-4-5). BLADE automatically evaluates this against the ground truth (box 6).",
20+
"visible": true,
21+
"pub_date": "2024-11-12",
22+
"mod_date": "2024-11-12",
23+
"authors": [
24+
{
25+
"first_name": "Ken",
26+
"last_name": "Gu",
27+
"url": "https://kenqgu.com/"
28+
},
29+
{
30+
"first_name": "Ruoxi",
31+
"last_name": "Shang"
32+
},
33+
{
34+
"first_name": "Ruien",
35+
"last_name": "Jiang"
36+
},
37+
{
38+
"first_name": "Keying",
39+
"last_name": "Kuang"
40+
},
41+
{
42+
"first_name": "Richard-John",
43+
"last_name": "Lin"
44+
},
45+
{
46+
"first_name": "Donghe",
47+
"last_name": "Lyu"
48+
},
49+
{
50+
"first_name": "Yue",
51+
"last_name": "Mao"
52+
},
53+
{
54+
"first_name": "Youran",
55+
"last_name": "Pan"
56+
},
57+
{
58+
"first_name": "Teng",
59+
"last_name": "Wu"
60+
},
61+
{
62+
"first_name": "Jiaqian",
63+
"last_name": "Yu"
64+
},
65+
{
66+
"first_name": "Yikun",
67+
"last_name": "Zhang"
68+
},
69+
{
70+
"first_name": "Tianmai",
71+
"last_name": "Zhang",
72+
"display_name": "Tianmai M. Zhang"
73+
},
74+
{
75+
"first_name": "Lanyi",
76+
"last_name": "Zhu"
77+
},
78+
{
79+
"first_name": "Mike",
80+
"last_name": "Merrill"
81+
},
82+
{
83+
"first_name": "Jeffrey",
84+
"last_name": "Heer",
85+
"url": "http://homes.cs.washington.edu/~jheer/"
86+
},
87+
{
88+
"first_name": "Tim",
89+
"last_name": "Althoff",
90+
"url": "http://timalthoff.de/"
91+
}
92+
],
93+
"materials": [
94+
{
95+
"name": "Software",
96+
"link": "https://github.com/behavioral-data/BLADE"
97+
},
98+
{
99+
"name": "Website",
100+
"link": "https://blade-bench.github.io/"
101+
}
102+
],
103+
"tags": []
104+
}

‎static/people.json‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4968,5 +4968,173 @@
49684968
"position": null,
49694969
"interests": null,
49704970
"org": null
4971+
},
4972+
{
4973+
"first_name": "Ruoxi",
4974+
"last_name": "Shang",
4975+
"display_name": null,
4976+
"email": "",
4977+
"url": "",
4978+
"status": "other",
4979+
"visible": true,
4980+
"image": null,
4981+
"nickname": null,
4982+
"position": null,
4983+
"interests": null,
4984+
"org": null
4985+
},
4986+
{
4987+
"first_name": "Ruien",
4988+
"last_name": "Jiang",
4989+
"display_name": null,
4990+
"email": "",
4991+
"url": "",
4992+
"status": "other",
4993+
"visible": true,
4994+
"image": null,
4995+
"nickname": null,
4996+
"position": null,
4997+
"interests": null,
4998+
"org": null
4999+
},
5000+
{
5001+
"first_name": "Keying",
5002+
"last_name": "Kuang",
5003+
"display_name": null,
5004+
"email": "",
5005+
"url": "",
5006+
"status": "other",
5007+
"visible": true,
5008+
"image": null,
5009+
"nickname": null,
5010+
"position": null,
5011+
"interests": null,
5012+
"org": null
5013+
},
5014+
{
5015+
"first_name": "Richard-John",
5016+
"last_name": "Lin",
5017+
"display_name": null,
5018+
"email": "",
5019+
"url": "",
5020+
"status": "other",
5021+
"visible": true,
5022+
"image": null,
5023+
"nickname": null,
5024+
"position": null,
5025+
"interests": null,
5026+
"org": null
5027+
},
5028+
{
5029+
"first_name": "Donghe",
5030+
"last_name": "Lyu",
5031+
"display_name": null,
5032+
"email": "",
5033+
"url": "",
5034+
"status": "other",
5035+
"visible": true,
5036+
"image": null,
5037+
"nickname": null,
5038+
"position": null,
5039+
"interests": null,
5040+
"org": null
5041+
},
5042+
{
5043+
"first_name": "Yue",
5044+
"last_name": "Mao",
5045+
"display_name": null,
5046+
"email": "",
5047+
"url": "",
5048+
"status": "other",
5049+
"visible": true,
5050+
"image": null,
5051+
"nickname": null,
5052+
"position": null,
5053+
"interests": null,
5054+
"org": null
5055+
},
5056+
{
5057+
"first_name": "Youran",
5058+
"last_name": "Pan",
5059+
"display_name": null,
5060+
"email": "",
5061+
"url": "",
5062+
"status": "other",
5063+
"visible": true,
5064+
"image": null,
5065+
"nickname": null,
5066+
"position": null,
5067+
"interests": null,
5068+
"org": null
5069+
},
5070+
{
5071+
"first_name": "Teng",
5072+
"last_name": "Wu",
5073+
"display_name": null,
5074+
"email": "",
5075+
"url": "",
5076+
"status": "other",
5077+
"visible": true,
5078+
"image": null,
5079+
"nickname": null,
5080+
"position": null,
5081+
"interests": null,
5082+
"org": null
5083+
},
5084+
{
5085+
"first_name": "Jiaqian",
5086+
"last_name": "Yu",
5087+
"display_name": null,
5088+
"email": "",
5089+
"url": "",
5090+
"status": "other",
5091+
"visible": true,
5092+
"image": null,
5093+
"nickname": null,
5094+
"position": null,
5095+
"interests": null,
5096+
"org": null
5097+
},
5098+
{
5099+
"first_name": "Yikun",
5100+
"last_name": "Zhang",
5101+
"display_name": null,
5102+
"email": "",
5103+
"url": "",
5104+
"status": "other",
5105+
"visible": true,
5106+
"image": null,
5107+
"nickname": null,
5108+
"position": null,
5109+
"interests": null,
5110+
"org": null
5111+
},
5112+
{
5113+
"first_name": "Tianmai",
5114+
"last_name": "Zhang",
5115+
"display_name": "Tianmai M. Zhang",
5116+
"email": "",
5117+
"url": "",
5118+
"status": "other",
5119+
"visible": true,
5120+
"image": null,
5121+
"nickname": null,
5122+
"position": null,
5123+
"interests": null,
5124+
"org": null
5125+
},
5126+
{
5127+
"first_name": "Lanyi",
5128+
"last_name": "Zhu",
5129+
"display_name": null,
5130+
"email": "",
5131+
"url": "",
5132+
"status": "other",
5133+
"visible": true,
5134+
"image": null,
5135+
"nickname": null,
5136+
"position": null,
5137+
"interests": null,
5138+
"org": null
49715139
}
49725140
]

0 commit comments

Comments
 (0)