Skip to content

Commit b2e82d7

Browse files
anyangmlanyangml
authored
Merge pull request #378 from anyangml2nd/feat/update-ood-datasets
Feat: add new ood datasets
2 parents 655a34e + 7b4ec88 commit b2e82d7

File tree

7 files changed

+177
-186
lines changed

7 files changed

+177
-186
lines changed

‎lambench/metrics/direct_task_weights.yml‎

Lines changed: 58 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,45 @@
1-
ANI:
1+
AQM:
22
domain: Molecules
33
energy_weight: 1.0
44
force_weight: 1.0
55
virial_weight: null
6-
energy_std: 0.19151186273205098
7-
force_std: 2.0784568378246826
8-
virial_std: null
6+
energy_std:
7+
wB97: 0.026886363885976483
8+
PBE: 0.0274237364941365
9+
force_std:
10+
wB97: 0.48079757014613167
11+
PBE: 0.36804974289536174
12+
virial_std:
13+
wB97: null
14+
PBE: null
15+
H_nature_2022:
16+
domain: Molecules
17+
energy_weight: 1.0
18+
force_weight: 1.0
19+
virial_weight: null
20+
energy_std:
21+
wB97: 0.39558742361833016
22+
PBE: 0.3680732527625157
23+
force_std:
24+
wB97: 1.9680702171137432
25+
PBE: 2.075534952128696
26+
virial_std:
27+
wB97 : null
28+
PBE: null
29+
AIMD_Chig:
30+
domain: Molecules
31+
energy_weight: 1.0
32+
force_weight: 1.0
33+
virial_weight: null
34+
energy_std:
35+
wB97: 0.011235040126809573
36+
PBE: 0.009639542241251055
37+
force_std:
38+
wB97: 0.8434356010200169
39+
PBE: 0.8628550274940282
40+
virial_std:
41+
wB97: null
42+
PBE: null
943
HEA25_S:
1044
domain: Inorganic Materials
1145
energy_weight: 1.0
@@ -30,13 +64,29 @@ MoS2:
3064
energy_std: 0.08167024857360158
3165
force_std: 0.9568415738836734
3266
virial_std: 0.4161647190798736
33-
MD22:
34-
domain: Molecules
67+
CompressBi:
68+
domain: Inorganic Materials
69+
energy_weight: 1.0
70+
force_weight: 1.0
71+
virial_weight: 1.0
72+
energy_std: 0.13320054684238294
73+
force_std: 0.667987100991502
74+
virial_std: 0.21138606885531494
75+
Carbon_growth:
76+
domain: Catalysis
77+
energy_weight: 1.0
78+
force_weight: 1.0
79+
virial_weight: 1.0
80+
energy_std: 0.6062335773750094
81+
force_std: 1.6286800187054977
82+
virial_std: 0.43007347637274035
83+
In2O3_CO2:
84+
domain: Catalysis
3585
energy_weight: 1.0
3686
force_weight: 1.0
3787
virial_weight: null
38-
energy_std: 0.008959619353114803
39-
force_std: 1.1964522496892305
88+
energy_std: 0.04897980313266302
89+
force_std: 0.3860350371558003
4090
virial_std: null
4191
REANN_CO2_Ni100:
4292
domain: Catalysis
@@ -54,14 +104,6 @@ NequIP_NC_2022:
54104
energy_std: 0.007043606595996931
55105
force_std: 0.9434989207476847
56106
virial_std: null
57-
AIMD-Chig:
58-
domain: Molecules
59-
energy_weight: 1.0
60-
force_weight: 1.0
61-
virial_weight: null
62-
energy_std: 0.00937981704049093
63-
force_std: 0.8699872017197637
64-
virial_std: null
65107
Cu_MgO_catalysts:
66108
domain: Catalysis
67109
energy_weight: 1.0
@@ -94,62 +136,3 @@ Si_ZEO22:
94136
energy_std: 0.035252909678594395
95137
force_std: 1.232529293429265
96138
virial_std: null
97-
98-
99-
## DEPRECATED
100-
# CGM_MLP_NC2023:
101-
# domain: Catalysis
102-
# energy_weight: 1.0
103-
# force_weight: 1.0
104-
# virial_weight: null
105-
# energy_std: 0.9875093490539156
106-
# force_std: 2.022155629243275
107-
# virial_std: null
108-
# Collision:
109-
# domain: Reactions
110-
# energy_weight: 1.0
111-
# force_weight: 1.0
112-
# virial_weight: null
113-
# energy_std: 0.390200350320283
114-
# force_std: 2.021733350505997
115-
# virial_std: null
116-
# H_nature_2022:
117-
# domain: Reactions
118-
# energy_weight: 1.0
119-
# force_weight: 1.0
120-
# virial_weight: null
121-
# energy_std: 0.34163860101107846
122-
# force_std: 2.056435289694542
123-
# virial_std: null
124-
# WBM_downsampled:
125-
# domain: Inorganic Materials
126-
# energy_weight: 1.0
127-
# force_weight: null
128-
# virial_weight: null
129-
# energy_std: 0.3743104865117501
130-
# force_std: null
131-
# virial_std: null
132-
# Subalex_9k:
133-
# domain: Inorganic Materials
134-
# energy_weight: 1.0
135-
# force_weight: 1.0
136-
# virial_weight: 1.0
137-
# energy_std: 0.7749643377228371
138-
# force_std: 1.1503770816187873
139-
# virial_std: 0.8678699239404154
140-
# HEMC_HEMB:
141-
# domain: Inorganic Materials
142-
# energy_weight: 1.0
143-
# force_weight: 1.0
144-
# virial_weight: 1.0
145-
# energy_std: 0.4750117425061965
146-
# force_std: 1.8089415904253994
147-
# virial_std: 0.4589409203427954
148-
# Torsionnet500:
149-
# domain: Molecules
150-
# energy_weight: 1.0
151-
# force_weight: 1.0
152-
# virial_weight: null
153-
# energy_std: 0.033879357854299824
154-
# force_std: 0.05088788030453556
155-
# virial_std: null

‎lambench/metrics/vishelper/results_fetcher.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,18 @@ def aggregate_ood_results_for_one_model(
4040
task_result = DirectPredictRecord.query(
4141
model_name=model.model_name, task_name=task
4242
)
43-
task_config = DIRECT_TASK_WEIGHTS[task]
43+
task_config = DIRECT_TASK_WEIGHTS[task].copy()
44+
45+
if task in ["AQM", "H_nature_2022", "AIMD_Chig"]:
46+
if model.supports_omol:
47+
task_config["energy_std"] = task_config["energy_std"]["wB97"]
48+
task_config["force_std"] = task_config["force_std"]["wB97"]
49+
task_config["virial_std"] = task_config["virial_std"]["wB97"]
50+
else:
51+
task_config["energy_std"] = task_config["energy_std"]["PBE"]
52+
task_config["force_std"] = task_config["force_std"]["PBE"]
53+
task_config["virial_std"] = task_config["virial_std"]["PBE"]
54+
4455
if task_config["virial_weight"] is not None:
4556
weight_virial = True
4657
if len(task_result) != 1:

‎lambench/tasks/direct/direct_tasks.yml‎

Lines changed: 32 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,39 @@
1+
AQM:
2+
test_data:
3+
PBE: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/AQM_PBE"
4+
wB97: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/AQM_wB97"
5+
H_nature_2022:
6+
test_data:
7+
PBE: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/H2combustion_PBE"
8+
wB97: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/H2combustion_wB97"
9+
AIMD_Chig:
10+
test_data:
11+
PBE: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/AIMDChig_PBE"
12+
wB97: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/AIMDChig_wB97"
113
HEA25_S:
2-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/HEA25S"
14+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/HEA25S"
315
HEA25_bulk:
4-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/HEA25"
16+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/HEA25"
517
MoS2:
6-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/MoS2"
7-
REANN_CO2_Ni100:
8-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/REANN_CO2_Ni100"
9-
NequIP_NC_2022:
10-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/NequIP_NC_2022"
11-
AIMD-Chig:
12-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/AIMD_chig"
13-
Cu_MgO_catalysts:
14-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/Cu_MgO_CO2"
18+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/MoS2"
19+
CompressBi:
20+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/CompressBi"
1521
dispersion_correction: d3zero
22+
NequIP_NC_2022:
23+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/NequIP_NC_2022"
24+
Ca_batteries_CM2021:
25+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/Ca_batteries"
1626
Si_ZEO22:
17-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/Si_ZEO22"
27+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/Si_ZEO22"
1828
dispersion_correction: d3bj
29+
REANN_CO2_Ni100:
30+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/REANN_CO2_Ni100"
31+
Cu_MgO_catalysts:
32+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/Cu_MgO_CO2"
33+
dispersion_correction: d3zero
1934
HPt_NC_2022:
20-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/HPt_NC2022"
21-
Ca_batteries_CM2021:
22-
test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/Ca_batteries"
23-
AQM:
24-
test_data:
25-
PBE: "/bohr/temp-lambench-ood-5zz5/v3/AQM-sol-PBE__downsampled_1000"
26-
wB97: "/bohr/temp-lambench-ood-5zz5/v3/AQM-sol-PBE__downsampled_1000_OMol-wb97mv-def2tzvpd-ORCA600"
27-
## DEPRECATED
28-
# Collision:
29-
# test_data: "/bohr/lambench-ood-zwtr/v2/LAMBench-TestData-v2/Collision"
30-
# H_nature_2022:
31-
# test_data: "/bohr/lambench-ood-zwtr/v2/LAMBench-TestData-v2/H_nature"
32-
# Subalex_9k:
33-
# test_data: "/bohr/lambench-ood-zwtr/v1/OOD_test_data_v2/subalex_downsample_9k"
34-
# WBM_downsampled:
35-
# test_data: "/bohr/lambench-ood-zwtr/v1/OOD_test_data_v2/WBM_downsampled"
36-
# CGM_MLP_NC2023:
37-
# test_data: "/bohr/lambench-ood-zwtr/v3/LAMBench-TestData-v3/CGM_MLP"
38-
# HEMC_HEMB:
39-
# test_data: "/bohr/lambench-ood-zwtr/v1/OOD_test_data_v2/HEMC_HEMB"
40-
# Torsionnet500:
41-
# test_data: "/bohr/lambench-ood-zwtr/v1/OOD_test_data_v2/raw_torsionnet500"
42-
# ANI:
43-
# test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/ANI"
44-
# MD22:
45-
# test_data: "/bohr/lambench-ood-zwtr/v4/LAMBench-TestData-v3/MD22"
35+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/HPt_NC2022"
36+
Carbon_growth:
37+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/carbon_film_growth"
38+
In2O3_CO2:
39+
test_data: "/bohr/lambench-ood-zwtr/v5/LAMBench-TestData-v4/In2O3_CO2"

‎lambench/workflow/entrypoint.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def gather_task_type(
7474
# model_domain = task.domain if task.domain else "" # in the future we may have tasks with specific domain.
7575

7676
# currently only need to distinguish direct tasks for molecules and materials due to OMol25 training set.
77-
if task_name in []: # to be added in a separate PR.
77+
if task_name in ["AQM", "H_nature_2022", "AIMD_Chig"]:
7878
model_domain = "molecules"
7979
else:
8080
model_domain = "materials"

0 commit comments

Comments
 (0)