We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 2dad280 commit 8ada805Copy full SHA for 8ada805
lmms_eval/tasks/scienceqa/utils.py
@@ -34,11 +34,11 @@ def sqa_doc_to_target(doc):
34
def sqa_process_results(doc, results):
35
# I know this is weird, but it's how llava parse it.
36
target = sqa_doc_to_target(doc).strip().lower()
37
- pred = results[0].strip().lower()
38
- if pred == target:
+ pred = results[0].strip()
+ if pred.lower() == target:
39
return {"exact_match": 1.0}
40
# pattern: ^[A-Z]\. .*
41
if len(pred) >= 2 and pred[0].isupper() and pred[1] == ".":
42
- result = 1.0 if pred[0] == target else 0.0
+ result = 1.0 if pred[0].lower() == target else 0.0
43
return {"exact_match": result}
44
return {"exact_match": 0.0}
0 commit comments