We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent d38f640 commit a5cc345Copy full SHA for a5cc345
lmms_eval/tasks/vitatecs/utils.py
@@ -130,7 +130,7 @@ def vitatecs_process_results(doc, result):
130
elif any(pred.startswith(prefix) for prefix in ["A)", "B)"]):
131
rating = 1 if pred.split(")")[0] == answer[1] else 0
132
elif any(pred.startswith(prefix) for prefix in ["(A)", "(B)"]):
133
- rating = 1 if pred.split(")")[1] == answer[1] else 0
+ rating = 1 if pred.split(")")[0][1] == answer[1] else 0
134
else:
135
# Fail to match answer in the video-llm response. Use ChatGPT to evaluate.
136
match_success = False
0 commit comments