File tree Expand file tree Collapse file tree 1 file changed +10
-2
lines changed
Expand file tree Collapse file tree 1 file changed +10
-2
lines changed Original file line number Diff line number Diff line change @@ -28,5 +28,13 @@ bigcodebench.generate \
2828 --subset $SUBSET \
2929 --backend $BACKEND
3030
31- bigcodebench.sanitize --samples $FILE_HEADER .jsonl
32- bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER -sanitized.jsonl
31+ bigcodebench.sanitize --samples $FILE_HEADER .jsonl --calibrate
32+
33+ # Check if the ground truth works on your machine
34+ bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER -sanitized-calibrated.jsonl
35+
36+ # If the ground truth does not work, you can skip it:
37+ bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER -sanitized-calibrated.jsonl --no-gt
38+
39+ # If not the execution is slow:
40+ bigcodebench.evaluate --subset $SUBSET --samples $FILE_HEADER -sanitized-calibrated.jsonl --parallel 32
You can’t perform that action at this time.
0 commit comments