update: update the workflows

Author: terryyz

script/eval.py

@@ -0,0 +1,52 @@
+import argparse
+import os
+import json
+from tqdm import tqdm
+from wildcode.gen.util import trusted_exec
+from wildcode.eval.utils import (
+    create_tempdir,
+    reliability_guard,
+)
+def get_groundtruth(problems):
+    print("\nAsserting the groundtruth...")
+    if os.path.exists("checkpoint.txt"):
+        with open("checkpoint.txt", "r") as f:
+            checkpoint = int(f.read())
+    else:
+        checkpoint = 0
+    for i, problem in tqdm(enumerate(problems[checkpoint:]), total=len(problems[checkpoint:])):
+        task_id = problem["task_id"]
+        try:
+            with create_tempdir():
+                maximum_memory_bytes = 32 * 1024 * 1024 * 1024
+                reliability_guard(maximum_memory_bytes=maximum_memory_bytes)
+                trusted_exec(
+                    problem["prompt"] + "\n" + problem["clean_canonical_solution"],
+                    problem["test"],
+                    problem["entry_point"],
+                )
+        except:
+            if i > 0:
+                with open("checkpoint.txt", "w") as f:
+                    f.write(str(i+checkpoint))
+            raise Exception(f"Error in task data/raw/{task_id}")
+    
+
+def read_problems(jsonl_file):
+    with open(jsonl_file, "r") as f:
+        problems = []
+        for line in f:
+            data = json.loads(line)
+            problems.append(data)
+    return problems
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--samples", type=str, help="Path to the samples")
+    flags = parser.parse_args()
+
+    problems = read_problems(flags.samples)
+    get_groundtruth(problems)
+if __name__ == "__main__":
+    main()

script/run.sh

@@ -1,4 +1,4 @@
-NAMES=(chien jenny wenhao niklas hanhu ratna simon ming zhihan james)
+NAMES=(chien jenny wenhao niklas hanhu ratna simon ming zhihan james xiaoheng)
 
 for name in "${NAMES[@]}"; do
     # Copy all files for other names
@@ -8,6 +8,12 @@ done
 flake8 data/clean/*.py --select=E9,F63,F7,F82 --show-source --statistics
 python script/parse.py
 
+gzip -c data/open-eval.jsonl > data/open-eval.jsonl.gz
+
+# # used for WildCode evaluation
+# pip install -U wild-code
+# python script/eval.py --samples data/open-eval.jsonl
+
 for name in "${NAMES[@]}"; do
 
     for file in data/processed/*"$name"*wo_doc.py; do