update

Author: amberlu

eval/data/background_comment_template.txt

@@ -21,6 +21,8 @@ RESPONSE GUIDELINES:
 5. Ensure your response is in the format of ```python``` and includes the necessary background as a comment at the top.
 
 Example:
+```python
 # Background: [Here, insert the necessary scientific knowledge required for the next step.]
 
 [Insert the Python code here based on the provided function header and dependencies.]
+```

eval/scripts/test_generated_code.py

@@ -116,7 +116,7 @@ def run_script(script_path):
     print(f'correct problems: {correct_prob_num}/{DEV_PROB_NUM if dev_set else PROB_NUM - DEV_PROB_NUM}')
     print(f'correct steps: {len(correct_step)}/{DEV_STEP_NUM if dev_set else STEP_NUM}')
 
-    output_dir.mkdir(parents=True, exist_ok=True)
+    Path(f'{output_dir}/{Path(model_name)}').mkdir(parents=True, exist_ok=True)
 
     with open(f'{output_dir}/{model_name}.txt', 'w') as f:
         f.write(f'correct problems: {correct_prob_num}/{DEV_PROB_NUM if dev_set else PROB_NUM - DEV_PROB_NUM}\n')

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.1.txt

@@ -0,0 +1 @@
+fail

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.10.txt

@@ -0,0 +1 @@
+pass

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.11.txt

@@ -0,0 +1 @@
+fail

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.12.txt

@@ -0,0 +1 @@
+fail

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.2.txt

@@ -0,0 +1 @@
+fail

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.3.txt

@@ -0,0 +1 @@
+fail

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.4.txt

@@ -0,0 +1 @@
+fail

logs/litellm/together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo/11.5.txt

@@ -0,0 +1 @@
+fail