Moved scripts/lm_eval/ to examples/lm_eval/

Author: jvm123

.gitignore

@@ -1,6 +1,6 @@
 results/
-scripts/lm_eval/prompts/system_message.txt
-scripts/lm_eval/prompts/evaluator_system_message.txt
+examples/lm_eval/prompts/system_message.txt
+examples/lm_eval/prompts/evaluator_system_message.txt
 
 # Python
 __pycache__/

scripts/lm_eval/README.md renamed to examples/lm_eval/README.md

@@ -7,7 +7,7 @@
 ## Usage
 
 ```bash
-$ python3 scripts/lm_eval/lm-eval.py -h
+$ python3 examples/lm_eval/lm-eval.py -h
 usage: lm-eval.py [-h] [--config CONFIG] [--init_file INIT_FILE] [--evaluator_file EVALUATOR_FILE] [--iterations ITERATIONS] [--limit LIMIT] [--tasks TASKS]
                   [--output_path OUTPUT_PATH]
 
@@ -30,26 +30,26 @@ options:
 
 Early examples that **were meant to** indicate that more evolution iterations improve task performance -- I suspect the prompting may not be ideal yet:
 ```
-$ python3 scripts/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 1
+$ python3 examples/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 1
 [..]
 Headline metrics:
   gsm8k           exact_match,strict-match 80.000%
 [..]
 
 
-$ python3 scripts/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 3
+$ python3 examples/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 3
 [..]
 Headline metrics:
   gsm8k           exact_match,strict-match 90.000%
 [..]
 
-$ python3 scripts/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 10
+$ python3 examples/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 10
 [..]
 Headline metrics:
   gsm8k           exact_match,strict-match 80.000%
 [..]
 
-$ python3 scripts/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 15
+$ python3 examples/lm_eval/lm-eval.py --tasks gsm8k --limit 10 --iterations 15
 [..]
 Headline metrics:
   gsm8k           exact_match,strict-match 70.000%

scripts/lm_eval/config.yml renamed to examples/lm_eval/config.yml

@@ -23,7 +23,7 @@ prompt:
   num_top_programs: 3
   use_template_stochasticity: true
   # System prompt is created dynamically during the benchmark in file system_message.txt!
-  template_dir: "scripts/lm_eval/prompts"
+  template_dir: "examples/lm_eval/prompts"
 
 # Database configuration
 database:

scripts/lm_eval/evaluator_stub.py renamed to examples/lm_eval/evaluator_stub.py

@@ -42,9 +42,9 @@ def __init__(
         self.config_file = config_file
 
         # folder must match prompt:template_dir in config.yml!
-        self.prompt_path = "scripts/lm_eval/prompts/system_message.txt"
-        self.evaluator_prompt_path = "scripts/lm_eval/prompts/evaluator_system_message.txt"
-        self.best_path = "scripts/lm_eval/openevolve_output/best/best_program.txt"
+        self.prompt_path = "examples/lm_eval/prompts/system_message.txt"
+        self.evaluator_prompt_path = "examples/lm_eval/prompts/evaluator_system_message.txt"
+        self.best_path = "examples/lm_eval/openevolve_output/best/best_program.txt"
         self.base_system_message = "You are an expert task solver, with a lot of commonsense, math, language and coding knowledge.\n\nConsider this task:\n```{prompt}´´´"
 
     def generate(self, prompts: List[str], max_gen_toks: int = None, stop=None, **kwargs):
@@ -138,14 +138,14 @@ def generate_until(self, requests: Iterable[Any], **kw) -> List[str]:
     p = argparse.ArgumentParser(
         description="OpenEvolve <-> lm-evaluation-harness adapter.",
     )
-    p.add_argument("--config", default="scripts/lm_eval/config.yml", help="config file")
+    p.add_argument("--config", default="examples/lm_eval/config.yml", help="config file")
     p.add_argument(
         "--init_file",
-        default="scripts/lm_eval/initial_content_stub.txt",
+        default="examples/lm_eval/initial_content_stub.txt",
         help="initial content file",
     )
     p.add_argument(
-        "--evaluator_file", default="scripts/lm_eval/evaluator_stub.py", help="evaluator file"
+        "--evaluator_file", default="examples/lm_eval/evaluator_stub.py", help="evaluator file"
     )
     p.add_argument("--iterations", default=5, type=int, help="number of iterations")
     p.add_argument("--limit", default=None, type=int, help="limit the number of examples per task that are executed")