|
| 1 | +""" |
| 2 | +OpenEvolve <-> lm-evaluation-harness adapter |
| 3 | +
|
| 4 | +Implements generation only, no loglikelihood. Tasks such as GSM8K / BoolQ / MMLU-Math / |
| 5 | +AQUA-RAT and most code suites should work fine because they grade on the generated |
| 6 | +answer string. |
| 7 | +""" |
| 8 | + |
| 9 | +from __future__ import annotations |
| 10 | +import subprocess, tempfile, json, os, argparse, math, pathlib |
| 11 | +from pathlib import Path |
| 12 | +from typing import List, Dict, Tuple, Any, Iterable |
| 13 | + |
| 14 | +import lm_eval |
| 15 | +from lm_eval.tasks import TaskManager |
| 16 | +from lm_eval.evaluator import evaluate |
| 17 | +from lm_eval.api.model import LM |
| 18 | +from lm_eval.api.registry import register_model |
| 19 | +from datetime import datetime |
| 20 | + |
| 21 | +# cd to the parent parent directory of this file |
| 22 | +os.chdir(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) |
| 23 | + |
| 24 | +PIPELINE_CMD = ["python3", "openevolve-run.py"] |
| 25 | + |
| 26 | +@register_model("openevolve") |
| 27 | +class OpenEvolve(LM): |
| 28 | + def __init__( |
| 29 | + self, |
| 30 | + init_file: str = "initial_content_stub.txt", |
| 31 | + evaluator_file: str = "evaluator_stub.py", |
| 32 | + config_file: str = "config.yml", |
| 33 | + iterations: int = 5, |
| 34 | + extra_param: List[str] = [], |
| 35 | + **kwargs, |
| 36 | + ): |
| 37 | + super().__init__() |
| 38 | + self.init_file = init_file |
| 39 | + self.evaluator_file = evaluator_file |
| 40 | + self.iterations = iterations |
| 41 | + self.extra_param = extra_param |
| 42 | + self.config_file = config_file |
| 43 | + |
| 44 | + # folder must match prompt:template_dir in config.yml! |
| 45 | + self.prompt_path = "scripts/lm_eval/prompts/system_message.txt" |
| 46 | + self.evaluator_prompt_path = "scripts/lm_eval/prompts/evaluator_system_message.txt" |
| 47 | + self.best_path = "scripts/lm_eval/openevolve_output/best/best_program.txt" |
| 48 | + self.base_system_message = "You are an expert task solver, with a lot of commonsense, math, language and coding knowledge.\n\nConsider this task:\n```{prompt}´´´" |
| 49 | + |
| 50 | + def generate(self, prompts: List[str], max_gen_toks: int = None, stop=None, **kwargs): |
| 51 | + outs = [] |
| 52 | + for prompt in prompts: |
| 53 | + # Task prompt becomes the system message. User prompt is the evolutionary logic. |
| 54 | + # We create temporary prompt files with the system message |
| 55 | + with Path(self.prompt_path).open("w") as f: |
| 56 | + f.write(self.base_system_message.format(prompt=prompt)) |
| 57 | + |
| 58 | + with Path(self.evaluator_prompt_path).open("w") as f: |
| 59 | + f.write(self.base_system_message.format(prompt=prompt)) |
| 60 | + |
| 61 | + cmd = ( |
| 62 | + PIPELINE_CMD |
| 63 | + + ["--config", self.config_file] |
| 64 | + + ["--iterations", str(self.iterations)] |
| 65 | + + self.extra_param |
| 66 | + + [self.init_file, self.evaluator_file] |
| 67 | + ) |
| 68 | + print(f"Running command: {' '.join(cmd)}") |
| 69 | + try: |
| 70 | + res = subprocess.run(cmd, capture_output=True, text=True, check=True) |
| 71 | + text = res.stdout.strip() |
| 72 | + print(f"Process output: {text}") |
| 73 | + except subprocess.CalledProcessError as e: |
| 74 | + print(f"Command failed with return code {e.returncode}") |
| 75 | + print(f"stderr: {e.stderr}") |
| 76 | + text = "" |
| 77 | + |
| 78 | + print(f"# Prompt: {prompt}") |
| 79 | + with Path(self.best_path).open("r") as f: |
| 80 | + best = f.read().strip() |
| 81 | + print(f"# Answer: {best}") |
| 82 | + |
| 83 | + # honour stop tokens |
| 84 | + if stop: |
| 85 | + for s in stop: |
| 86 | + idx = best.find(s) |
| 87 | + if idx != -1: |
| 88 | + best = best[:idx] |
| 89 | + break |
| 90 | + outs.append(best) |
| 91 | + return outs |
| 92 | + |
| 93 | + # for tasks that ask for log likelihood, indicate that it is unsupported |
| 94 | + def loglikelihood(self, requests: Iterable[Tuple[str, str]], **kw): |
| 95 | + # return [(-math.inf, False) for _ in requests] |
| 96 | + raise NotImplementedError |
| 97 | + |
| 98 | + def loglikelihood_rolling(self, requests: Iterable[str], **kw): |
| 99 | + # return [(-math.inf, False) for _ in requests] |
| 100 | + raise NotImplementedError |
| 101 | + |
| 102 | + def generate_until(self, requests: Iterable[Any], **kw) -> List[str]: |
| 103 | + ctxs, stops = [], [] |
| 104 | + |
| 105 | + for req in requests: |
| 106 | + # ---------------- old: plain tuple ---------------- |
| 107 | + if isinstance(req, tuple): |
| 108 | + ctx, until = req |
| 109 | + |
| 110 | + # -------------- new: Instance object -------------- |
| 111 | + else: |
| 112 | + ctx = req.args[0] # first positional arg |
| 113 | + until = [] |
| 114 | + # if a second positional arg exists and is list-like, |
| 115 | + # treat it as the stop sequence |
| 116 | + if len(req.args) > 1 and isinstance(req.args[1], (list, tuple)): |
| 117 | + until = list(req.args[1]) |
| 118 | + |
| 119 | + ctxs.append(ctx) |
| 120 | + stops.append(until) |
| 121 | + |
| 122 | + # 2) run your real generator once per context |
| 123 | + gens = self.generate(ctxs, stop=None) |
| 124 | + |
| 125 | + # 3) post-trim at the first stop sequence |
| 126 | + cleaned = [] |
| 127 | + for g, until in zip(gens, stops): |
| 128 | + for s in until: |
| 129 | + idx = g.find(s) |
| 130 | + if idx != -1: |
| 131 | + g = g[:idx] |
| 132 | + break |
| 133 | + cleaned.append(g) |
| 134 | + return cleaned |
| 135 | + |
| 136 | +if __name__ == "__main__": |
| 137 | + # cli arguments for primary model, secondary model, iterations, config and tasks |
| 138 | + p = argparse.ArgumentParser( |
| 139 | + description="OpenEvolve <-> lm-evaluation-harness adapter.", |
| 140 | + ) |
| 141 | + p.add_argument("--config", default="scripts/lm_eval/config.yml", help="config file") |
| 142 | + p.add_argument( |
| 143 | + "--init_file", |
| 144 | + default="scripts/lm_eval/initial_content_stub.txt", |
| 145 | + help="initial content file", |
| 146 | + ) |
| 147 | + p.add_argument( |
| 148 | + "--evaluator_file", default="scripts/lm_eval/evaluator_stub.py", help="evaluator file" |
| 149 | + ) |
| 150 | + p.add_argument("--iterations", default=5, type=int, help="number of iterations") |
| 151 | + p.add_argument("--limit", default=None, type=int, help="limit the number of examples per task that are executed") |
| 152 | + # p.add_argument("--tasks", default="boolq,gsm8k,mmlu", help="comma-list of tasks to evaluate") |
| 153 | + p.add_argument("--tasks", default="gsm8k", help="list of tasks to evaluate") |
| 154 | + p.add_argument("--output_path", default="results", help="output path for results") |
| 155 | + args = p.parse_args() |
| 156 | + |
| 157 | + lm_obj = OpenEvolve( |
| 158 | + init_file=args.init_file, |
| 159 | + evaluator_file=args.evaluator_file, |
| 160 | + iterations=args.iterations, |
| 161 | + config_file=args.config, |
| 162 | + ) |
| 163 | + |
| 164 | + task_dict = lm_eval.tasks.get_task_dict(args.tasks.split(",")) |
| 165 | + |
| 166 | + results = evaluate( |
| 167 | + lm=lm_obj, |
| 168 | + task_dict=task_dict, |
| 169 | + limit=args.limit, |
| 170 | + ) |
| 171 | + |
| 172 | + # write out the results |
| 173 | + pathlib.Path( |
| 174 | + args.output_path, |
| 175 | + ).mkdir(exist_ok=True) |
| 176 | + |
| 177 | + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") |
| 178 | + results_path = pathlib.Path(os.path.join( |
| 179 | + args.output_path, |
| 180 | + f"{timestamp}_iter{args.iterations}.json", |
| 181 | + )) |
| 182 | + |
| 183 | + with results_path.open("w") as f: |
| 184 | + json.dump(results, f, indent=2) |
| 185 | + |
| 186 | + # print result summary |
| 187 | + short = {} |
| 188 | + for task, metrics in results["results"].items(): |
| 189 | + # pick the first value that is a real number |
| 190 | + for key, val in metrics.items(): |
| 191 | + if isinstance(val, (int, float)): |
| 192 | + short[task] = (key, val) # store *both* name & value |
| 193 | + break |
| 194 | + |
| 195 | + print(f"Full results written to {results_path}\n") |
| 196 | + print("Headline metrics:") |
| 197 | + for task, (name, value) in short.items(): |
| 198 | + print(f" {task:<15} {name:<12} {value:.3%}") |
| 199 | + |
| 200 | + print("\nNote: Never cite the overall average when some components were skipped!") |
0 commit comments