Merge branch 'main' into feat-lm-eval

Author: jvm123

examples/function_minimization/evaluator.py

@@ -5,8 +5,9 @@
 import importlib.util
 import numpy as np
 import time
-import multiprocessing
+import concurrent.futures
 import traceback
+import signal
 
 
 def run_with_timeout(func, args=(), kwargs={}, timeout_seconds=5):
@@ -22,31 +23,13 @@ def run_with_timeout(func, args=(), kwargs={}, timeout_seconds=5):
     Returns:
         Result of the function or raises TimeoutError
     """
-
-    def wrapper(queue, func, args, kwargs):
+    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(func, *args, **kwargs)
         try:
-            result = func(*args, **kwargs)
-            queue.put(("success", result))
-        except Exception as e:
-            queue.put(("error", e))
-
-    queue = multiprocessing.Queue()
-    process = multiprocessing.Process(target=wrapper, args=(queue, func, args, kwargs))
-    process.start()
-    process.join(timeout=timeout_seconds)
-
-    if process.is_alive():
-        process.terminate()
-        process.join()
-        raise TimeoutError(f"Function timed out after {timeout_seconds} seconds")
-
-    if queue.empty():
-        raise TimeoutError("Function ended without returning a result")
-
-    status, result = queue.get()
-    if status == "error":
-        raise result
-    return result
+            result = future.result(timeout=timeout_seconds)
+            return result
+        except concurrent.futures.TimeoutError:
+            raise TimeoutError(f"Function timed out after {timeout_seconds} seconds")
 
 
 def safe_float(value):
@@ -107,15 +90,27 @@ def evaluate(program_path):
                 # Run with timeout
                 result = run_with_timeout(program.run_search, timeout_seconds=5)
 
-                # Check if we got a tuple of 3 values
-                if not isinstance(result, tuple) or len(result) != 3:
+                # Handle different result formats
+                if isinstance(result, tuple):
+                    if len(result) == 3:
+                        x, y, value = result
+                    elif len(result) == 2:
+                        # Assume it's (x, y) and calculate value
+                        x, y = result
+                        # Calculate the function value since it wasn't returned
+                        value = np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20
+                        print(f"Trial {trial}: Got 2 values, calculated function value: {value}")
+                    else:
+                        print(
+                            f"Trial {trial}: Invalid result format, expected tuple of 2 or 3 values but got {len(result)}"
+                        )
+                        continue
+                else:
                     print(
-                        f"Trial {trial}: Invalid result format, expected tuple of 3 values but got {type(result)}"
+                        f"Trial {trial}: Invalid result format, expected tuple but got {type(result)}"
                     )
                     continue
 
-                x, y, value = result
-
                 end_time = time.time()
 
                 # Ensure all values are float
@@ -264,15 +259,25 @@ def evaluate_stage1(program_path):
             # Run a single trial with timeout
             result = run_with_timeout(program.run_search, timeout_seconds=5)
 
-            # Check if we got a tuple of 3 values
-            if not isinstance(result, tuple) or len(result) != 3:
-                print(
-                    f"Stage 1: Invalid result format, expected tuple of 3 values but got {type(result)}"
-                )
+            # Handle different result formats
+            if isinstance(result, tuple):
+                if len(result) == 3:
+                    x, y, value = result
+                elif len(result) == 2:
+                    # Assume it's (x, y) and calculate value
+                    x, y = result
+                    # Calculate the function value since it wasn't returned
+                    value = np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20
+                    print(f"Stage 1: Got 2 values, calculated function value: {value}")
+                else:
+                    print(
+                        f"Stage 1: Invalid result format, expected tuple of 2 or 3 values but got {len(result)}"
+                    )
+                    return {"runs_successfully": 0.0, "error": "Invalid result format"}
+            else:
+                print(f"Stage 1: Invalid result format, expected tuple but got {type(result)}")
                 return {"runs_successfully": 0.0, "error": "Invalid result format"}
 
-            x, y, value = result
-
             # Ensure all values are float
             x = safe_float(x)
             y = safe_float(y)

examples/function_minimization/requirements.txt

@@ -0,0 +1 @@
+scipy

openevolve/controller.py

@@ -24,6 +24,10 @@
     parse_evolve_blocks,
     parse_full_rewrite,
 )
+from openevolve.utils.format_utils import (
+    format_metrics_safe,
+    format_improvement_safe,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -314,9 +318,7 @@ async def run(
                     logger.info(
                         f"🌟 New best solution found at iteration {i+1}: {child_program.id}"
                     )
-                    logger.info(
-                        f"Metrics: {', '.join(f'{name}={value:.4f}' for name, value in child_program.metrics.items())}"
-                    )
+                    logger.info(f"Metrics: {format_metrics_safe(child_program.metrics)}")
 
                 # Save checkpoint
                 if (i + 1) % self.config.checkpoint_interval == 0:
@@ -371,7 +373,7 @@ async def run(
         if best_program:
             logger.info(
                 f"Evolution complete. Best program has metrics: "
-                f"{', '.join(f'{name}={value:.4f}' for name, value in best_program.metrics.items())}"
+                f"{format_metrics_safe(best_program.metrics)}"
             )
 
             # Save the best program (using our tracked best program)
@@ -399,19 +401,13 @@ def _log_iteration(
             child: Child program
             elapsed_time: Elapsed time in seconds
         """
-        # Calculate improvement
-        improvement = {}
-        for metric, value in child.metrics.items():
-            if metric in parent.metrics:
-                diff = value - parent.metrics[metric]
-                improvement[metric] = diff
-
-        improvement_str = ", ".join(f"{name}={diff:+.4f}" for name, diff in improvement.items())
+        # Calculate improvement using safe formatting
+        improvement_str = format_improvement_safe(parent.metrics, child.metrics)
 
         logger.info(
             f"Iteration {iteration+1}: Child {child.id} from parent {parent.id} "
             f"in {elapsed_time:.2f}s. Metrics: "
-            f"{', '.join(f'{name}={value:.4f}' for name, value in child.metrics.items())} "
+            f"{format_metrics_safe(child.metrics)} "
             f"(Δ: {improvement_str})"
         )
 
@@ -467,7 +463,7 @@ def _save_checkpoint(self, iteration: int) -> None:
 
             logger.info(
                 f"Saved best program at checkpoint {iteration} with metrics: "
-                f"{', '.join(f'{name}={value:.4f}' for name, value in best_program.metrics.items())}"
+                f"{format_metrics_safe(best_program.metrics)}"
             )
 
         logger.info(f"Saved checkpoint at iteration {iteration} to {checkpoint_path}")

openevolve/database.py

@@ -15,6 +15,7 @@
 
 from openevolve.config import DatabaseConfig
 from openevolve.utils.code_utils import calculate_edit_distance
+from openevolve.utils.metrics_utils import safe_numeric_average
 
 logger = logging.getLogger(__name__)
 
@@ -227,10 +228,10 @@ def get_best_program(self, metric: Optional[str] = None) -> Optional[Program]:
             if sorted_programs:
                 logger.debug(f"Found best program by combined_score: {sorted_programs[0].id}")
         else:
-            # Sort by average of all metrics as fallback
+            # Sort by average of all numeric metrics as fallback
             sorted_programs = sorted(
                 self.programs.values(),
-                key=lambda p: sum(p.metrics.values()) / max(1, len(p.metrics)),
+                key=lambda p: safe_numeric_average(p.metrics),
                 reverse=True,
             )
             if sorted_programs:
@@ -281,10 +282,10 @@ def get_top_programs(self, n: int = 10, metric: Optional[str] = None) -> List[Pr
                 reverse=True,
             )
         else:
-            # Sort by average of all metrics
+            # Sort by average of all numeric metrics
             sorted_programs = sorted(
                 self.programs.values(),
-                key=lambda p: sum(p.metrics.values()) / max(1, len(p.metrics)),
+                key=lambda p: safe_numeric_average(p.metrics),
                 reverse=True,
             )
 
@@ -436,7 +437,7 @@ def _calculate_feature_coords(self, program: Program) -> List[int]:
                 if not program.metrics:
                     bin_idx = 0
                 else:
-                    avg_score = sum(program.metrics.values()) / len(program.metrics)
+                    avg_score = safe_numeric_average(program.metrics)
                     bin_idx = min(int(avg_score * self.feature_bins), self.feature_bins - 1)
                 coords.append(bin_idx)
             elif dim in program.metrics:
@@ -487,9 +488,9 @@ def _is_better(self, program1: Program, program2: Program) -> bool:
         if "combined_score" in program1.metrics and "combined_score" in program2.metrics:
             return program1.metrics["combined_score"] > program2.metrics["combined_score"]
 
-        # Fallback to average of all metrics
-        avg1 = sum(program1.metrics.values()) / len(program1.metrics)
-        avg2 = sum(program2.metrics.values()) / len(program2.metrics)
+        # Fallback to average of all numeric metrics
+        avg1 = safe_numeric_average(program1.metrics)
+        avg2 = safe_numeric_average(program2.metrics)
 
         return avg1 > avg2
 
@@ -507,9 +508,7 @@ def _update_archive(self, program: Program) -> None:
 
         # Otherwise, find worst program in archive
         archive_programs = [self.programs[pid] for pid in self.archive]
-        worst_program = min(
-            archive_programs, key=lambda p: sum(p.metrics.values()) / max(1, len(p.metrics))
-        )
+        worst_program = min(archive_programs, key=lambda p: safe_numeric_average(p.metrics))
 
         # Replace if new program is better
         if self._is_better(program, worst_program):
@@ -716,7 +715,7 @@ def _enforce_population_limit(self) -> None:
         # Sort by average metric (worst first)
         sorted_programs = sorted(
             all_programs,
-            key=lambda p: sum(p.metrics.values()) / max(1, len(p.metrics)) if p.metrics else 0.0,
+            key=lambda p: safe_numeric_average(p.metrics),
         )
 
         # Remove worst programs, but never remove the best program
@@ -811,9 +810,7 @@ def migrate_programs(self) -> None:
 
             # Sort by fitness (using combined_score or average metrics)
             island_programs.sort(
-                key=lambda p: p.metrics.get(
-                    "combined_score", sum(p.metrics.values()) / max(1, len(p.metrics))
-                ),
+                key=lambda p: p.metrics.get("combined_score", safe_numeric_average(p.metrics)),
                 reverse=True,
             )
 
@@ -858,9 +855,7 @@ def get_island_stats(self) -> List[dict]:
 
             if island_programs:
                 scores = [
-                    p.metrics.get(
-                        "combined_score", sum(p.metrics.values()) / max(1, len(p.metrics))
-                    )
+                    p.metrics.get("combined_score", safe_numeric_average(p.metrics))
                     for p in island_programs
                 ]
 

openevolve/evaluator.py

@@ -20,6 +20,7 @@
 from openevolve.llm.ensemble import LLMEnsemble
 from openevolve.utils.async_utils import TaskPool, run_in_executor
 from openevolve.prompt.sampler import PromptSampler
+from openevolve.utils.format_utils import format_metrics_safe
 
 logger = logging.getLogger(__name__)
 
@@ -123,7 +124,7 @@ async def evaluate_program(
                 elapsed = time.time() - start_time
                 logger.info(
                     f"Evaluated program{program_id_str} in {elapsed:.2f}s: "
-                    f"{', '.join(f'{name}={value:.4f}' for name, value in metrics.items())}"
+                    f"{format_metrics_safe(metrics)}"
                 )
 
                 return metrics