simplify metrics

Author: codelion

examples/function_minimization/config.yaml

@@ -5,11 +5,14 @@ log_level: "INFO"
 
 # LLM configuration
 llm:
-  primary_model: "gemini-2.0-flash-lite"
+  # primary_model: "gemini-2.0-flash-lite"
+  primary_model: "llama3.1-8b"
   primary_model_weight: 0.8
-  secondary_model: "gemini-2.0-flash"
+  # secondary_model: "gemini-2.0-flash"
+  secondary_model: "llama-4-scout-17b-16e-instruct"
   secondary_model_weight: 0.2
-  api_base: "https://generativelanguage.googleapis.com/v1beta/openai/"
+  # api_base: "https://generativelanguage.googleapis.com/v1beta/openai/"
+  api_base: "https://api.cerebras.ai/v1"
   temperature: 0.7
   top_p: 0.95
   max_tokens: 4096

examples/function_minimization/evaluator.py

@@ -154,15 +154,32 @@ def evaluate(program_path):
         # Add reliability score based on success rate
         reliability_score = float(success_count / num_trials)
 
-        # Calculate combined score
-        combined_score = float(0.5 * value_score + 0.2 * distance_score + 0.1 * speed_score + 0.2 * reliability_score)
+        # Calculate a single combined score that prioritizes finding good solutions
+        # over secondary metrics like speed and reliability
+        # Value and distance scores (quality of solution) get 90% of the weight
+        # Speed and reliability get only 10% combined
+        combined_score = float(0.6 * value_score + 0.3 * distance_score + 0.05 * speed_score + 0.05 * reliability_score)
+        
+        # Also compute an "overall" score that will be the primary metric for selection
+        # This adds a bonus for finding solutions close to the global minimum
+        # and heavily penalizes solutions that aren't finding the right region
+        if distance_to_global < 1.0:  # Very close to the correct solution
+            solution_quality = 1.0
+        elif distance_to_global < 3.0:  # In the right region
+            solution_quality = 0.5
+        else:  # Not finding the right region
+            solution_quality = 0.1
+            
+        # Overall score is dominated by solution quality but also factors in the combined score
+        overall_score = 0.8 * solution_quality + 0.2 * combined_score
 
         return {
             "value_score": value_score,
             "distance_score": distance_score,
             "speed_score": speed_score,
             "reliability_score": reliability_score,
             "combined_score": combined_score,
+            "overall_score": overall_score,  # This will be the primary selection metric
             "success_rate": reliability_score
         }
     except Exception as e:
@@ -222,11 +239,26 @@ def evaluate_stage1(program_path):
             y_diff = float(y) - GLOBAL_MIN_Y
             distance = float(np.sqrt(x_diff**2 + y_diff**2))
 
-            # Basic metrics
+            # Calculate value-based score
+            value_score = float(1.0 / (1.0 + abs(value - GLOBAL_MIN_VALUE)))
+            distance_score = float(1.0 / (1.0 + distance))
+            
+            # Calculate solution quality metric
+            if distance < 1.0:  # Very close to the correct solution
+                solution_quality = 1.0
+            elif distance < 3.0:  # In the right region
+                solution_quality = 0.5
+            else:  # Not finding the right region
+                solution_quality = 0.1
+            
+            # Basic metrics with overall score
             return {
                 "runs_successfully": 1.0,
                 "value": float(value),
-                "distance": distance
+                "distance": distance,
+                "value_score": value_score,
+                "distance_score": distance_score,
+                "overall_score": solution_quality  # This becomes a strong guiding metric
             }
         except TimeoutError as e:
             print(f"Stage 1 evaluation timed out: {e}")

openevolve/controller.py

@@ -276,6 +276,16 @@ async def run(
         if best_program is None:
             best_program = self.database.get_best_program()
             logger.info("Using calculated best program (tracked program not found)")
+            
+        # Check if there's a better program by combined_score that wasn't tracked
+        if "combined_score" in best_program.metrics:
+            best_by_combined = self.database.get_best_program(metric="combined_score")
+            if best_by_combined and best_by_combined.id != best_program.id and "combined_score" in best_by_combined.metrics:
+                # If the combined_score of this program is significantly better, use it instead
+                if best_by_combined.metrics["combined_score"] > best_program.metrics["combined_score"] + 0.02:
+                    logger.warning(f"Found program with better combined_score: {best_by_combined.id}")
+                    logger.warning(f"Score difference: {best_program.metrics['combined_score']:.4f} vs {best_by_combined.metrics['combined_score']:.4f}")
+                    best_program = best_by_combined
 
         if best_program:
             logger.info(

openevolve/database.py

@@ -159,7 +159,7 @@ def get_best_program(self, metric: Optional[str] = None) -> Optional[Program]:
         Get the best program based on a metric
         
         Args:
-            metric: Metric to use for ranking (uses average if None)
+            metric: Metric to use for ranking (uses combined_score or average if None)
             
         Returns:
             Best program or None if database is empty
@@ -169,6 +169,7 @@ def get_best_program(self, metric: Optional[str] = None) -> Optional[Program]:
 
         # If no specific metric and we have a tracked best program, return it
         if metric is None and self.best_program_id and self.best_program_id in self.programs:
+            logger.debug(f"Using tracked best program: {self.best_program_id}")
             return self.programs[self.best_program_id]
 
         if metric:
@@ -178,20 +179,40 @@ def get_best_program(self, metric: Optional[str] = None) -> Optional[Program]:
                 key=lambda p: p.metrics[metric],
                 reverse=True
             )
+            if sorted_programs:
+                logger.debug(f"Found best program by metric '{metric}': {sorted_programs[0].id}")
+        elif self.programs and all("combined_score" in p.metrics for p in self.programs.values()):
+            # Sort by combined_score if it exists (preferred method)
+            sorted_programs = sorted(
+                self.programs.values(),
+                key=lambda p: p.metrics["combined_score"],
+                reverse=True
+            )
+            if sorted_programs:
+                logger.debug(f"Found best program by combined_score: {sorted_programs[0].id}")
         else:
-            # Sort by average of all metrics
+            # Sort by average of all metrics as fallback
             sorted_programs = sorted(
                 self.programs.values(),
                 key=lambda p: sum(p.metrics.values()) / max(1, len(p.metrics)),
                 reverse=True
             )
+            if sorted_programs:
+                logger.debug(f"Found best program by average metrics: {sorted_programs[0].id}")
 
-            # Update the best program tracking if we found a better program
-            if sorted_programs and (self.best_program_id is None or 
-                                  sorted_programs[0].id != self.best_program_id):
-                old_id = self.best_program_id
-                self.best_program_id = sorted_programs[0].id
-                logger.info(f"Updated best program tracking: {self.best_program_id} ")
+        # Update the best program tracking if we found a better program
+        if sorted_programs and (self.best_program_id is None or 
+                             sorted_programs[0].id != self.best_program_id):
+            old_id = self.best_program_id
+            self.best_program_id = sorted_programs[0].id
+            logger.info(f"Updated best program tracking from {old_id} to {self.best_program_id}")
+            
+            # Also log the scores to help understand the update
+            if old_id and old_id in self.programs and "combined_score" in self.programs[old_id].metrics \
+               and "combined_score" in self.programs[self.best_program_id].metrics:
+                old_score = self.programs[old_id].metrics["combined_score"]
+                new_score = self.programs[self.best_program_id].metrics["combined_score"]
+                logger.info(f"Score change: {old_score:.4f} → {new_score:.4f} ({new_score-old_score:+.4f})")
 
         return sorted_programs[0] if sorted_programs else None
 
@@ -416,7 +437,11 @@ def _is_better(self, program1: Program, program2: Program) -> bool:
         if not program1.metrics and program2.metrics:
             return False
 
-        # Compare average of metrics
+        # Check for combined_score first (this is the preferred metric)
+        if "combined_score" in program1.metrics and "combined_score" in program2.metrics:
+            return program1.metrics["combined_score"] > program2.metrics["combined_score"]
+            
+        # Fallback to average of all metrics
         avg1 = sum(program1.metrics.values()) / len(program1.metrics)
         avg2 = sum(program2.metrics.values()) / len(program2.metrics)
 
@@ -466,18 +491,15 @@ def _update_best_program(self, program: Program) -> None:
         if self._is_better(program, current_best):
             old_id = self.best_program_id
             self.best_program_id = program.id
-            logger.info(f"New best program {program.id} replaces {old_id}")
 
-            # Log improvement in metrics
-            if program.metrics and current_best.metrics:
-                improvements = []
-                for metric, value in program.metrics.items():
-                    if metric in current_best.metrics:
-                        diff = value - current_best.metrics[metric]
-                        improvements.append(f"{metric}: {diff:+.4f}")
-                
-                if improvements:
-                    logger.info(f"Metric improvements: {', '.join(improvements)}")
+            # Log the change
+            if "combined_score" in program.metrics and "combined_score" in current_best.metrics:
+                old_score = current_best.metrics["combined_score"]
+                new_score = program.metrics["combined_score"]
+                score_diff = new_score - old_score
+                logger.info(f"New best program {program.id} replaces {old_id} (combined_score: {old_score:.4f} → {new_score:.4f}, +{score_diff:.4f})")
+            else:
+                logger.info(f"New best program {program.id} replaces {old_id}")
 
     def _sample_parent(self) -> Program:
         """