Add MAP-Elites logging, island migration validation, and tests

Enhanced ProgramDatabase with detailed MAP-Elites cell logging, coverage milestones, and cell improvement events. Added validation and cleanup for island best program tracking and migration consistency. Improved Evaluator error context for cascade failures. Added comprehensive tests for cascade validation, island migration, and island best program tracking.

Author: codelion

openevolve/database.py

@@ -189,6 +189,28 @@ def add(
                 should_replace = self._is_better(program, self.programs[existing_program_id])
 
         if should_replace:
+            # Log significant MAP-Elites events
+            coords_dict = {self.config.feature_dimensions[i]: feature_coords[i] for i in range(len(feature_coords))}
+            
+            if feature_key not in self.feature_map:
+                # New cell occupation
+                logging.info("New MAP-Elites cell occupied: %s", coords_dict)
+                # Check coverage milestone
+                total_possible_cells = self.feature_bins ** len(self.config.feature_dimensions)
+                coverage = (len(self.feature_map) + 1) / total_possible_cells
+                if coverage in [0.1, 0.25, 0.5, 0.75, 0.9]:
+                    logging.info("MAP-Elites coverage reached %.1f%% (%d/%d cells)", 
+                               coverage * 100, len(self.feature_map) + 1, total_possible_cells)
+            else:
+                # Cell replacement - existing program being replaced
+                existing_program_id = self.feature_map[feature_key]
+                if existing_program_id in self.programs:
+                    existing_program = self.programs[existing_program_id]
+                    new_fitness = safe_numeric_average(program.metrics)
+                    existing_fitness = safe_numeric_average(existing_program.metrics)
+                    logging.info("MAP-Elites cell improved: %s (fitness: %.3f -> %.3f)", 
+                               coords_dict, existing_fitness, new_fitness)
+            
             self.feature_map[feature_key] = program.id
 
         # Add to specific island (not random!)
@@ -515,6 +537,9 @@ def _reconstruct_islands(self, saved_islands: List[List[str]]) -> None:
                 feature_keys_to_remove.append(key)
         for key in feature_keys_to_remove:
             del self.feature_map[key]
+        
+        # Clean up island best programs - remove stale references
+        self._cleanup_stale_island_bests()
 
         # Check best program
         if self.best_program_id and self.best_program_id not in self.programs:
@@ -641,7 +666,8 @@ def _calculate_feature_coords(self, program: Program) -> List[int]:
             else:
                 # Default to middle bin if feature not found
                 coords.append(self.feature_bins // 2)
-        logging.info(
+        # Only log coordinates at debug level for troubleshooting
+        logging.debug(
             "MAP-Elites coords: %s",
             str({self.config.feature_dimensions[i]: coords[i] for i in range(len(coords))}),
         )
@@ -1138,6 +1164,9 @@ def _enforce_population_limit(self, exclude_program_id: Optional[str] = None) ->
             logger.debug(f"Removed program {program_id} due to population limit")
 
         logger.info(f"Population size after cleanup: {len(self.programs)}")
+        
+        # Clean up any stale island best program references after removal
+        self._cleanup_stale_island_bests()
 
     # Island management methods
     def set_current_island(self, island_idx: int) -> None:
@@ -1215,13 +1244,102 @@ def migrate_programs(self) -> None:
                     # Update island-specific best program if migrant is better
                     self._update_island_best_program(migrant_copy, target_island)
 
-                    logger.debug(
-                        f"Migrated program {migrant.id} from island {i} to island {target_island}"
-                    )
+                    # Log migration with MAP-Elites coordinates
+                    feature_coords = self._calculate_feature_coords(migrant_copy)
+                    coords_dict = {self.config.feature_dimensions[j]: feature_coords[j] for j in range(len(feature_coords))}
+                    logger.info("Program migrated to island %d at MAP-Elites coords: %s", 
+                              target_island, coords_dict)
 
         # Update last migration generation
         self.last_migration_generation = max(self.island_generations)
         logger.info(f"Migration completed at generation {self.last_migration_generation}")
+        
+        # Validate migration results
+        self._validate_migration_results()
+
+    def _validate_migration_results(self) -> None:
+        """
+        Validate migration didn't create inconsistencies
+        
+        Checks that:
+        1. Program island metadata matches actual island assignment
+        2. No programs are assigned to multiple islands
+        3. All island best programs exist and are in correct islands
+        """
+        seen_program_ids = set()
+        
+        for i, island in enumerate(self.islands):
+            for program_id in island:
+                # Check for duplicate assignments
+                if program_id in seen_program_ids:
+                    logger.error(f"Program {program_id} assigned to multiple islands")
+                    continue
+                seen_program_ids.add(program_id)
+                
+                # Check program exists
+                if program_id not in self.programs:
+                    logger.warning(f"Island {i} contains nonexistent program {program_id}")
+                    continue
+                    
+                # Check metadata consistency
+                program = self.programs[program_id]
+                stored_island = program.metadata.get("island")
+                if stored_island != i:
+                    logger.warning(
+                        f"Island mismatch for program {program_id}: "
+                        f"in island {i} but metadata says {stored_island}"
+                    )
+        
+        # Validate island best programs
+        for i, best_id in enumerate(self.island_best_programs):
+            if best_id is not None:
+                if best_id not in self.programs:
+                    logger.warning(f"Island {i} best program {best_id} does not exist")
+                elif best_id not in self.islands[i]:
+                    logger.warning(f"Island {i} best program {best_id} not in island")
+
+    def _cleanup_stale_island_bests(self) -> None:
+        """
+        Remove stale island best program references
+        
+        Cleans up references to programs that no longer exist in the database
+        or are not actually in their assigned islands.
+        """
+        cleaned_count = 0
+        
+        for i, best_id in enumerate(self.island_best_programs):
+            if best_id is not None:
+                should_clear = False
+                
+                # Check if program still exists
+                if best_id not in self.programs:
+                    logger.debug(f"Clearing stale island {i} best program {best_id} (program deleted)")
+                    should_clear = True
+                # Check if program is still in the island
+                elif best_id not in self.islands[i]:
+                    logger.debug(f"Clearing stale island {i} best program {best_id} (not in island)")
+                    should_clear = True
+                
+                if should_clear:
+                    self.island_best_programs[i] = None
+                    cleaned_count += 1
+        
+        if cleaned_count > 0:
+            logger.info(f"Cleaned up {cleaned_count} stale island best program references")
+            
+            # Recalculate best programs for islands that were cleared
+            for i, best_id in enumerate(self.island_best_programs):
+                if best_id is None and len(self.islands[i]) > 0:
+                    # Find new best program for this island
+                    island_programs = [self.programs[pid] for pid in self.islands[i] if pid in self.programs]
+                    if island_programs:
+                        # Sort by fitness and update
+                        best_program = max(
+                            island_programs,
+                            key=lambda p: p.metrics.get("combined_score", safe_numeric_average(p.metrics))
+                        )
+                        self.island_best_programs[i] = best_program.id
+                        logger.debug(f"Recalculated island {i} best program: {best_program.id}")
 
     def get_island_stats(self) -> List[dict]:
         """Get statistics for each island"""

openevolve/evaluator.py

@@ -383,13 +383,14 @@ async def run_stage1():
                 )
             except Exception as e:
                 logger.error(f"Error in stage 1 evaluation: {str(e)}")
-                # Capture stage 1 failure as artifacts
+                # Capture stage 1 failure with enhanced context
+                error_context = self._create_cascade_error_context("stage1", e)
                 return EvaluationResult(
                     metrics={"stage1_passed": 0.0, "error": 0.0},
                     artifacts={
                         "stderr": str(e),
                         "traceback": traceback.format_exc(),
-                        "failure_stage": "stage1",
+                        **error_context,
                     },
                 )
 
@@ -510,13 +511,14 @@ async def run_stage3():
 
         except Exception as e:
             logger.error(f"Error in cascade evaluation: {str(e)}")
-            # Return proper cascade failure result instead of re-raising
+            # Return proper cascade failure result with enhanced context
+            error_context = self._create_cascade_error_context("cascade_setup", e)
             return EvaluationResult(
                 metrics={"stage1_passed": 0.0, "error": 0.0},
                 artifacts={
                     "stderr": str(e),
                     "traceback": traceback.format_exc(),
-                    "failure_stage": "cascade_setup",
+                    **error_context,
                 },
             )
 
@@ -611,6 +613,29 @@ async def _llm_evaluate(self, program_code: str, program_id: str = "") -> Dict[s
             traceback.print_exc()
             return {}
 
+    def _create_cascade_error_context(self, stage: str, error: Exception) -> dict:
+        """
+        Create rich error context for cascade failures
+        
+        Args:
+            stage: The stage where the error occurred
+            error: The exception that was raised
+            
+        Returns:
+            Dictionary with enhanced error context
+        """
+        import time
+        return {
+            "failure_stage": stage,
+            "error_type": type(error).__name__,
+            "error_message": str(error),
+            "timestamp": time.time(),
+            "cascade_config": self.config.cascade_evaluation,
+            "cascade_thresholds": getattr(self.config, 'cascade_thresholds', []),
+            "timeout_config": self.config.timeout,
+            "evaluation_file": self.evaluation_file,
+        }
+
     def _passes_threshold(self, metrics: Dict[str, float], threshold: float) -> bool:
         """
         Check if metrics pass a threshold