fix

Author: codelion

examples/matrix_multiplication/evaluate.py

@@ -8,6 +8,7 @@
 import logging
 import sys
 import time
+import traceback
 from typing import Dict, List, Tuple, Any
 
 import numpy as np
@@ -49,6 +50,17 @@ def evaluate(program_path: str) -> Dict[str, float]:
     Returns:
         Dictionary of metric name to score
     """
+    # First perform basic validation
+    stage1_result = evaluate_stage1(program_path)
+    
+    # If validation fails, return early
+    if stage1_result["correctness"] < 0.8:
+        return {
+            "correctness": stage1_result["correctness"], 
+            "rank_quality": 0.0, 
+            "time_efficiency": 0.0
+        }
+    
     # Import the program
     try:
         spec = importlib.util.spec_from_file_location("program_module", program_path)
@@ -97,14 +109,42 @@ def evaluate(program_path: str) -> Dict[str, float]:
 
 def evaluate_stage1(program_path: str) -> Dict[str, float]:
     """
-    First stage of evaluation: test correctness
+    First stage of evaluation: basic validation and test correctness
     
     Args:
         program_path: Path to the program file
         
     Returns:
         Dictionary of metric name to score
     """
+    # First, perform static code analysis and basic validation
+    try:
+        with open(program_path, 'r') as f:
+            code_content = f.read()
+        
+        # Basic syntax check
+        try:
+            compile(code_content, program_path, 'exec')
+        except SyntaxError as e:
+            logger.error(f"Syntax error in program: {str(e)}")
+            return {"correctness": 0.0}
+        
+        # Check for common issues
+        if "TensorDecomposition" not in code_content:
+            logger.error("Program does not contain 'TensorDecomposition' class")
+            return {"correctness": 0.0}
+        
+        # Look for variable reference issues (e.g., 'u_factors' being used before definition)
+        if "u_factors" in code_content and "_initialize_decomposition" in code_content:
+            # Very basic check - not exhaustive but catches simple issues
+            if code_content.find("u_factors") < code_content.find("_initialize_decomposition"):
+                if "def _initialize_decomposition" in code_content:
+                    logger.error("Possible reference to 'u_factors' before initialization")
+                    return {"correctness": 0.0}
+    except Exception as e:
+        logger.error(f"Error during static code validation: {str(e)}")
+        return {"correctness": 0.0}
+    
     # Import the program
     try:
         spec = importlib.util.spec_from_file_location("program_module", program_path)
@@ -113,20 +153,50 @@ def evaluate_stage1(program_path: str) -> Dict[str, float]:
 
         module = importlib.util.module_from_spec(spec)
         sys.modules["program_module"] = module
-        spec.loader.exec_module(module)
 
+        # Use a safety wrapper to catch any import-time errors
+        try:
+            spec.loader.exec_module(module)
+        except Exception as e:
+            logger.error(f"Error during module execution: {str(e)}")
+            traceback.print_exc()
+            return {"correctness": 0.0}
+        
+        # Check for the required class
         if not hasattr(module, "TensorDecomposition"):
-            raise AttributeError(f"Program does not contain a 'TensorDecomposition' class")
+            logger.error("Program does not contain a 'TensorDecomposition' class")
+            return {"correctness": 0.0}
 
+        # Check basic class structure
         TensorDecomposition = module.TensorDecomposition
+        required_methods = ["__init__", "optimize", "_initialize_decomposition"]
+        for method in required_methods:
+            if not hasattr(TensorDecomposition, method):
+                logger.error(f"TensorDecomposition class missing required method: {method}")
+                return {"correctness": 0.0}
+        
+        # Try to instantiate the class with minimal parameters
+        try:
+            test_instance = TensorDecomposition(target_shape=(2, 2, 2), rank=7)
+        except Exception as e:
+            logger.error(f"Failed to instantiate TensorDecomposition: {str(e)}")
+            traceback.print_exc()
+            return {"correctness": 0.0}
+        
     except Exception as e:
         logger.error(f"Error importing program: {str(e)}")
+        traceback.print_exc()
         return {"correctness": 0.0}
 
-    # Test correctness
-    correctness_score = evaluate_correctness(TensorDecomposition)
-    
-    return {"correctness": correctness_score}
+    # If we get here, the basic validation passed
+    # Now perform a simple correctness test
+    try:
+        correctness_score = evaluate_correctness(TensorDecomposition)
+        return {"correctness": correctness_score}
+    except Exception as e:
+        logger.error(f"Error in correctness evaluation: {str(e)}")
+        traceback.print_exc()
+        return {"correctness": 0.0}
 
 
 def evaluate_correctness(TensorDecomposition) -> float:

examples/matrix_multiplication/optimize.py

@@ -77,55 +77,55 @@ async def main():
 
     # System message focusing on tensor decomposition and optimization
     system_template = """You are an expert in computational mathematics, numerical optimization, and algorithm design. 
-Your task is to optimize a tensor decomposition algorithm for discovering efficient matrix multiplication.
+Your task is to carefully optimize a tensor decomposition algorithm for discovering efficient matrix multiplication.
 
 When matrix multiplication is viewed as a tensor problem, the goal is to find a minimum-rank decomposition
 of the corresponding 3D tensor. Each term in the decomposition corresponds to a scalar multiplication in 
 the algorithm, so minimizing the rank directly leads to faster matrix multiplication.
 
-Your focus should be on targeted, specific improvements to the tensor decomposition optimization process. 
-Make small, focused changes rather than rewriting large sections. Key areas to consider include:
+Your focus should be on making ONE targeted, specific improvement. Code quality is critical - buggy
+code will fail evaluation and waste computing resources. Pay special attention to:
 
-1. Loss function improvements (regularization terms that encourage specific properties)
-2. Adding noise injection or annealing schedules to avoid local minima
-3. Improving initialization strategies for better convergence
-4. Numerical stability enhancements for complex-valued operations
-5. Techniques to encourage integer or half-integer coefficients
+1. Variable scope and references - don't use variables before they're defined
+2. Make only small, focused changes rather than large rewrites
+3. Test your change carefully, ensuring it will work when added to the existing code
+4. Respect the class structure and function interfaces
 
-The best algorithms from the literature for various matrix sizes include:
-- 2x2 matrices: Strassen's algorithm (7 multiplications)
-- 3x3 matrices: Laderman's algorithm (23 multiplications) 
-- 4x4 matrices: Recursive Strassen (49 multiplications, improved to 48 with complex values)
+The best matrix multiplication algorithms in the literature include Strassen's algorithm (7 multiplications
+for 2x2) and Laderman's algorithm (23 multiplications for 3x3).
 
-Focus on making 1-3 specific, high-impact changes rather than comprehensive rewrites.
+Make only ONE specific, high-impact change rather than multiple modifications.
 """
 
-    # User message template for tensor decomposition optimization (shortened for compatibility)
+    # User message template for tensor decomposition optimization (focused and clear)
     user_template = """
-Improve the tensor decomposition algorithm below with 1-3 specific changes.
+Focus on fixing ONE specific issue or making ONE targeted improvement in this tensor decomposition algorithm.
 
-CURRENT CODE:
+CODE:
 {current_program}
 
 METRICS:
 {metrics}
 
-FOCUS AREAS:
-{improvement_areas}
+FOCUS AREA: Make ONE change to improve the algorithm, focusing on either:
+1. The loss function to guide optimization better
+2. Initialization strategy for faster convergence 
+3. Adding regularization for integer/half-integer coefficients
 
-Make targeted changes using this format:
+Use SEARCH/REPLACE (keep the search section SHORT):
 
 <<<<<<< SEARCH
-// exact code to match (keep short)
+// exact code to match (small section only)
 =======
 // improved code
 >>>>>>> REPLACE
 
-RULES:
-1. Each SEARCH block must exactly match existing code
-2. Focus on 1-3 specific changes only
-3. Explain each change briefly
-4. Avoid large rewrites
+IMPORTANT RULES:
+1. Make only ONE change and explain it briefly
+2. Ensure the SEARCH block EXACTLY matches existing code
+3. NEVER use variables before they're defined
+4. Don't reference u_factors, v_factors, or w_factors outside _initialize_decomposition, _loss_fn, etc.
+5. ALWAYS test your change mentally to ensure it would work
 """
 
     # Add the templates to OpenEvolve's template manager