fix metrics for estimators; improve error handling

SimonBlanke · SimonBlanke · commit 67af4b28ef5c · 2025-08-09T13:43:50.000+02:00
diff --git a/examples/ml_visualization_example.py b/examples/ml_visualization_example.py
@@ -19,7 +19,7 @@ def main():
     print("=" * 40)
     
     # Create KNN classifier function
-    knn_func = KNeighborsClassifierFunction(metric="score")
+    knn_func = KNeighborsClassifierFunction(metric="accuracy")
     
     # Override search space with smaller values for faster demo
     def reduced_search_space(**kwargs):
diff --git a/scripts/generate_ml_plots.py b/scripts/generate_ml_plots.py
@@ -68,8 +68,11 @@
 
 def create_reduced_ml_function(func_class, reduced_space):
     """Create ML function instance with reduced search space for faster evaluation."""
-    # Create instance
-    func = func_class(metric="score")
+    # Create instance with appropriate metric
+    if 'Classifier' in func_class.__name__:
+        func = func_class(metric="accuracy")  # Use accuracy for classification
+    else:
+        func = func_class(metric="neg_mean_squared_error")  # Use MSE for regression
     
     # Override search_space method with reduced space
     original_search_space = func.search_space
@@ -92,71 +95,58 @@ def generate_ml_plots():
         print(f"Processing {func_name}...")
         print(f"{'='*50}")
         
-        try:
-            # Create function instance with reduced search space
-            func_class = config['class']
-            reduced_space = config['reduced_search_space']
-            ml_func = create_reduced_ml_function(func_class, reduced_space)
+        # Create function instance with reduced search space
+        func_class = config['class']
+        reduced_space = config['reduced_search_space']
+        ml_func = create_reduced_ml_function(func_class, reduced_space)
+        
+        # Create function-specific output directory
+        func_output_dir = os.path.join(ml_output_dir, ml_func._name_)
+        os.makedirs(func_output_dir, exist_ok=True)
+        
+        print(f"Search space: {ml_func.search_space()}")
+        
+        # 1. Generate hyperparameter vs hyperparameter plots
+        print("\n1. Generating hyperparameter interaction plots...")
+        for param1, param2 in config['hyperparameter_pairs']:
+            print(f"  Creating {param1} vs {param2} plot...")
             
-            # Create function-specific output directory
-            func_output_dir = os.path.join(ml_output_dir, ml_func._name_)
-            os.makedirs(func_output_dir, exist_ok=True)
+            fig = plotly_ml_hyperparameter_heatmap(
+                ml_func, param1, param2,
+                title=f"{ml_func.name} - {param1} vs {param2}"
+            )
             
-            print(f"Search space: {ml_func.search_space()}")
+            # Save as image
+            output_path = os.path.join(func_output_dir, f"{param1}_vs_{param2}_heatmap.jpg")
+            fig.write_image(output_path, format="jpeg", width=900, height=700)
             
-            # 1. Generate hyperparameter vs hyperparameter plots
-            print("\n1. Generating hyperparameter interaction plots...")
-            for param1, param2 in config['hyperparameter_pairs']:
-                print(f"  Creating {param1} vs {param2} plot...")
-                
-                try:
-                    fig = plotly_ml_hyperparameter_heatmap(
-                        ml_func, param1, param2,
-                        title=f"{ml_func.name} - {param1} vs {param2}"
-                    )
-                    
-                    # Save as image
-                    output_path = os.path.join(func_output_dir, f"{param1}_vs_{param2}_heatmap.jpg")
-                    fig.write_image(output_path, format="jpeg", width=900, height=700)
-                    
-                    # Also save to main images directory for README
-                    main_output_path = os.path.join(output_dir, f"{ml_func._name_}_{param1}_vs_{param2}_heatmap.jpg")
-                    fig.write_image(main_output_path, format="jpeg", width=900, height=700)
-                    
-                    print(f"    ✓ Saved {param1} vs {param2} heatmap")
-                    
-                except Exception as e:
-                    print(f"    ✗ Error creating {param1} vs {param2} plot: {e}")
+            # Also save to main images directory for README
+            main_output_path = os.path.join(output_dir, f"{ml_func._name_}_{param1}_vs_{param2}_heatmap.jpg")
+            fig.write_image(main_output_path, format="jpeg", width=900, height=700)
+            
+            print(f"    ✓ Saved {param1} vs {param2} heatmap")
+        
+        # 2. Generate dataset vs hyperparameter plots  
+        print("\n2. Generating dataset analysis plots...")
+        for hyperparameter in config['dataset_analyses']:
+            print(f"  Creating dataset vs {hyperparameter} plot...")
+            
+            fig = plotly_dataset_hyperparameter_analysis(
+                ml_func, hyperparameter,
+                title=f"{ml_func.name} - Dataset vs {hyperparameter}"
+            )
+            
+            # Save as image
+            output_path = os.path.join(func_output_dir, f"dataset_vs_{hyperparameter}_analysis.jpg")
+            fig.write_image(output_path, format="jpeg", width=1000, height=700)
+            
+            # Also save to main images directory for README
+            main_output_path = os.path.join(output_dir, f"{ml_func._name_}_dataset_vs_{hyperparameter}_analysis.jpg")
+            fig.write_image(main_output_path, format="jpeg", width=1000, height=700)
             
-            # 2. Generate dataset vs hyperparameter plots  
-            print("\n2. Generating dataset analysis plots...")
-            for hyperparameter in config['dataset_analyses']:
-                print(f"  Creating dataset vs {hyperparameter} plot...")
+            print(f"    ✓ Saved dataset vs {hyperparameter} analysis")
                 
-                try:
-                    fig = plotly_dataset_hyperparameter_analysis(
-                        ml_func, hyperparameter,
-                        title=f"{ml_func.name} - Dataset vs {hyperparameter}"
-                    )
-                    
-                    # Save as image
-                    output_path = os.path.join(func_output_dir, f"dataset_vs_{hyperparameter}_analysis.jpg")
-                    fig.write_image(output_path, format="jpeg", width=1000, height=700)
-                    
-                    # Also save to main images directory for README
-                    main_output_path = os.path.join(output_dir, f"{ml_func._name_}_dataset_vs_{hyperparameter}_analysis.jpg")
-                    fig.write_image(main_output_path, format="jpeg", width=1000, height=700)
-                    
-                    print(f"    ✓ Saved dataset vs {hyperparameter} analysis")
-                    
-                except Exception as e:
-                    print(f"    ✗ Error creating dataset vs {hyperparameter} plot: {e}")
-                    
-            print(f"✓ Completed {func_name} analysis")
-                    
-        except Exception as e:
-            print(f"✗ Error processing {func_name}: {e}")
-            continue
+        print(f"✓ Completed {func_name} analysis")
     
     print(f"\n✓ ML plot generation complete! Images saved to:")
     print(f"  - Individual function plots: {ml_output_dir}")
@@ -166,37 +156,37 @@ def generate_sample_plots():
     """Generate a few sample plots for testing."""
     print("Generating sample ML plots for testing...")
     
-    # Test with KNeighborsClassifier (smallest/fastest)
-    try:
-        ml_func = create_reduced_ml_function(
-            KNeighborsClassifierFunction, 
-            {
-                'n_neighbors': [3, 10, 20],  # Very small for testing
-                'algorithm': ['auto', 'ball_tree'], 
-                'cv': [3],
-            }
-        )
-        
-        print("Creating sample hyperparameter plot...")
-        fig1 = plotly_ml_hyperparameter_heatmap(
-            ml_func, 'n_neighbors', 'algorithm',
-            title="Sample: KNN Hyperparameter Analysis"
-        )
-        fig1.write_image(os.path.join(output_dir, "sample_knn_hyperparams.jpg"), 
-                        format="jpeg", width=900, height=700)
-        
-        print("Creating sample dataset analysis plot...")
-        fig2 = plotly_dataset_hyperparameter_analysis(
-            ml_func, 'n_neighbors',
-            title="Sample: Dataset vs n_neighbors"
-        )
-        fig2.write_image(os.path.join(output_dir, "sample_knn_datasets.jpg"), 
-                        format="jpeg", width=1000, height=700)
-        
-        print("✓ Sample plots generated successfully!")
-        
-    except Exception as e:
-        print(f"✗ Error generating sample plots: {e}")
+    # Create with proper metric
+    ml_func = KNeighborsClassifierFunction(metric="accuracy")
+    
+    # Override search space with small values for testing
+    def reduced_search_space_method(**kwargs):
+        return {
+            'n_neighbors': [3, 10, 20],  # Very small for testing
+            'algorithm': ['auto', 'ball_tree'], 
+            'cv': [3],
+            'dataset': ml_func.dataset_default  # Keep all datasets
+        }
+    
+    ml_func.search_space = reduced_search_space_method
+    
+    print("Creating sample hyperparameter plot...")
+    fig1 = plotly_ml_hyperparameter_heatmap(
+        ml_func, 'n_neighbors', 'algorithm',
+        title="Sample: KNN Hyperparameter Analysis"
+    )
+    fig1.write_image(os.path.join(output_dir, "sample_knn_hyperparams.jpg"), 
+                    format="jpeg", width=900, height=700)
+    
+    print("Creating sample dataset analysis plot...")
+    fig2 = plotly_dataset_hyperparameter_analysis(
+        ml_func, 'n_neighbors',
+        title="Sample: Dataset vs n_neighbors"
+    )
+    fig2.write_image(os.path.join(output_dir, "sample_knn_datasets.jpg"), 
+                    format="jpeg", width=1000, height=700)
+    
+    print("✓ Sample plots generated successfully!")
 
 if __name__ == "__main__":
     # Check if we should generate sample plots first (faster for testing)
diff --git a/src/surfaces/visualize.py b/src/surfaces/visualize.py
@@ -514,59 +514,56 @@ def plotly_ml_hyperparameter_heatmap(
         Plotly Figure object
     """
     search_space = ml_function.search_space()
-    
-    # Use provided fixed params or defaults
-    if fixed_params is None:
-        fixed_params = {}
+    fixed_params = fixed_params or {}
     
     # Get parameter ranges
     param1_values = search_space[param1]
     param2_values = search_space[param2]
     
-    # Create grid for evaluation
+    # SINGLE VALIDATION TEST - fail fast if configuration is wrong
+    test_params = fixed_params.copy()
+    test_params[param1] = param1_values[0]
+    test_params[param2] = param2_values[0]
+    
+    # Fill in missing required parameters
+    for param_name in search_space:
+        if param_name not in test_params:
+            test_params[param_name] = search_space[param_name][0]
+    
+    # Test once - if this fails, the whole configuration is wrong
+    test_result = ml_function.objective_function(test_params)
+    print(f"✓ ML function validation successful (test result: {test_result:.4f})")
+    
+    # Create evaluation grid - NO TRY-CATCH, let errors surface
     results = []
-    param1_grid = []
-    param2_grid = []
     
     print(f"Evaluating {len(param1_values)} x {len(param2_values)} = {len(param1_values) * len(param2_values)} combinations...")
     pbar = tqdm(total=len(param1_values) * len(param2_values), desc="ML Evaluation")
     
     for p1_val in param1_values:
         row_results = []
-        row_p1 = []
-        row_p2 = []
         
         for p2_val in param2_values:
             # Create parameter dict
             params = fixed_params.copy()
             params[param1] = p1_val
             params[param2] = p2_val
             
-            # Fill in any missing required parameters with defaults
+            # Fill in missing required parameters
             for param_name in search_space:
                 if param_name not in params:
-                    default_val = search_space[param_name][0]  # Use first value as default
-                    params[param_name] = default_val
+                    params[param_name] = search_space[param_name][0]
             
-            # Evaluate
-            try:
-                score = ml_function.objective_function(params)
-                row_results.append(score)
-            except Exception as e:
-                print(f"Error evaluating {params}: {e}")
-                row_results.append(np.nan)
-            
-            row_p1.append(p1_val)
-            row_p2.append(p2_val)
+            # Evaluate - NO TRY-CATCH, let real errors surface
+            score = ml_function.objective_function(params)
+            row_results.append(float(score))
             pbar.update(1)
         
         results.append(row_results)
-        param1_grid.append(row_p1)
-        param2_grid.append(row_p2)
     
     pbar.close()
     
-    # Convert to numpy arrays
+    # Convert to numpy and create plot
     z_values = np.array(results)
     
     # Handle categorical parameters for display
@@ -633,10 +630,21 @@ def plotly_dataset_hyperparameter_analysis(
         Plotly Figure object
     """
     search_space = ml_function.search_space()
-    
-    # Use provided fixed params or defaults
-    if fixed_params is None:
-        fixed_params = {}
+    fixed_params = fixed_params or {}
+        
+    # SINGLE VALIDATION TEST - fail fast if configuration is wrong
+    test_params = fixed_params.copy()
+    test_params['dataset'] = search_space['dataset'][0]
+    test_params[hyperparameter] = search_space[hyperparameter][0]
+    
+    # Fill in missing required parameters
+    for param_name in search_space:
+        if param_name not in test_params:
+            test_params[param_name] = search_space[param_name][0]
+    
+    # Test once - if this fails, the whole configuration is wrong
+    test_result = ml_function.objective_function(test_params)
+    print(f"✓ ML function validation successful (test result: {test_result:.4f})")
         
     datasets = search_space['dataset']
     hyperparameter_values = search_space[hyperparameter]
@@ -647,7 +655,7 @@ def plotly_dataset_hyperparameter_analysis(
         name = dataset_func.__name__.replace('_data', '').replace('_', ' ').title()
         dataset_names.append(name)
     
-    # Evaluate across datasets and hyperparameter values
+    # Evaluate across datasets and hyperparameter values - NO TRY-CATCH
     results = []
     
     print(f"Evaluating {len(datasets)} datasets x {len(hyperparameter_values)} {hyperparameter} values...")
@@ -665,17 +673,11 @@ def plotly_dataset_hyperparameter_analysis(
             # Fill in missing required parameters with defaults
             for param_name in search_space:
                 if param_name not in params:
-                    default_val = search_space[param_name][0]
-                    params[param_name] = default_val
-            
-            # Evaluate
-            try:
-                score = ml_function.objective_function(params)
-                dataset_results.append(score)
-            except Exception as e:
-                print(f"Error evaluating {params}: {e}")
-                dataset_results.append(np.nan)
+                    params[param_name] = search_space[param_name][0]
             
+            # Evaluate - NO TRY-CATCH, let real errors surface
+            score = ml_function.objective_function(params)
+            dataset_results.append(float(score))
             pbar.update(1)
         
         results.append(dataset_results)
@@ -741,7 +743,7 @@ def create_ml_function_analysis_suite(
     search_space = ml_function.search_space()
     figures = {}
     
-    # Get numeric hyperparameters (exclude dataset and cv)
+    # Get hyperparameters (exclude dataset and cv)
     numeric_params = []
     categorical_params = []
     
@@ -761,26 +763,20 @@ def create_ml_function_analysis_suite(
                 continue
                 
             plot_name = f"hyperparam_{param1}_vs_{param2}"
-            try:
-                fig = plotly_ml_hyperparameter_heatmap(
-                    ml_function, param1, param2,
-                    title=f"{ml_function.name} - Hyperparameter Analysis"
-                )
-                figures[plot_name] = fig
-            except Exception as e:
-                print(f"Error creating plot {plot_name}: {e}")
+            fig = plotly_ml_hyperparameter_heatmap(
+                ml_function, param1, param2,
+                title=f"{ml_function.name} - Hyperparameter Analysis"
+            )
+            figures[plot_name] = fig
     
     # 2. Dataset vs Hyperparameter plots
     print("Creating dataset analysis plots...")
     for param_name in numeric_params + categorical_params:
         plot_name = f"dataset_vs_{param_name}"
-        try:
-            fig = plotly_dataset_hyperparameter_analysis(
-                ml_function, param_name,
-                title=f"{ml_function.name} - Dataset Analysis"
-            )
-            figures[plot_name] = fig
-        except Exception as e:
-            print(f"Error creating plot {plot_name}: {e}")
+        fig = plotly_dataset_hyperparameter_analysis(
+            ml_function, param_name,
+            title=f"{ml_function.name} - Dataset Analysis"
+        )
+        figures[plot_name] = fig
     
     return figures
