test_comparison.py

# %% [markdown]
# # Test Band Importance Comparison
# 
# This script tests the band importance comparison functionality with sample data
# to ensure everything works correctly before running on real results.

# %%
import os
import json
import numpy as np
import pandas as pd
from pathlib import Path

# %%
def create_sample_results():
    """Create sample band importance results for testing."""
    
    # Sample band names
    band_names = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 
                  'B08', 'B8A', 'B09', 'B10', 'B11', 'B12']
    
    # Create sample EuroSAT results
    eurosat_permutation = {
        'importance_stats': {},
        'sorted_bands': [],
        'baseline_accuracy': 0.958,
        'n_repeats': 10
    }
    
    # Generate sample importance scores
    np.random.seed(42)
    eurosat_scores = np.random.uniform(0.01, 0.15, len(band_names))
    eurosat_scores = np.sort(eurosat_scores)[::-1]  # Sort descending
    
    for i, band in enumerate(band_names):
        eurosat_permutation['importance_stats'][band] = {
            'mean': float(eurosat_scores[i]),
            'std': float(eurosat_scores[i] * 0.1),
            'min': float(eurosat_scores[i] * 0.8),
            'max': float(eurosat_scores[i] * 1.2),
            'scores': [float(eurosat_scores[i])] * 10
        }
        eurosat_permutation['sorted_bands'].append((band, eurosat_permutation['importance_stats'][band]))
    
    # Create sample ablation results
    eurosat_ablation = {
        'ablation_results': {},
        'sorted_ablation': [],
        'baseline_accuracy': 0.958
    }
    
    for i, band in enumerate(band_names):
        performance_drop = float(eurosat_scores[i] * 0.8)
        eurosat_ablation['ablation_results'][band] = {
            'accuracy': 0.958 - performance_drop,
            'performance_drop': performance_drop,
            'relative_drop': performance_drop / 0.958
        }
        eurosat_ablation['sorted_ablation'].append((band, eurosat_ablation['ablation_results'][band]))
    
    # Create sample OSCD results (slightly different pattern)
    oscd_permutation = {
        'importance_stats': {},
        'sorted_bands': [],
        'baseline_accuracy': 0.892,
        'n_repeats': 10
    }
    
    # Generate different sample scores for OSCD
    np.random.seed(123)
    oscd_scores = np.random.uniform(0.01, 0.12, len(band_names))
    oscd_scores = np.sort(oscd_scores)[::-1]  # Sort descending
    
    for i, band in enumerate(band_names):
        oscd_permutation['importance_stats'][band] = {
            'mean': float(oscd_scores[i]),
            'std': float(oscd_scores[i] * 0.1),
            'min': float(oscd_scores[i] * 0.8),
            'max': float(oscd_scores[i] * 1.2),
            'scores': [float(oscd_scores[i])] * 10
        }
        oscd_permutation['sorted_bands'].append((band, oscd_permutation['importance_stats'][band]))
    
    # Create sample OSCD ablation results
    oscd_ablation = {
        'ablation_results': {},
        'sorted_ablation': [],
        'baseline_accuracy': 0.892
    }
    
    for i, band in enumerate(band_names):
        performance_drop = float(oscd_scores[i] * 0.8)
        oscd_ablation['ablation_results'][band] = {
            'accuracy': 0.892 - performance_drop,
            'performance_drop': performance_drop,
            'relative_drop': performance_drop / 0.892
        }
        oscd_ablation['sorted_ablation'].append((band, oscd_ablation['ablation_results'][band]))
    
    return eurosat_permutation, eurosat_ablation, oscd_permutation, oscd_ablation

def save_sample_results():
    """Save sample results to test directories."""
    
    # Create test directories
    test_dirs = [
        './test_results/eurosat_band_importance',
        './test_results/oscd_band_importance'
    ]
    
    for dir_path in test_dirs:
        Path(dir_path).mkdir(parents=True, exist_ok=True)
    
    # Generate sample data
    eurosat_perm, eurosat_ablation, oscd_perm, oscd_ablation = create_sample_results()
    
    # Save EuroSAT results
    with open('./test_results/eurosat_band_importance/permutation_importance.json', 'w') as f:
        json.dump(eurosat_perm, f, indent=2)
    
    with open('./test_results/eurosat_band_importance/ablation_study.json', 'w') as f:
        json.dump(eurosat_ablation, f, indent=2)
    
    # Save OSCD results
    with open('./test_results/oscd_band_importance/permutation_importance.json', 'w') as f:
        json.dump(oscd_perm, f, indent=2)
    
    with open('./test_results/oscd_band_importance/ablation_study.json', 'w') as f:
        json.dump(oscd_ablation, f, indent=2)
    
    print("Sample results saved to test directories")

def test_comparison():
    """Test the comparison functionality."""
    
    print("=" * 60)
    print("TESTING BAND IMPORTANCE COMPARISON")
    print("=" * 60)
    
    # Import the comparison module
    from compare_band_importance import BandImportanceComparator
    
    # Initialize comparator with test data
    comparator = BandImportanceComparator(
        eurosat_results_dir='./test_results/eurosat_band_importance',
        oscd_results_dir='./test_results/oscd_band_importance'
    )
    
    # Test correlation calculation
    print("\nTesting correlation calculation...")
    perm_corr = comparator.calculate_correlations('permutation')
    ablation_corr = comparator.calculate_correlations('ablation')
    
    print(f"Permutation correlation: {perm_corr}")
    print(f"Ablation correlation: {ablation_corr}")
    
    # Test DataFrame creation
    print("\nTesting DataFrame creation...")
    perm_df = comparator.create_comparison_dataframe('permutation')
    ablation_df = comparator.create_comparison_dataframe('ablation')
    
    print(f"Permutation DataFrame shape: {perm_df.shape}")
    print(f"Ablation DataFrame shape: {ablation_df.shape}")
    
    # Test comprehensive table
    print("\nTesting comprehensive table creation...")
    comprehensive_df = comparator.create_comprehensive_table()
    print(f"Comprehensive DataFrame shape: {comprehensive_df.shape}")
    
    # Test common bands finding
    print("\nTesting common bands finding...")
    common_bands = comparator._find_common_important_bands()
    print(f"Common important bands: {common_bands}")
    
    # Test report generation
    print("\nTesting report generation...")
    report = comparator.generate_summary_report()
    print("Report generated successfully!")
    print(f"Report length: {len(report)} characters")
    
    # Test plotting (without saving)
    print("\nTesting plotting functionality...")
    try:
        comparator.plot_comparison('permutation')
        print("Permutation comparison plot created successfully!")
    except Exception as e:
        print(f"Plotting failed: {e}")
    
    print("\n" + "=" * 60)
    print("ALL TESTS COMPLETED SUCCESSFULLY!")
    print("=" * 60)

def main():
    """Main test function."""
    
    # Create sample results
    print("Creating sample results...")
    save_sample_results()
    
    # Run tests
    test_comparison()
    
    # Clean up (optional)
    print("\nTest completed! Sample data saved in ./test_results/")
    print("You can manually delete ./test_results/ if you want to clean up.")

# %%
if __name__ == "__main__":
    main()