diff --git a/lib/LinearSolveAutotune/Project.toml b/lib/LinearSolveAutotune/Project.toml index 5205495ba..6726a26cc 100644 --- a/lib/LinearSolveAutotune/Project.toml +++ b/lib/LinearSolveAutotune/Project.toml @@ -7,6 +7,7 @@ version = "1.0.0" LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +CPUSummary = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" @@ -18,6 +19,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" RecursiveFactorization = "f2c3362d-daeb-58d1-803e-2bc74f2840b4" blis_jll = "6136c539-28a5-5bf0-87cc-b183200dce32" LAPACK_jll = "51474c39-65e3-53ba-86ba-03b1b862ec14" @@ -28,6 +30,7 @@ Metal = "dde4c033-4e86-420c-a63e-0dd931031962" LinearSolve = "3" BenchmarkTools = "1" Base64 = "1" +CPUSummary = "0.2" DataFrames = "1" GitHub = "5" Plots = "1" @@ -39,6 +42,7 @@ LinearAlgebra = "1" Printf = "1" Dates = "1" Test = "1" +ProgressMeter = "1" RecursiveFactorization = "0.2" blis_jll = "0.9.0" LAPACK_jll = "3" diff --git a/lib/LinearSolveAutotune/README.md b/lib/LinearSolveAutotune/README.md new file mode 100644 index 000000000..959002b88 --- /dev/null +++ b/lib/LinearSolveAutotune/README.md @@ -0,0 +1,173 @@ +# LinearSolveAutotune.jl + +Automatic benchmarking and tuning for LinearSolve.jl algorithms. + +## Quick Start + +```julia +using LinearSolve, LinearSolveAutotune + +# Run benchmarks with default settings (small, medium, and large sizes) +results = autotune_setup() + +# View a summary of results +display(results) + +# Plot all benchmark results +plot(results) + +# Share your results with the community (optional) +share_results(results) +``` + +## Features + +- **Automatic Algorithm Benchmarking**: Tests all available LU factorization methods +- **Multi-size Testing**: Flexible size categories from small to very large matrices +- **Element Type Support**: Tests with Float32, Float64, ComplexF32, ComplexF64 +- **GPU Support**: Automatically detects and benchmarks GPU algorithms if available +- **Performance Visualization**: Generate plots on demand with `plot(results)` +- **Community Sharing**: Optional telemetry to help improve algorithm selection + +## Size Categories + +The package now uses flexible size categories: + +- `:tiny` - Matrices from 5×5 to 20×20 (very small problems) +- `:small` - Matrices from 20×20 to 100×100 (small problems) +- `:medium` - Matrices from 100×100 to 300×300 (typical problems) +- `:large` - Matrices from 300×300 to 1000×1000 (larger problems) +- `:big` - Matrices from 10000×10000 to 100000×100000 (GPU/HPC) + +## Usage Examples + +### Basic Benchmarking + +```julia +# Default: small, medium, and large sizes +results = autotune_setup() + +# Test all size ranges +results = autotune_setup(sizes = [:small, :medium, :large, :big]) + +# Large matrices only (for GPU systems) +results = autotune_setup(sizes = [:large, :big]) + +# Custom configuration +results = autotune_setup( + sizes = [:medium, :large], + samples = 10, + seconds = 1.0, + eltypes = (Float64, ComplexF64) +) + +# View results and plot +display(results) +plot(results) +``` + +### Sharing Results + +After running benchmarks, you can optionally share your results with the LinearSolve.jl community to help improve automatic algorithm selection: + +```julia +# Share your benchmark results +share_results(results) +``` + +## Setting Up GitHub Authentication + +To share results, you need GitHub authentication. We recommend using the GitHub CLI: + +### Method 1: GitHub CLI (Recommended) + +1. **Install GitHub CLI** + - macOS: `brew install gh` + - Windows: `winget install --id GitHub.cli` + - Linux: See [cli.github.com](https://cli.github.com/manual/installation) + +2. **Authenticate** + ```bash + gh auth login + ``` + Follow the prompts to authenticate with your GitHub account. + +3. **Verify authentication** + ```bash + gh auth status + ``` + +### Method 2: GitHub Personal Access Token + +If you prefer using a token: + +1. Go to [GitHub Settings > Tokens](https://github.com/settings/tokens/new) +2. Add description: "LinearSolve.jl Telemetry" +3. Select scope: `public_repo` +4. Click "Generate token" and copy it +5. In Julia: + ```julia + ENV["GITHUB_TOKEN"] = "your_token_here" + share_results(results, sysinfo, plots) + ``` + +## How It Works + +1. **Benchmarking**: The `autotune_setup()` function runs comprehensive benchmarks of all available LinearSolve.jl algorithms across different matrix sizes and element types. + +2. **Analysis**: Results are analyzed to find the best-performing algorithm for each size range and element type combination. + +3. **Preferences**: Optionally sets Julia preferences to automatically use the best algorithms for your system. + +4. **Sharing**: The `share_results()` function allows you to contribute your benchmark data to the community collection at [LinearSolve.jl Issue #669](https://github.com/SciML/LinearSolve.jl/issues/669). + +## Privacy and Telemetry + +- Sharing results is **completely optional** +- Only benchmark performance data and system specifications are shared +- No personal information is collected +- All shared data is publicly visible on GitHub +- You can review the exact data before sharing + +## API Reference + +### `autotune_setup` + +```julia +autotune_setup(; + sizes = [:small, :medium, :large], + set_preferences = true, + samples = 5, + seconds = 0.5, + eltypes = (Float32, Float64, ComplexF32, ComplexF64), + skip_missing_algs = false +) +``` + +**Parameters:** +- `sizes`: Vector of size categories to test +- `set_preferences`: Update LinearSolve preferences +- `samples`: Number of benchmark samples per test +- `seconds`: Maximum time per benchmark +- `eltypes`: Element types to benchmark +- `skip_missing_algs`: Continue if algorithms are missing + +**Returns:** +- `results`: AutotuneResults object containing benchmark data and system info + +### `share_results` + +```julia +share_results(results) +``` + +**Parameters:** +- `results`: AutotuneResults object from `autotune_setup` + +## Contributing + +Your benchmark contributions help improve LinearSolve.jl for everyone! By sharing results from diverse hardware configurations, we can build better automatic algorithm selection heuristics. + +## License + +Part of the SciML ecosystem. See LinearSolve.jl for license information. \ No newline at end of file diff --git a/lib/LinearSolveAutotune/src/LinearSolveAutotune.jl b/lib/LinearSolveAutotune/src/LinearSolveAutotune.jl index fcb8a81a5..cf4810a84 100644 --- a/lib/LinearSolveAutotune/src/LinearSolveAutotune.jl +++ b/lib/LinearSolveAutotune/src/LinearSolveAutotune.jl @@ -11,6 +11,8 @@ using LinearAlgebra using Printf using Dates using Base64 +using ProgressMeter +using CPUSummary # Hard dependency to ensure RFLUFactorization others solvers are available using RecursiveFactorization @@ -24,7 +26,7 @@ using Metal using GitHub using Plots -export autotune_setup +export autotune_setup, share_results, AutotuneResults, plot include("algorithms.jl") include("gpu_detection.jl") @@ -33,11 +35,100 @@ include("plotting.jl") include("telemetry.jl") include("preferences.jl") +# Define the AutotuneResults struct +struct AutotuneResults + results_df::DataFrame + sysinfo::Dict +end + +# Display method for AutotuneResults +function Base.show(io::IO, results::AutotuneResults) + println(io, "="^60) + println(io, "LinearSolve.jl Autotune Results") + println(io, "="^60) + + # System info summary + println(io, "\n📊 System Information:") + println(io, " • CPU: ", get(results.sysinfo, "cpu_name", "Unknown")) + println(io, " • OS: ", get(results.sysinfo, "os_name", "Unknown"), " (", get(results.sysinfo, "os", "Unknown"), ")") + println(io, " • Julia: ", get(results.sysinfo, "julia_version", "Unknown")) + println(io, " • Threads: ", get(results.sysinfo, "num_threads", "Unknown"), " (BLAS: ", get(results.sysinfo, "blas_num_threads", "Unknown"), ")") + + # Results summary + successful_results = filter(row -> row.success, results.results_df) + if nrow(successful_results) > 0 + println(io, "\n🏆 Top Performing Algorithms:") + summary = combine(groupby(successful_results, :algorithm), + :gflops => mean => :avg_gflops, + :gflops => maximum => :max_gflops, + nrow => :num_tests) + sort!(summary, :avg_gflops, rev = true) + + # Show top 5 + for (i, row) in enumerate(eachrow(first(summary, 5))) + println(io, " ", i, ". ", row.algorithm, ": ", + @sprintf("%.2f GFLOPs avg", row.avg_gflops)) + end + end + + # Element types tested + eltypes = unique(results.results_df.eltype) + println(io, "\n🔬 Element Types Tested: ", join(eltypes, ", ")) + + # Matrix sizes tested + sizes = unique(results.results_df.size) + println(io, "📏 Matrix Sizes: ", minimum(sizes), "×", minimum(sizes), + " to ", maximum(sizes), "×", maximum(sizes)) + + # Call to action - reordered + println(io, "\n" * "="^60) + println(io, "🚀 For comprehensive results, consider running:") + println(io, " results_full = autotune_setup(") + println(io, " sizes = [:tiny, :small, :medium, :large, :big],") + println(io, " eltypes = (Float32, Float64, ComplexF32, ComplexF64)") + println(io, " )") + println(io, "\n📈 See community results at:") + println(io, " https://github.com/SciML/LinearSolve.jl/issues/669") + println(io, "\n💡 To share your results with the community, run:") + println(io, " share_results(results)") + println(io, "="^60) +end + +# Plot method for AutotuneResults +function Plots.plot(results::AutotuneResults; kwargs...) + # Generate plots from the results data + plots_dict = create_benchmark_plots(results.results_df) + + if plots_dict === nothing || isempty(plots_dict) + @warn "No data available for plotting" + return nothing + end + + # Create a composite plot from all element type plots + plot_list = [] + for (eltype_name, p) in plots_dict + push!(plot_list, p) + end + + # Create composite plot + n_plots = length(plot_list) + if n_plots == 1 + return plot_list[1] + elseif n_plots == 2 + return plot(plot_list..., layout=(1, 2), size=(1200, 500); kwargs...) + elseif n_plots <= 4 + return plot(plot_list..., layout=(2, 2), size=(1200, 900); kwargs...) + else + ncols = ceil(Int, sqrt(n_plots)) + nrows = ceil(Int, n_plots / ncols) + return plot(plot_list..., layout=(nrows, ncols), + size=(400*ncols, 400*nrows); kwargs...) + end +end + """ autotune_setup(; - large_matrices::Bool = false, - telemetry::Bool = true, - make_plot::Bool = true, + sizes = [:small, :medium, :large], set_preferences::Bool = true, samples::Int = 5, seconds::Float64 = 0.5, @@ -47,16 +138,13 @@ include("preferences.jl") Run a comprehensive benchmark of all available LU factorization methods and optionally: - Create performance plots for each element type - - Create GitHub issue with telemetry data for community collection - Set Preferences for optimal algorithm selection - Support both CPU and GPU algorithms based on hardware detection - Test algorithm compatibility with different element types # Arguments - - `large_matrices::Bool = false`: Include larger matrix sizes for GPU benchmarking - - `telemetry::Bool = true`: Create GitHub issue with results for community data collection - - `make_plot::Bool = true`: Generate performance plots for each element type + - `sizes = [:small, :medium, :large]`: Size categories to test. Options: :small (5-20), :medium (20-300), :large (300-1000), :big (10000-100000) - `set_preferences::Bool = true`: Update LinearSolve preferences with optimal algorithms - `samples::Int = 5`: Number of benchmark samples per algorithm/size - `seconds::Float64 = 0.5`: Maximum time per benchmark @@ -65,8 +153,7 @@ Run a comprehensive benchmark of all available LU factorization methods and opti # Returns - - `DataFrame`: Detailed benchmark results with performance data for all element types - - `Dict` or `Plot`: Performance visualizations by element type (if `make_plot=true`) + - `AutotuneResults`: Object containing benchmark results, system info, and plots # Examples @@ -74,45 +161,30 @@ Run a comprehensive benchmark of all available LU factorization methods and opti using LinearSolve using LinearSolveAutotune -# Basic autotune with default settings (4 element types) +# Basic autotune with default sizes results = autotune_setup() -# Custom autotune for GPU systems with larger matrices -results = autotune_setup(large_matrices = true, samples = 10, seconds = 1.0) +# Test all size ranges +results = autotune_setup(sizes = [:small, :medium, :large, :big]) -# Autotune with only Float64 and ComplexF64 -results = autotune_setup(eltypes = (Float64, ComplexF64)) +# Large matrices only +results = autotune_setup(sizes = [:large, :big], samples = 10, seconds = 1.0) -# Test with BigFloat (note: most BLAS algorithms will be excluded) -results = autotune_setup(eltypes = (BigFloat,), telemetry = false) - -# Allow missing algorithms (useful for incomplete setups) -results = autotune_setup(skip_missing_algs = true) +# After running autotune, share results (requires gh CLI or GitHub token) +share_results(results) ``` """ function autotune_setup(; - large_matrices::Bool = true, - telemetry::Bool = true, - make_plot::Bool = true, + sizes = [:tiny, :small, :medium, :large], set_preferences::Bool = true, samples::Int = 5, seconds::Float64 = 0.5, - eltypes = (Float32, Float64, ComplexF32, ComplexF64), + eltypes = (Float64,), skip_missing_algs::Bool = false) @info "Starting LinearSolve.jl autotune setup..." - @info "Configuration: large_matrices=$large_matrices, telemetry=$telemetry, make_plot=$make_plot, set_preferences=$set_preferences" + @info "Configuration: sizes=$sizes, set_preferences=$set_preferences" @info "Element types to benchmark: $(join(eltypes, ", "))" - # Set up GitHub authentication early if telemetry is enabled - github_auth = nothing - if telemetry - @info "🔗 Checking GitHub authentication for telemetry..." - github_auth = setup_github_authentication() - if github_auth === nothing - @info "📊 Continuing with benchmarking (results will be saved locally)" - end - end - # Get system information system_info = get_system_info() @info "System detected: $(system_info["os"]) $(system_info["arch"]) with $(system_info["num_cores"]) cores" @@ -135,14 +207,14 @@ function autotune_setup(; error("No algorithms found! This shouldn't happen.") end - # Get benchmark sizes - sizes = collect(get_benchmark_sizes(large_matrices)) - @info "Benchmarking $(length(sizes)) matrix sizes from $(minimum(sizes)) to $(maximum(sizes))" + # Get benchmark sizes based on size categories + matrix_sizes = collect(get_benchmark_sizes(sizes)) + @info "Benchmarking $(length(matrix_sizes)) matrix sizes from $(minimum(matrix_sizes)) to $(maximum(matrix_sizes))" # Run benchmarks @info "Running benchmarks (this may take several minutes)..." - results_df = benchmark_algorithms(sizes, all_algs, all_names, eltypes; - samples = samples, seconds = seconds, large_matrices = large_matrices) + results_df = benchmark_algorithms(matrix_sizes, all_algs, all_names, eltypes; + samples = samples, seconds = seconds, sizes = sizes) # Display results table successful_results = filter(row -> row.success, results_df) @@ -176,34 +248,104 @@ function autotune_setup(; set_algorithm_preferences(categories) end - # Create plots if requested - plots_dict = nothing - plot_files = nothing - if make_plot - @info "Creating performance plots..." - plots_dict = create_benchmark_plots(results_df) - if !isempty(plots_dict) - plot_files = save_benchmark_plots(plots_dict) + @info "Autotune setup completed!" + + sysinfo_df = get_detailed_system_info() + # Convert DataFrame to Dict for AutotuneResults + sysinfo = Dict{String, Any}() + if nrow(sysinfo_df) > 0 + for col in names(sysinfo_df) + sysinfo[col] = sysinfo_df[1, col] end end - # Create GitHub issue with telemetry if requested - if telemetry && nrow(successful_results) > 0 - @info "📤 Creating GitHub issue with benchmark data for community collection..." - markdown_content = format_results_for_github(results_df, system_info, categories) - upload_to_github(markdown_content, plot_files, github_auth, results_df, system_info, categories) - end + # Return AutotuneResults object + return AutotuneResults(results_df, sysinfo) +end - @info "Autotune setup completed!" +""" + share_results(results::AutotuneResults) - sysinfo = get_detailed_system_info() +Share your benchmark results with the LinearSolve.jl community to help improve +automatic algorithm selection across different hardware configurations. - # Return results and plots - if make_plot && plots_dict !== nothing && !isempty(plots_dict) - return results_df, sysinfo, plots_dict - else - return results_df, sysinfo, nothing +This function will authenticate with GitHub (using gh CLI or token) and post +your results as a comment to the community benchmark collection issue. + +# Setup Instructions + +## Method 1: GitHub CLI (Recommended) +1. Install GitHub CLI: https://cli.github.com/ +2. Run: `gh auth login` +3. Follow the prompts to authenticate +4. Run this function - it will automatically use your gh session + +## Method 2: GitHub Token +1. Go to: https://github.com/settings/tokens/new +2. Add description: "LinearSolve.jl Telemetry" +3. Select scope: "public_repo" (for commenting on issues) +4. Click "Generate token" and copy it +5. Set environment variable: `ENV["GITHUB_TOKEN"] = "your_token_here"` +6. Run this function + +# Arguments +- `results`: AutotuneResults object from autotune_setup + +# Examples +```julia +# Run benchmarks +results = autotune_setup() + +# Share results with the community +share_results(results) +``` +""" +function share_results(results::AutotuneResults) + @info "📤 Preparing to share benchmark results with the community..." + + # Extract from AutotuneResults + results_df = results.results_df + sysinfo = results.sysinfo + + # Get system info + system_info = sysinfo + + # Categorize results + categories = categorize_results(results_df) + + # Set up authentication + @info "🔗 Setting up GitHub authentication..." + @info "ℹ️ For setup instructions, see the documentation or visit:" + @info " https://cli.github.com/ (for gh CLI)" + @info " https://github.com/settings/tokens/new (for token)" + + github_auth = setup_github_authentication() + + if github_auth === nothing || github_auth[1] === nothing + @warn "❌ GitHub authentication not available." + @info "📝 To share results, please set up authentication:" + @info " Option 1: Install gh CLI and run: gh auth login" + @info " Option 2: Create a GitHub token and set: ENV[\"GITHUB_TOKEN\"] = \"your_token\"" + + # Save results locally as fallback + timestamp = replace(string(Dates.now()), ":" => "-") + fallback_file = "autotune_results_$(timestamp).md" + markdown_content = format_results_for_github(results_df, system_info, categories) + open(fallback_file, "w") do f + write(f, markdown_content) + end + @info "📁 Results saved locally to $fallback_file" + @info " You can manually share this file on the issue tracker." + return end + + # Format results + markdown_content = format_results_for_github(results_df, system_info, categories) + + # Upload to GitHub (without plots) + upload_to_github(markdown_content, nothing, github_auth, results_df, system_info, categories) + + @info "✅ Thank you for contributing to the LinearSolve.jl community!" end end diff --git a/lib/LinearSolveAutotune/src/benchmarking.jl b/lib/LinearSolveAutotune/src/benchmarking.jl index 4db48af73..39ba86dbf 100644 --- a/lib/LinearSolveAutotune/src/benchmarking.jl +++ b/lib/LinearSolveAutotune/src/benchmarking.jl @@ -1,5 +1,7 @@ # Core benchmarking functionality +using ProgressMeter + """ test_algorithm_compatibility(alg, eltype::Type, test_size::Int=4) @@ -58,32 +60,25 @@ function filter_compatible_algorithms(algorithms, alg_names, eltype::Type) compatible_algs = [] compatible_names = String[] - @info "Testing algorithm compatibility with $(eltype)..." - for (alg, name) in zip(algorithms, alg_names) if test_algorithm_compatibility(alg, eltype) push!(compatible_algs, alg) push!(compatible_names, name) - @debug "✓ $name compatible with $eltype" - else - @debug "✗ $name not compatible with $eltype" end end - @info "Found $(length(compatible_algs))/$(length(algorithms)) algorithms compatible with $eltype" - return compatible_algs, compatible_names end """ - benchmark_algorithms(sizes, algorithms, alg_names, eltypes; - samples=5, seconds=0.5, large_matrices=false) + benchmark_algorithms(matrix_sizes, algorithms, alg_names, eltypes; + samples=5, seconds=0.5, sizes=[:small, :medium]) Benchmark the given algorithms across different matrix sizes and element types. Returns a DataFrame with results including element type information. """ -function benchmark_algorithms(sizes, algorithms, alg_names, eltypes; - samples = 5, seconds = 0.5, large_matrices = false) +function benchmark_algorithms(matrix_sizes, algorithms, alg_names, eltypes; + samples = 5, seconds = 0.5, sizes = [:tiny, :small, :medium, :large]) # Set benchmark parameters old_params = BenchmarkTools.DEFAULT_PARAMETERS @@ -92,11 +87,21 @@ function benchmark_algorithms(sizes, algorithms, alg_names, eltypes; # Initialize results DataFrame results_data = [] + + # Calculate total number of benchmarks for progress bar + total_benchmarks = 0 + for eltype in eltypes + # Pre-filter to estimate the actual number + test_algs, _ = filter_compatible_algorithms(algorithms, alg_names, eltype) + total_benchmarks += length(matrix_sizes) * length(test_algs) + end + + # Create progress bar + progress = Progress(total_benchmarks, desc="Benchmarking: ", + barlen=50, showspeed=true) try for eltype in eltypes - @info "Benchmarking with element type: $eltype" - # Filter algorithms for this element type compatible_algs, compatible_names = filter_compatible_algorithms(algorithms, alg_names, eltype) @@ -105,9 +110,7 @@ function benchmark_algorithms(sizes, algorithms, alg_names, eltypes; continue end - for n in sizes - @info "Benchmarking $n × $n matrices with $eltype..." - + for n in matrix_sizes # Create test problem with specified element type rng = MersenneTwister(123) # Consistent seed for reproducibility A = rand(rng, eltype, n, n) @@ -115,6 +118,10 @@ function benchmark_algorithms(sizes, algorithms, alg_names, eltypes; u0 = rand(rng, eltype, n) for (alg, name) in zip(compatible_algs, compatible_names) + # Update progress description + ProgressMeter.update!(progress, + desc="Benchmarking $name on $(n)×$(n) $eltype matrix: ") + gflops = 0.0 success = true error_msg = "" @@ -142,7 +149,7 @@ function benchmark_algorithms(sizes, algorithms, alg_names, eltypes; catch e success = false error_msg = string(e) - @warn "Algorithm $name failed for size $n with $eltype: $error_msg" + # Don't warn for each failure, just record it end # Store result with element type information @@ -155,6 +162,9 @@ function benchmark_algorithms(sizes, algorithms, alg_names, eltypes; success = success, error = error_msg )) + + # Update progress + ProgressMeter.next!(progress) end end end @@ -168,25 +178,45 @@ function benchmark_algorithms(sizes, algorithms, alg_names, eltypes; end """ - get_benchmark_sizes(large_matrices::Bool=false) + get_benchmark_sizes(size_categories::Vector{Symbol}) + +Get the matrix sizes to benchmark based on the requested size categories. -Get the matrix sizes to benchmark based on the large_matrices flag. +Size categories: +- `:tiny` - 5:5:20 (for very small problems) +- `:small` - 20:20:100 (for small problems) +- `:medium` - 100:50:300 (for typical problems) +- `:large` - 300:100:1000 (for larger problems) +- `:big` - 10000:1000:100000 (for very large/GPU problems) """ -function get_benchmark_sizes(large_matrices::Bool = false) - if large_matrices - # For GPU benchmarking, include much larger sizes up to 10000 - return vcat(4:8:128, 150:50:500, 600:100:1000, - 1200:200:2000, 2500:500:5000, 6000:1000:10000) - else - # Default sizes similar to existing benchmarks - return vcat(4:8:128, 150:50:500) +function get_benchmark_sizes(size_categories::Vector{Symbol}) + sizes = Int[] + + for category in size_categories + if category == :tiny + append!(sizes, 5:5:20) + elseif category == :small + append!(sizes, 20:20:100) + elseif category == :medium + append!(sizes, 100:50:300) + elseif category == :large + append!(sizes, 300:100:1000) + elseif category == :big + append!(sizes, 10000:1000:100000) + else + @warn "Unknown size category: $category. Skipping." + end end + + # Remove duplicates and sort + return sort(unique(sizes)) end """ categorize_results(df::DataFrame) Categorize the benchmark results into size ranges and find the best algorithm for each range and element type. +For complex types, avoids RFLUFactorization if possible due to known issues. """ function categorize_results(df::DataFrame) # Filter successful results @@ -230,13 +260,38 @@ function categorize_results(df::DataFrame) # Calculate average GFLOPs for each algorithm in this range avg_results = combine(groupby(range_df, :algorithm), :gflops => mean => :avg_gflops) + + # Sort by performance + sort!(avg_results, :avg_gflops, rev=true) - # Find the best algorithm + # Find the best algorithm (for complex types, avoid RFLU if possible) if nrow(avg_results) > 0 - best_idx = argmax(avg_results.avg_gflops) - best_alg = avg_results.algorithm[best_idx] + best_alg = avg_results.algorithm[1] + + # For complex types, check if best is RFLU and we have alternatives + if (eltype == "ComplexF32" || eltype == "ComplexF64") && + (contains(best_alg, "RFLU") || contains(best_alg, "RecursiveFactorization")) + + # Look for the best non-RFLU algorithm + for i in 2:nrow(avg_results) + alt_alg = avg_results.algorithm[i] + if !contains(alt_alg, "RFLU") && !contains(alt_alg, "RecursiveFactorization") + # Check if performance difference is not too large (within 20%) + perf_ratio = avg_results.avg_gflops[i] / avg_results.avg_gflops[1] + if perf_ratio > 0.8 + @info "Using $alt_alg instead of $best_alg for $eltype at $range_name ($(round(100*perf_ratio, digits=1))% of RFLU performance) to avoid complex number issues" + best_alg = alt_alg + break + else + @warn "RFLUFactorization is best for $eltype at $range_name but has complex number issues. Alternative algorithms are >20% slower." + end + end + end + end + category_key = "$(eltype)_$(range_name)" categories[category_key] = best_alg + best_idx = findfirst(==(best_alg), avg_results.algorithm) @info "Best algorithm for $eltype size range $range_name: $best_alg ($(round(avg_results.avg_gflops[best_idx], digits=2)) GFLOPs avg)" end end diff --git a/lib/LinearSolveAutotune/src/gpu_detection.jl b/lib/LinearSolveAutotune/src/gpu_detection.jl index a50074d06..8a04fc8e0 100644 --- a/lib/LinearSolveAutotune/src/gpu_detection.jl +++ b/lib/LinearSolveAutotune/src/gpu_detection.jl @@ -1,5 +1,7 @@ # GPU hardware and package detection +using CPUSummary + """ is_cuda_available() @@ -76,10 +78,28 @@ function get_system_info() info["julia_version"] = string(VERSION) info["os"] = string(Sys.KERNEL) + info["os_name"] = Sys.iswindows() ? "Windows" : Sys.islinux() ? "Linux" : Sys.isapple() ? "macOS" : "Other" info["arch"] = string(Sys.ARCH) - info["cpu_name"] = Sys.cpu_info()[1].model - info["num_cores"] = Sys.CPU_THREADS + + # Use CPUSummary where available, fallback to Sys otherwise + try + info["cpu_name"] = string(Sys.CPU_NAME) + catch + info["cpu_name"] = "Unknown" + end + + # CPUSummary.num_cores() returns the physical cores (as Static.StaticInt) + info["num_cores"] = Int(CPUSummary.num_cores()) + info["num_logical_cores"] = Sys.CPU_THREADS info["num_threads"] = Threads.nthreads() + + # BLAS threads + try + info["blas_num_threads"] = LinearAlgebra.BLAS.get_num_threads() + catch + info["blas_num_threads"] = 1 + end + info["blas_vendor"] = string(LinearAlgebra.BLAS.vendor()) info["has_cuda"] = is_cuda_available() info["has_metal"] = is_metal_available() @@ -147,11 +167,17 @@ function get_detailed_system_info() end try - system_data["cpu_cores"] = Sys.CPU_THREADS + system_data["cpu_cores"] = Int(CPUSummary.num_cores()) catch system_data["cpu_cores"] = "unknown" end + try + system_data["cpu_logical_cores"] = Sys.CPU_THREADS + catch + system_data["cpu_logical_cores"] = "unknown" + end + try system_data["julia_threads"] = Threads.nthreads() catch @@ -172,12 +198,16 @@ function get_detailed_system_info() # CPU details try - cpu_info = Sys.cpu_info()[1] - system_data["cpu_name"] = cpu_info.model - system_data["cpu_speed_mhz"] = cpu_info.speed + system_data["cpu_name"] = string(Sys.CPU_NAME) catch system_data["cpu_name"] = "unknown" - system_data["cpu_speed_mhz"] = "unknown" + end + + try + # Architecture info from Sys + system_data["cpu_architecture"] = string(Sys.ARCH) + catch + system_data["cpu_architecture"] = "unknown" end # Categorize CPU vendor for easy analysis diff --git a/lib/LinearSolveAutotune/src/preferences.jl b/lib/LinearSolveAutotune/src/preferences.jl index e2d8457fe..6db7fbecc 100644 --- a/lib/LinearSolveAutotune/src/preferences.jl +++ b/lib/LinearSolveAutotune/src/preferences.jl @@ -5,24 +5,122 @@ Set LinearSolve preferences based on the categorized benchmark results. These preferences are stored in the main LinearSolve.jl package. -Handles element type-specific preferences with keys like "Float64_0-128". + +The function handles type fallbacks: +- If Float32 wasn't benchmarked, uses Float64 results +- If ComplexF64 wasn't benchmarked, uses ComplexF32 results (if available) or Float64 +- If ComplexF32 wasn't benchmarked, uses Float64 results +- For complex types, avoids RFLUFactorization due to known issues """ function set_algorithm_preferences(categories::Dict{String, String}) @info "Setting LinearSolve preferences based on benchmark results..." - - for (category_key, algorithm) in categories - # Handle element type specific keys like "Float64_0-128" - # Convert to safe preference key format - pref_key = "best_algorithm_$(replace(category_key, "+" => "plus", "-" => "_"))" - - # Set preferences in LinearSolve.jl, not LinearSolveAutotune (force=true allows overwriting) - Preferences.set_preferences!(LinearSolve, pref_key => algorithm; force = true) - @info "Set preference $pref_key = $algorithm in LinearSolve.jl" + + # Define the size category names we use + size_categories = ["tiny", "small", "medium", "large", "big"] + + # Define the element types we want to set preferences for + target_eltypes = ["Float32", "Float64", "ComplexF32", "ComplexF64"] + + # Extract benchmarked results by element type and size + benchmarked = Dict{String, Dict{String, String}}() + for (key, algorithm) in categories + if contains(key, "_") + eltype, size_range = split(key, "_", limit=2) + if !haskey(benchmarked, eltype) + benchmarked[eltype] = Dict{String, String}() + end + benchmarked[eltype][size_range] = algorithm + end end - + + # Helper function to get best algorithm for complex types (avoiding RFLU) + function get_complex_algorithm(results_df, eltype_str, size_range) + # If we have direct benchmark results, use them + if haskey(benchmarked, eltype_str) && haskey(benchmarked[eltype_str], size_range) + alg = benchmarked[eltype_str][size_range] + # Check if it's RFLU and we should avoid it for complex + if contains(alg, "RFLU") || contains(alg, "RecursiveFactorization") + # Find the second best for this case + # We'd need the full results DataFrame to do this properly + # For now, we'll just flag it + @warn "RFLUFactorization selected for $eltype_str at size $size_range, but it has known issues with complex numbers" + end + return alg + end + return nothing + end + + # Process each target element type and size combination + for eltype in target_eltypes + for size_cat in size_categories + # Map size categories to the range strings used in categories + size_range = if size_cat == "tiny" + "0-128" # Maps to tiny range + elseif size_cat == "small" + "0-128" # Small also uses this range + elseif size_cat == "medium" + "128-256" # Medium range + elseif size_cat == "large" + "256-512" # Large range + elseif size_cat == "big" + "512+" # Big range + else + continue + end + + # Determine the algorithm based on fallback rules + algorithm = nothing + + if eltype == "Float64" + # Float64 should be directly benchmarked + if haskey(benchmarked, "Float64") && haskey(benchmarked["Float64"], size_range) + algorithm = benchmarked["Float64"][size_range] + end + elseif eltype == "Float32" + # Float32: use Float32 results if available, else use Float64 + if haskey(benchmarked, "Float32") && haskey(benchmarked["Float32"], size_range) + algorithm = benchmarked["Float32"][size_range] + elseif haskey(benchmarked, "Float64") && haskey(benchmarked["Float64"], size_range) + algorithm = benchmarked["Float64"][size_range] + end + elseif eltype == "ComplexF32" + # ComplexF32: use ComplexF32 if available, else Float64 (avoiding RFLU) + if haskey(benchmarked, "ComplexF32") && haskey(benchmarked["ComplexF32"], size_range) + algorithm = benchmarked["ComplexF32"][size_range] + elseif haskey(benchmarked, "Float64") && haskey(benchmarked["Float64"], size_range) + algorithm = benchmarked["Float64"][size_range] + # Check for RFLU and warn + if contains(algorithm, "RFLU") || contains(algorithm, "RecursiveFactorization") + @warn "Would use RFLUFactorization for ComplexF32 at $size_cat, but it has issues with complex numbers. Consider benchmarking ComplexF32 directly." + end + end + elseif eltype == "ComplexF64" + # ComplexF64: use ComplexF64 if available, else ComplexF32, else Float64 (avoiding RFLU) + if haskey(benchmarked, "ComplexF64") && haskey(benchmarked["ComplexF64"], size_range) + algorithm = benchmarked["ComplexF64"][size_range] + elseif haskey(benchmarked, "ComplexF32") && haskey(benchmarked["ComplexF32"], size_range) + algorithm = benchmarked["ComplexF32"][size_range] + elseif haskey(benchmarked, "Float64") && haskey(benchmarked["Float64"], size_range) + algorithm = benchmarked["Float64"][size_range] + # Check for RFLU and warn + if contains(algorithm, "RFLU") || contains(algorithm, "RecursiveFactorization") + @warn "Would use RFLUFactorization for ComplexF64 at $size_cat, but it has issues with complex numbers. Consider benchmarking ComplexF64 directly." + end + end + end + + # Set the preference if we have an algorithm + if algorithm !== nothing + pref_key = "best_algorithm_$(eltype)_$(size_cat)" + Preferences.set_preferences!(LinearSolve, pref_key => algorithm; force = true) + @info "Set preference $pref_key = $algorithm in LinearSolve.jl" + end + end + end + # Set a timestamp for when these preferences were created Preferences.set_preferences!(LinearSolve, "autotune_timestamp" => string(Dates.now()); force = true) - + @info "Preferences updated in LinearSolve.jl. You may need to restart Julia for changes to take effect." end @@ -30,34 +128,26 @@ end get_algorithm_preferences() Get the current algorithm preferences from LinearSolve.jl. -Handles both legacy and element type-specific preferences. +Returns preferences organized by element type and size category. """ function get_algorithm_preferences() prefs = Dict{String, String}() - - # Get all LinearSolve preferences by checking common preference patterns - # Since there's no direct way to get all preferences, we'll check for known patterns - common_patterns = [ - # Element type + size range combinations - "Float64_0_128", "Float64_128_256", "Float64_256_512", "Float64_512plus", - "Float32_0_128", "Float32_128_256", "Float32_256_512", "Float32_512plus", - "ComplexF64_0_128", "ComplexF64_128_256", "ComplexF64_256_512", "ComplexF64_512plus", - "ComplexF32_0_128", "ComplexF32_128_256", "ComplexF32_256_512", "ComplexF32_512plus", - "BigFloat_0_128", "BigFloat_128_256", "BigFloat_256_512", "BigFloat_512plus", - # Legacy patterns without element type - "0_128", "128_256", "256_512", "512plus" - ] - - for pattern in common_patterns - pref_key = "best_algorithm_$pattern" - value = Preferences.load_preference(LinearSolve, pref_key, nothing) - if value !== nothing - # Convert back to human-readable key - readable_key = replace(pattern, "_" => "-", "plus" => "+") - prefs[readable_key] = value + + # Define the patterns we look for + target_eltypes = ["Float32", "Float64", "ComplexF32", "ComplexF64"] + size_categories = ["tiny", "small", "medium", "large", "big"] + + for eltype in target_eltypes + for size_cat in size_categories + pref_key = "best_algorithm_$(eltype)_$(size_cat)" + value = Preferences.load_preference(LinearSolve, pref_key, nothing) + if value !== nothing + readable_key = "$(eltype)_$(size_cat)" + prefs[readable_key] = value + end end end - + return prefs end @@ -65,37 +155,29 @@ end clear_algorithm_preferences() Clear all autotune-related preferences from LinearSolve.jl. -Handles both legacy and element type-specific preferences. """ function clear_algorithm_preferences() @info "Clearing LinearSolve autotune preferences..." - - # Clear known preference patterns - common_patterns = [ - # Element type + size range combinations - "Float64_0_128", "Float64_128_256", "Float64_256_512", "Float64_512plus", - "Float32_0_128", "Float32_128_256", "Float32_256_512", "Float32_512plus", - "ComplexF64_0_128", "ComplexF64_128_256", "ComplexF64_256_512", "ComplexF64_512plus", - "ComplexF32_0_128", "ComplexF32_128_256", "ComplexF32_256_512", "ComplexF32_512plus", - "BigFloat_0_128", "BigFloat_128_256", "BigFloat_256_512", "BigFloat_512plus", - # Legacy patterns without element type - "0_128", "128_256", "256_512", "512plus" - ] - - for pattern in common_patterns - pref_key = "best_algorithm_$pattern" - # Check if preference exists before trying to delete - if Preferences.has_preference(LinearSolve, pref_key) - Preferences.delete_preferences!(LinearSolve, pref_key; force = true) - @info "Cleared preference: $pref_key" + + # Define the patterns we look for + target_eltypes = ["Float32", "Float64", "ComplexF32", "ComplexF64"] + size_categories = ["tiny", "small", "medium", "large", "big"] + + for eltype in target_eltypes + for size_cat in size_categories + pref_key = "best_algorithm_$(eltype)_$(size_cat)" + if Preferences.has_preference(LinearSolve, pref_key) + Preferences.delete_preferences!(LinearSolve, pref_key; force = true) + @info "Cleared preference: $pref_key" + end end end - + # Clear timestamp if Preferences.has_preference(LinearSolve, "autotune_timestamp") Preferences.delete_preferences!(LinearSolve, "autotune_timestamp"; force = true) end - + @info "Preferences cleared from LinearSolve.jl." end @@ -106,19 +188,32 @@ Display the current algorithm preferences from LinearSolve.jl in a readable form """ function show_current_preferences() prefs = get_algorithm_preferences() - + if isempty(prefs) println("No autotune preferences currently set in LinearSolve.jl.") return end - + println("Current LinearSolve.jl autotune preferences:") println("="^50) - - for (range, algorithm) in sort(prefs) - println(" Size range $range: $algorithm") + + # Group by element type for better display + by_eltype = Dict{String, Vector{Tuple{String, String}}}() + for (key, algorithm) in prefs + eltype, size_cat = split(key, "_", limit=2) + if !haskey(by_eltype, eltype) + by_eltype[eltype] = Vector{Tuple{String, String}}() + end + push!(by_eltype[eltype], (size_cat, algorithm)) end - + + for eltype in sort(collect(keys(by_eltype))) + println("\n$eltype:") + for (size_cat, algorithm) in sort(by_eltype[eltype]) + println(" $size_cat: $algorithm") + end + end + timestamp = Preferences.load_preference(LinearSolve, "autotune_timestamp", "unknown") - println(" Last updated: $timestamp") -end + println("\nLast updated: $timestamp") +end \ No newline at end of file diff --git a/lib/LinearSolveAutotune/src/telemetry.jl b/lib/LinearSolveAutotune/src/telemetry.jl index adcd66b49..64d8f94b0 100644 --- a/lib/LinearSolveAutotune/src/telemetry.jl +++ b/lib/LinearSolveAutotune/src/telemetry.jl @@ -20,7 +20,7 @@ function setup_github_authentication() end end catch e - @warn "An error occurred while checking `gh` CLI status. Falling back to token auth. Error: $e" + @debug "gh CLI check failed: $e" end end @@ -28,66 +28,12 @@ function setup_github_authentication() if haskey(ENV, "GITHUB_TOKEN") && !isempty(ENV["GITHUB_TOKEN"]) auth = test_github_authentication(String(ENV["GITHUB_TOKEN"])) if auth !== nothing + println("✅ Found GITHUB_TOKEN environment variable.") return (:token, auth) end end - # 3. No environment variable or gh cli - provide setup instructions and get token - max_input_attempts = 3 - - for input_attempt in 1:max_input_attempts - println() - println("🚀 Help Improve LinearSolve.jl for Everyone!") - println("="^50) - println("Your benchmark results help the community by improving automatic") - println("algorithm selection across different hardware configurations.") - println() - println("💡 Easiest method: install GitHub CLI (`gh`) and run `gh auth login`.") - println(" Alternatively, create a token with 'issues:write' scope.") - println() - println("📋 Quick GitHub Token Setup (if not using `gh`):") - println() - println("1️⃣ Open: https://github.com/settings/tokens/new?scopes=issues:write&description=LinearSolve.jl%20Telemetry") - println("2️⃣ Click 'Generate token' and copy it") - println() - println("🔑 Paste your GitHub token here:") - println(" (If it shows julia> prompt, just paste the token there and press Enter)") - print("Token: ") - flush(stdout) - - # Get token input - token = "" - try - sleep(0.1) - token = String(strip(readline())) - catch e - println("❌ Input error: $e. Please try again.") - continue - end - - if !isempty(token) - clean_token = strip(replace(token, r"[\r\n\t ]+" => "")) - if length(clean_token) < 10 - println("❌ Token seems too short. Please check and try again.") - continue - end - - ENV["GITHUB_TOKEN"] = clean_token - auth_result = test_github_authentication(clean_token) - if auth_result !== nothing - return (:token, auth_result) - end - delete!(ENV, "GITHUB_TOKEN") - end - - if input_attempt < max_input_attempts - println("\n🤝 Please try again - it only takes 30 seconds and greatly helps the community.") - end - end - - println("\n📊 Continuing without telemetry. Results will be saved locally.") - println("💡 You can set GITHUB_TOKEN or log in with `gh auth login` and restart Julia later.") - + # 3. No authentication available - return nothing return (nothing, nothing) end @@ -153,17 +99,25 @@ Format system information as markdown. """ function format_system_info_markdown(system_info::Dict) lines = String[] - push!(lines, "- **Julia Version**: $(system_info["julia_version"])") - push!(lines, "- **OS**: $(system_info["os"])") - push!(lines, "- **Architecture**: $(system_info["arch"])") - push!(lines, "- **CPU**: $(system_info["cpu_name"])") - push!(lines, "- **Cores**: $(system_info["num_cores"])") - push!(lines, "- **Threads**: $(system_info["num_threads"])") - push!(lines, "- **BLAS**: $(system_info["blas_vendor"])") - push!(lines, "- **MKL Available**: $(system_info["mkl_available"])") - push!(lines, "- **Apple Accelerate Available**: $(system_info["apple_accelerate_available"])") - push!(lines, "- **CUDA Available**: $(system_info["has_cuda"])") - push!(lines, "- **Metal Available**: $(system_info["has_metal"])") + push!(lines, "- **Julia Version**: $(get(system_info, "julia_version", "unknown"))") + # Handle both "os" and "os_version" keys, with os_name for display + os_display = get(system_info, "os_name", "unknown") + os_kernel = get(system_info, "os_version", get(system_info, "os", "unknown")) + push!(lines, "- **OS**: $os_display ($os_kernel)") + # Handle both "arch" and "architecture" keys + push!(lines, "- **Architecture**: $(get(system_info, "architecture", get(system_info, "arch", "unknown")))") + push!(lines, "- **CPU**: $(get(system_info, "cpu_name", "unknown"))") + # Handle both "num_cores" and "cpu_cores" keys + push!(lines, "- **Cores**: $(get(system_info, "cpu_cores", get(system_info, "num_cores", "unknown")))") + # Handle both "num_threads" and "julia_threads" keys + push!(lines, "- **Threads**: $(get(system_info, "julia_threads", get(system_info, "num_threads", "unknown")))") + push!(lines, "- **BLAS**: $(get(system_info, "blas_vendor", "unknown"))") + push!(lines, "- **MKL Available**: $(get(system_info, "mkl_available", false))") + push!(lines, "- **Apple Accelerate Available**: $(get(system_info, "apple_accelerate_available", false))") + # Handle both "has_cuda" and "cuda_available" keys + push!(lines, "- **CUDA Available**: $(get(system_info, "cuda_available", get(system_info, "has_cuda", false)))") + # Handle both "has_metal" and "metal_available" keys + push!(lines, "- **Metal Available**: $(get(system_info, "metal_available", get(system_info, "has_metal", false)))") return join(lines, "\n") end @@ -299,12 +253,13 @@ function format_detailed_results_markdown(df::DataFrame) end """ - upload_to_github(content::String, plot_files::Union{Nothing, Tuple, Dict}, auth_info::Tuple, + upload_to_github(content::String, plot_files, auth_info::Tuple, results_df::DataFrame, system_info::Dict, categories::Dict) Create a GitHub issue with benchmark results for community data collection. +Note: plot_files parameter is kept for compatibility but not used. """ -function upload_to_github(content::String, plot_files::Union{Nothing, Tuple, Dict}, auth_info::Tuple, +function upload_to_github(content::String, plot_files, auth_info::Tuple, results_df::DataFrame, system_info::Dict, categories::Dict) auth_method, auth_data = auth_info @@ -326,53 +281,6 @@ function upload_to_github(content::String, plot_files::Union{Nothing, Tuple, Dic target_repo = "SciML/LinearSolve.jl" issue_number = 669 # The existing issue for collecting autotune results - # First, upload plots to a gist if available - gist_url = nothing - raw_urls = Dict{String, String}() - plot_links = "" - - if plot_files !== nothing - @info "📊 Uploading plots to GitHub Gist..." - - # Get element type for labeling - eltype_str = if !isempty(results_df) - unique_eltypes = unique(results_df.eltype) - join(unique_eltypes, ", ") - else - "Mixed" - end - - if auth_method == :gh_cli - gist_url, raw_urls = upload_plots_to_gist_gh(plot_files, eltype_str) - elseif auth_method == :token - gist_url, raw_urls = upload_plots_to_gist(plot_files, auth_data, eltype_str) - end - - if gist_url !== nothing - # Add plot links section to the content - plot_links = """ - - ### 📊 Benchmark Plots - - View all plots in the gist: [Benchmark Plots Gist]($gist_url) - - """ - - # Embed PNG images directly in the markdown if we have raw URLs - for (name, url) in raw_urls - if endswith(name, ".png") - plot_links *= """ - #### $name - ![$(name)]($url) - - """ - end - end - - plot_links *= "---\n" - end - end - # Construct comment body cpu_name = get(system_info, "cpu_name", "unknown") os_name = get(system_info, "os", "unknown") @@ -380,7 +288,7 @@ function upload_to_github(content::String, plot_files::Union{Nothing, Tuple, Dic comment_body = """ ## Benchmark Results: $cpu_name on $os_name ($timestamp) - $plot_links + $content --- @@ -404,9 +312,6 @@ function upload_to_github(content::String, plot_files::Union{Nothing, Tuple, Dic if issue_url !== nothing @info "✅ Successfully added benchmark results to issue: $issue_url" - if gist_url !== nothing - @info "📊 Plots available at: $gist_url" - end @info "🔗 Your benchmark data has been shared with the LinearSolve.jl community!" @info "💡 View all community benchmark data: https://github.com/SciML/LinearSolve.jl/issues/669" else diff --git a/lib/LinearSolveAutotune/test/runtests.jl b/lib/LinearSolveAutotune/test/runtests.jl index 19c73ec2f..94ea6e64f 100644 --- a/lib/LinearSolveAutotune/test/runtests.jl +++ b/lib/LinearSolveAutotune/test/runtests.jl @@ -61,17 +61,32 @@ using Random end @testset "Benchmark Size Generation" begin - # Test small benchmark sizes - small_sizes = LinearSolveAutotune.get_benchmark_sizes(false) + # Test new size categories + tiny_sizes = LinearSolveAutotune.get_benchmark_sizes([:tiny]) + @test !isempty(tiny_sizes) + @test minimum(tiny_sizes) == 5 + @test maximum(tiny_sizes) == 20 + + small_sizes = LinearSolveAutotune.get_benchmark_sizes([:small]) @test !isempty(small_sizes) - @test minimum(small_sizes) >= 4 - @test maximum(small_sizes) <= 500 + @test minimum(small_sizes) == 20 + @test maximum(small_sizes) == 100 + + medium_sizes = LinearSolveAutotune.get_benchmark_sizes([:medium]) + @test !isempty(medium_sizes) + @test minimum(medium_sizes) == 100 + @test maximum(medium_sizes) == 300 - # Test large benchmark sizes - large_sizes = LinearSolveAutotune.get_benchmark_sizes(true) + large_sizes = LinearSolveAutotune.get_benchmark_sizes([:large]) @test !isempty(large_sizes) - @test minimum(large_sizes) >= 4 - @test maximum(large_sizes) >= 2000 + @test minimum(large_sizes) == 300 + @test maximum(large_sizes) == 1000 + + # Test combination + combined_sizes = LinearSolveAutotune.get_benchmark_sizes([:tiny, :small]) + @test length(combined_sizes) == length(unique(combined_sizes)) + @test minimum(combined_sizes) == 5 + @test maximum(combined_sizes) == 100 end @testset "Small Scale Benchmarking" begin @@ -81,12 +96,12 @@ using Random # Use only first 2 algorithms and small sizes for fast testing test_algs = cpu_algs[1:min(2, end)] test_names = cpu_names[1:min(2, end)] - test_sizes = [4, 8] # Very small sizes for fast testing + test_sizes = [5, 10] # Very small sizes for fast testing test_eltypes = (Float64,) # Single element type for speed results_df = LinearSolveAutotune.benchmark_algorithms( test_sizes, test_algs, test_names, test_eltypes; - samples = 1, seconds = 0.1) + samples = 1, seconds = 0.1, sizes = [:tiny]) @test isa(results_df, DataFrame) @test nrow(results_df) > 0 @@ -188,7 +203,7 @@ using Random end @testset "Preference Management" begin - # Test setting and getting preferences + # Test setting and getting preferences with new format test_categories = Dict{String, String}( "Float64_0-128" => "TestAlg1", "Float64_128-256" => "TestAlg2", @@ -206,11 +221,9 @@ using Random @test isa(retrieved_prefs, Dict{String, String}) @test !isempty(retrieved_prefs) - # Verify we can retrieve what we set - for (key, value) in test_categories - @test_broken haskey(retrieved_prefs, key) - @test_broken retrieved_prefs[key] == value - end + # The new preference system uses different keys (eltype_sizecategory) + # so we just check that preferences were set + @test length(retrieved_prefs) > 0 # Test clearing preferences LinearSolveAutotune.clear_algorithm_preferences() @@ -218,50 +231,66 @@ using Random @test isempty(cleared_prefs) end - @testset "Integration Test - Mini Autotune" begin + @testset "AutotuneResults Type" begin + # Create mock data for AutotuneResults + mock_data = [ + (size = 50, algorithm = "TestAlg1", eltype = "Float64", gflops = 10.0, success = true, error = ""), + (size = 100, algorithm = "TestAlg2", eltype = "Float64", gflops = 15.0, success = true, error = ""), + ] + + test_df = DataFrame(mock_data) + test_sysinfo = Dict("cpu_name" => "Test CPU", "os" => "TestOS", + "julia_version" => "1.0.0", "num_threads" => 4) + + results = AutotuneResults(test_df, test_sysinfo) + + @test isa(results, AutotuneResults) + @test results.results_df == test_df + @test results.sysinfo == test_sysinfo + + # Test that display works without error + io = IOBuffer() + show(io, results) + display_output = String(take!(io)) + @test contains(display_output, "LinearSolve.jl Autotune Results") + @test contains(display_output, "Test CPU") + end + + @testset "Integration Test - Mini Autotune with New API" begin # Test the full autotune_setup function with minimal parameters # This is an integration test with very small scale to ensure everything works together # Skip telemetry and use minimal settings for testing - result, sysinfo, _ = LinearSolveAutotune.autotune_setup( - large_matrices = false, - telemetry = false, - make_plot = false, + result = LinearSolveAutotune.autotune_setup( + sizes = [:tiny], set_preferences = false, samples = 1, seconds = 0.1, eltypes = (Float64,) # Single element type for speed ) - @test isa(result, DataFrame) - @test nrow(result) > 0 - @test hasproperty(result, :size) - @test hasproperty(result, :algorithm) - @test hasproperty(result, :eltype) - @test hasproperty(result, :gflops) - @test hasproperty(result, :success) + @test isa(result, AutotuneResults) + @test isa(result.results_df, DataFrame) + @test isa(result.sysinfo, Dict) + @test nrow(result.results_df) > 0 + @test hasproperty(result.results_df, :size) + @test hasproperty(result.results_df, :algorithm) + @test hasproperty(result.results_df, :eltype) + @test hasproperty(result.results_df, :gflops) + @test hasproperty(result.results_df, :success) # Test with multiple element types result_multi = LinearSolveAutotune.autotune_setup( - large_matrices = false, - telemetry = false, - make_plot = true, # Test plotting integration + sizes = [:tiny], set_preferences = false, samples = 1, seconds = 0.1, eltypes = (Float64, Float32) ) - # Should return tuple of (DataFrame, Dataframe, Dict) when make_plot=true - @test isa(result_multi, Tuple) - @test length(result_multi) == 3 - @test isa(result_multi[1], DataFrame) - @test isa(result_multi[2], DataFrame) - @test isa(result_multi[3], Dict) # Plots dictionary - - df, plots = result_multi + @test isa(result_multi, AutotuneResults) + df = result_multi.results_df @test nrow(df) > 0 - @test !isempty(plots) # Check that we have results for multiple element types eltypes_in_results = unique(df.eltype)