Update of benchmarks in agentic examples

Author: emmanuellujan

examples/agentic/generate-cpu-linear-solver/Project.toml

@@ -1,5 +1,7 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 SmartSolve = "4fbb3a3c-2fa1-4c19-8d57-bae8bc1e16ac"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 

examples/agentic/generate-cpu-linear-solver/benchmark.jl

@@ -1,17 +1,126 @@
 using LinearAlgebra
 using SparseArrays
+using CUDA
 using BenchmarkTools
+using OrderedCollections
+using Plots
+
+println("CPU benchmark with error-vs-time plot:\n")
 
 include("solver.jl")
 
-println("Simple CPU benchmark:\n")
-A = sprand(10000, 10000, 0.1)
-b = rand(10000)
-display(@benchmark $A \ $b seconds=20)
-display(@benchmark proposed_fn($A, $b) seconds=10)
-x1 = A \ b
-e1 = norm(A * x1 - b)
-x2 = proposed_fn(A, b)
-e2 = norm(A * x2 - b)
-println("Error of default solver: $e1")
-println("Error of generated solver: $e2")
+# Configuration
+N = 15_000
+sparsity_levels = [0.1, 0.5, 0.9]
+solvers = OrderedDict(
+    "Default" => (A, b) -> (A \ b),
+    "Generated" => (A, b) -> proposed_fn(A, b)
+)
+
+# Store results for plotting
+results = Dict()
+
+for sparsity in sparsity_levels
+    println("\n=== Sparsity: $sparsity ===")
+    
+    # Generate problem
+    A = sprand(N, N, sparsity)
+    b = rand(N)
+    
+    results[sparsity] = Dict()
+    
+    for (solver_name, solver_fn) in solvers
+        println("  $solver_name...")
+        
+        # Warm-up
+        b_warm = copy(b)
+        try
+            x_warm = solver_fn(A, b_warm)
+        catch e
+            println("    Warning: solver failed during warm-up: $e")
+            continue
+        end
+        
+        # Benchmark
+        b_bench = copy(b)
+        try
+            bench = @benchmark begin
+                x = $(solver_fn)($A, $b_bench)
+            end seconds = 5 samples = 10
+            
+            time_ms = median(bench.times) / 1e9  # Convert to s
+            
+            # Compute error
+            b_err = copy(b)
+            x_sol = solver_fn(A, b_err)
+            error = norm(A*x_sol - b_err) / norm(b_err)
+            
+            results[sparsity][solver_name] = (time=time_ms, error=error)
+            println("    Time: $(round(time_ms, digits=3)) ms, Error: $(round(error, sigdigits=3))")
+        catch e
+            println("    Error during benchmark: $e")
+        end
+    end
+end
+
+# Create error-vs-time plot
+p = plot(
+    size=(800, 800),
+    #legend=:topright,
+    legend=:bottomright,
+    xlabel="Time (s)",
+    ylabel="Relative residual: ||Ax - b||₂ / ||b||₂",
+ #   xscale=:log10,
+    yscale=:log10,
+    guidefontsize=22,#18,
+    tickfontsize=20, #16,
+    legendfontsize=18, #14,
+    margin=5*Plots.mm,
+    framestyle=:box,
+    title="Random Matrices of Size $(N)x$(N),\n Varying Sparsity Levels (ρ) and\n CPU Solvers",
+    titlefontsize=22,
+)
+
+# Symbols encode sparsity levels; colors encode solvers.
+# Define marker for each sparsity and a color for each solver.
+## Sparsity shapes
+marker_map_sparsity = OrderedDict(0.1=>:circle, 0.5=>:square, 0.9=>:utriangle)
+## Solver color shades
+color_map_solver = OrderedDict("Default"=>:red, "Generated"=>:green)
+
+# Plot each point individually so marker shape shows sparsity and color shows solver.
+for solver_name in keys(solvers)
+    for sparsity in sparsity_levels
+        if sparsity in keys(results) && solver_name in keys(results[sparsity])
+            t = results[sparsity][solver_name].time
+            e = results[sparsity][solver_name].error
+            scatter!(p, [t], [e];
+                     label="",
+                     marker=marker_map_sparsity[sparsity],
+                     markersize=15,
+                     color=color_map_solver[solver_name],
+                     markerstrokecolor=:black,
+                     markerstrokewidth=0.0,#0.8,
+                     alpha=0.45)
+        end
+    end
+end
+
+# Create a combined legend
+for solver_name in keys(solvers)
+    for s in sparsity_levels
+        lbl = "$(solver_name), ρ:$(s)"
+        scatter!(p, [NaN], [NaN]; label=lbl,
+                 marker=marker_map_sparsity[s],
+                 markersize=15,
+                 color=color_map_solver[solver_name],
+                 markerstrokecolor=:black,
+                 markerstrokewidth=0.0, 
+                 alpha=0.45)
+    end
+end
+
+savefig(p, "error_vs_time.pdf")
+println("\n✓ Plot saved as error_vs_time.pdf")
+
+display(p)

examples/agentic/generate-cpu-linear-solver/error_vs_time.pdf

@@ -10,7 +10,7 @@ println("GPU benchmark with error-vs-time plot:\n")
 include("solver.jl")
 
 # Configuration
-N = 10_000
+N = 15_000
 sparsity_levels = [0.1, 0.5, 0.9]
 solvers = OrderedDict(
     "Default" => (Ad, bd) -> (Ad \ bd),
@@ -86,7 +86,9 @@ p = plot(
     tickfontsize=20, #16,
     legendfontsize=18, #14,
     margin=5*Plots.mm,
-    framestyle=:box
+    framestyle=:box,
+    title="Random Matrices of Size $(N)x$(N),\n Varying Sparsity Levels (ρ) and\n GPU Solvers",
+    titlefontsize=22,
 )
 
 # Symbols encode sparsity levels; colors encode solvers.