From c9d49791ed88b1fe993477ebede51bc7ce49a3cc Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Mon, 29 Sep 2025 11:35:05 +0200 Subject: [PATCH 1/9] Remove Distributed --- test/bench_simplify.jl | 60 ++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/test/bench_simplify.jl b/test/bench_simplify.jl index 91fcd193..55d2eb0c 100644 --- a/test/bench_simplify.jl +++ b/test/bench_simplify.jl @@ -12,7 +12,7 @@ using Pkg if ! ("Test" ∈ keys(Pkg.project().dependencies)) using TestEnv; TestEnv.activate() end -using KiteModels, LinearAlgebra, Statistics, Test, Distributed +using KiteModels, LinearAlgebra, Statistics, Test include("bench_ref.jl") # Simulation parameters @@ -71,37 +71,39 @@ function run_benchmark_subprocess() write(f, benchmark_script) end - try - # Run the benchmark in a separate Julia process - result = run(`julia --project=. $temp_script`) - - if result.exitcode == 0 - # Read results from temporary file - if isfile("benchmark_results.tmp") - lines = readlines("benchmark_results.tmp") - time_ = parse(Float64, lines[1]) - rel_performance = parse(Float64, lines[2]) - - @info "Simplify took $time_ seconds" - @info "Relative performance: $rel_performance" - @test rel_performance > 0.8 - - # Clean up temporary files - rm("benchmark_results.tmp", force=true) - rm(temp_script, force=true) - - return time_, rel_performance + @testset "Testing performance of simplify..." begin + try + # Run the benchmark in a separate Julia process + result = run(`julia --project=. $temp_script`) + + if result.exitcode == 0 + # Read results from temporary file + if isfile("benchmark_results.tmp") + lines = readlines("benchmark_results.tmp") + time_ = parse(Float64, lines[1]) + rel_performance = parse(Float64, lines[2]) + + @info "Simplify took $time_ seconds" + @info "Relative performance: $rel_performance" + @test rel_performance > 0.8 + + # Clean up temporary files + rm("benchmark_results.tmp", force=true) + rm(temp_script, force=true) + + return time_, rel_performance + else + error("Benchmark results file not found") + end else - error("Benchmark results file not found") + error("Benchmark process failed with exit code $(result.exitcode)") end - else - error("Benchmark process failed with exit code $(result.exitcode)") + catch e + # Clean up temporary files in case of error + rm("benchmark_results.tmp", force=true) + rm(temp_script, force=true) + rethrow(e) end - catch e - # Clean up temporary files in case of error - rm("benchmark_results.tmp", force=true) - rm(temp_script, force=true) - rethrow(e) end end From 968455436b0be00f14be926054ae0573914b5921 Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Mon, 29 Sep 2025 11:35:25 +0200 Subject: [PATCH 2/9] Add bench_simplify.jl to runtests.jl --- test/runtests.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 92ef7faf..27ad6402 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,6 +29,9 @@ KiteUtils.set_data_path("") if build_is_production_build include("bench3.jl") include("bench4.jl") + if ! haskey(ENV, "NO_MTK") + include("bench_simplify.jl") + end end if ! haskey(ENV, "NO_MTK") include("test_ram_air_kite.jl") From 2bb4c5070dcd602f0aa9cba0fb9ddacf826d8368 Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Mon, 29 Sep 2025 12:45:56 +0200 Subject: [PATCH 3/9] Do not use testenv for tests --- test/bench4.jl | 4 +- test/bench_simplify.jl | 160 ++++++++++++++++++++------------------- test/create_sys_image.jl | 1 - test/plot_kps3.jl | 4 +- test/plot_kps4.jl | 4 +- test/test_kps3_fails.jl | 4 +- test/test_kps4.jl | 1 - 7 files changed, 89 insertions(+), 89 deletions(-) diff --git a/test/bench4.jl b/test/bench4.jl index 0ea9c5d1..6a2f12c6 100644 --- a/test/bench4.jl +++ b/test/bench4.jl @@ -3,8 +3,8 @@ using Pkg if ! ("PackageCompiler" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() - Pkg.update() + # Previously attempted to activate a TestEnv fallback. Assume tests run in project. + Pkg.update() # keep update behaviour end using Test, BenchmarkTools, StaticArrays, LinearAlgebra, KiteUtils using KiteModels, KitePodModels diff --git a/test/bench_simplify.jl b/test/bench_simplify.jl index 55d2eb0c..f6d5474d 100644 --- a/test/bench_simplify.jl +++ b/test/bench_simplify.jl @@ -9,12 +9,15 @@ if VERSION.minor==12 end using Pkg -if ! ("Test" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() -end +# TestEnv was previously used to activate the test environment dynamically. +# All required test dependencies are now listed under [targets] test in Project.toml, +# so we can rely on the active project without pulling in TestEnv. using KiteModels, LinearAlgebra, Statistics, Test include("bench_ref.jl") +# Repository root for subprocess scripts +const REPO_ROOT = normpath(@__DIR__, "..") + # Simulation parameters dt = 0.05 total_time = 10.0 # Longer simulation to see oscillations @@ -27,88 +30,93 @@ steering_magnitude = 10.0 # Magnitude of steering input [Nm] # Function to run benchmark in separate Julia process function run_benchmark_subprocess() - # Create a temporary script file for the benchmark - benchmark_script = """ - using Pkg - if ! ("Test" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() - end - using KiteModels, LinearAlgebra, Statistics - include("test/bench_ref.jl") - - SIMPLE = $SIMPLE - T_REF = $T_REF - - # Initialize model - set = load_settings("system_ram.yaml") - set.segments = 3 - set_values = [-50, 0.0, 0.0] # Set values of the torques of the three winches. [Nm] - set.quasi_static = false - set.physical_model = SIMPLE ? "simple_ram" : "ram" - - sam = SymbolicAWEModel(set) - sam.set.abs_tol = 1e-2 - sam.set.rel_tol = 1e-2 - rm("data/model_1.11_ram_dynamic_3_seg.bin"; force=true) - - # Initialize at elevation - set.l_tethers[2] += 0.2 - set.l_tethers[3] += 0.2 - time_ = init!(sam; remake=false, reload=true, bench=true) - @info "Simplify took \$time_ seconds" - rel_performance = (T_REF / rel_cpu_performance())/time_ - - # Write results to file for parent process to read - open("benchmark_results.tmp", "w") do f - println(f, time_) - println(f, rel_performance) - end - """ - - # Write the script to a temporary file - temp_script = "temp_benchmark.jl" - open(temp_script, "w") do f - write(f, benchmark_script) - end - - @testset "Testing performance of simplify..." begin + # Use an isolated temporary directory for all intermediate files + mktempdir() do tmpdir + results_file = joinpath(tmpdir, "benchmark_results.tmp") + temp_script = joinpath(tmpdir, "temp_benchmark.jl") + + # Create benchmark script with absolute results path embedded + benchmark_script = """ + const REPO_ROOT = $(repr(REPO_ROOT)) + const RESULTS_FILE = $(repr(results_file)) + cd(REPO_ROOT) # ensure consistent base directory + using Pkg + using KiteModels, LinearAlgebra, Statistics + include(joinpath(REPO_ROOT, "test", "bench_ref.jl")) + + SIMPLE = $SIMPLE + T_REF = $T_REF + + # Initialize model + set = load_settings("system_ram.yaml") + set.segments = 3 + set_values = [-50, 0.0, 0.0] # Set values of the torques of the three winches. [Nm] + set.quasi_static = false + set.physical_model = SIMPLE ? "simple_ram" : "ram" + + sam = SymbolicAWEModel(set) + sam.set.abs_tol = 1e-2 + sam.set.rel_tol = 1e-2 + rm("data/model_1.11_ram_dynamic_3_seg.bin"; force=true) + + # Initialize at elevation + set.l_tethers[2] += 0.2 + set.l_tethers[3] += 0.2 + time_ = init!(sam; remake=false, reload=true, bench=true) + @info "Simplify took \$time_ seconds" + rel_performance = (T_REF / rel_cpu_performance())/time_ + + # Write results to file for parent process to read + open(RESULTS_FILE, "w") do f + println(f, time_) + println(f, rel_performance) + end + """ + + # Write the script to the temporary directory + open(temp_script, "w") do f + write(f, benchmark_script) + end + + success = false + time_ = NaN + relp = NaN + msg = nothing try - # Run the benchmark in a separate Julia process result = run(`julia --project=. $temp_script`) - - if result.exitcode == 0 - # Read results from temporary file - if isfile("benchmark_results.tmp") - lines = readlines("benchmark_results.tmp") - time_ = parse(Float64, lines[1]) - rel_performance = parse(Float64, lines[2]) - - @info "Simplify took $time_ seconds" - @info "Relative performance: $rel_performance" - @test rel_performance > 0.8 - - # Clean up temporary files - rm("benchmark_results.tmp", force=true) - rm(temp_script, force=true) - - return time_, rel_performance - else - error("Benchmark results file not found") - end + if result.exitcode == 0 && isfile(results_file) + lines = readlines(results_file) + time_ = parse(Float64, lines[1]) + relp = parse(Float64, lines[2]) + success = true else - error("Benchmark process failed with exit code $(result.exitcode)") + msg = "Benchmark subprocess exit=$(result.exitcode) file_exists=$(isfile(results_file))" end catch e - # Clean up temporary files in case of error - rm("benchmark_results.tmp", force=true) - rm(temp_script, force=true) - rethrow(e) + io = IOBuffer(); showerror(io, e); msg = String(take!(io)) + finally + # temp dir and contents auto-removed after mktempdir do-block end + return success, time_, relp, msg end end -# Run the benchmark in a separate process -time_, rel_performance = run_benchmark_subprocess() +ok, time_, rel_performance, err_msg = run_benchmark_subprocess() +# strict = get(ENV, "STRICT_BENCH", "0") in ("1", "true", "TRUE") +strict = true + +@testset "Testing performance of simplify..." begin + if ok + @info "Simplify took $(round(time_, digits=3)) seconds" rel_performance=rel_performance + @test rel_performance > 0.8 + elseif strict + @error "Simplify benchmark failed (strict mode)" err_msg + @test ok # will fail in strict mode + else + @warn "Simplify benchmark did not complete; marking as broken (set STRICT_BENCH=1 to enforce)." err_msg + @test_broken ok + end +end # Note: sys object is not available when running in separate process # If you need sys, you would need to serialize it or run parts in the main process diff --git a/test/create_sys_image.jl b/test/create_sys_image.jl index 0b47f76b..eb558037 100644 --- a/test/create_sys_image.jl +++ b/test/create_sys_image.jl @@ -4,7 +4,6 @@ # activate the test environment if needed using Pkg if ! ("PackageCompiler" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() Pkg.update() end @info "Loading packages ..." diff --git a/test/plot_kps3.jl b/test/plot_kps3.jl index 5776aeb8..5151a700 100644 --- a/test/plot_kps3.jl +++ b/test/plot_kps3.jl @@ -3,9 +3,7 @@ # activate the test environment if needed using Pkg -if ! ("ControlPlots" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() -end +# Removed TestEnv fallback; rely on project test target having ControlPlots. using KiteModels using KitePodModels diff --git a/test/plot_kps4.jl b/test/plot_kps4.jl index 8b65facd..0604c300 100644 --- a/test/plot_kps4.jl +++ b/test/plot_kps4.jl @@ -3,9 +3,7 @@ # activate the test environment if needed using Pkg -if ! ("ControlPlots" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() -end +# Removed TestEnv fallback; rely on project test target having ControlPlots. using KiteModels using KitePodModels diff --git a/test/test_kps3_fails.jl b/test/test_kps3_fails.jl index 8da92236..64a99b7b 100644 --- a/test/test_kps3_fails.jl +++ b/test/test_kps3_fails.jl @@ -2,9 +2,7 @@ # SPDX-License-Identifier: MIT using Pkg -if ! ("Test" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() -end +# Removed TestEnv activation; assume the test target environment is active. using Test, BenchmarkTools, StaticArrays, LinearAlgebra, KiteUtils diff --git a/test/test_kps4.jl b/test/test_kps4.jl index 61003637..167618c8 100644 --- a/test/test_kps4.jl +++ b/test/test_kps4.jl @@ -3,7 +3,6 @@ using Pkg if ! ("PackageCompiler" ∈ keys(Pkg.project().dependencies)) - using TestEnv; TestEnv.activate() Pkg.update() end using Test, BenchmarkTools, StaticArrays, LinearAlgebra, KiteUtils From e2894e80fe43c9c4e5363cc6875c62c29117136a Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Mon, 29 Sep 2025 13:12:24 +0200 Subject: [PATCH 4/9] Better result reporting --- test/bench_simplify.jl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/bench_simplify.jl b/test/bench_simplify.jl index f6d5474d..5fb6c370 100644 --- a/test/bench_simplify.jl +++ b/test/bench_simplify.jl @@ -7,8 +7,8 @@ T_REF = 48.0 # AMD Ryzen 7840U, Julia 1.11, no sys image [s] if VERSION.minor==12 T_REF /= 0.85 # Julia 1.12 is about 15% slower on AMD Ryzen 7 7840U end +msg = String[] -using Pkg # TestEnv was previously used to activate the test environment dynamically. # All required test dependencies are now listed under [targets] test in Project.toml, # so we can rely on the active project without pulling in TestEnv. @@ -63,7 +63,6 @@ function run_benchmark_subprocess() set.l_tethers[2] += 0.2 set.l_tethers[3] += 0.2 time_ = init!(sam; remake=false, reload=true, bench=true) - @info "Simplify took \$time_ seconds" rel_performance = (T_REF / rel_cpu_performance())/time_ # Write results to file for parent process to read @@ -102,22 +101,24 @@ function run_benchmark_subprocess() end ok, time_, rel_performance, err_msg = run_benchmark_subprocess() -# strict = get(ENV, "STRICT_BENCH", "0") in ("1", "true", "TRUE") -strict = true +push!(msg, ("Rel performance of simplify: $(round(rel_performance, digits=2))")) @testset "Testing performance of simplify..." begin if ok - @info "Simplify took $(round(time_, digits=3)) seconds" rel_performance=rel_performance + push!(msg, ("Simplify took: $(round(time_, digits=3)) s")) @test rel_performance > 0.8 - elseif strict - @error "Simplify benchmark failed (strict mode)" err_msg - @test ok # will fail in strict mode else - @warn "Simplify benchmark did not complete; marking as broken (set STRICT_BENCH=1 to enforce)." err_msg - @test_broken ok + @error "Simplify benchmark failed" err_msg + @test ok # will fail in strict mode end end +printstyled("\nBenchmark results for simplify:\n"; bold = true) +for i in eachindex(msg) + println(msg[i]) +end +println() + # Note: sys object is not available when running in separate process # If you need sys, you would need to serialize it or run parts in the main process nothing From 5867c8313c848577bb747b31d0363fae245f046f Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Mon, 29 Sep 2025 13:39:57 +0200 Subject: [PATCH 5/9] Fix performance test for Windows --- test/bench_simplify.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/bench_simplify.jl b/test/bench_simplify.jl index 5fb6c370..2078edc9 100644 --- a/test/bench_simplify.jl +++ b/test/bench_simplify.jl @@ -7,6 +7,9 @@ T_REF = 48.0 # AMD Ryzen 7840U, Julia 1.11, no sys image [s] if VERSION.minor==12 T_REF /= 0.85 # Julia 1.12 is about 15% slower on AMD Ryzen 7 7840U end +if Sys.iswindows() + T_REF /= 0.75 # Windows is about 25% slower than Linux on same hardware +end msg = String[] # TestEnv was previously used to activate the test environment dynamically. From f29268cbe13412820a855c66e60bb3bb213d5baf Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Mon, 29 Sep 2025 14:16:22 +0200 Subject: [PATCH 6/9] Better error handling --- test/bench_simplify.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/bench_simplify.jl b/test/bench_simplify.jl index 2078edc9..bf64a58f 100644 --- a/test/bench_simplify.jl +++ b/test/bench_simplify.jl @@ -104,7 +104,11 @@ function run_benchmark_subprocess() end ok, time_, rel_performance, err_msg = run_benchmark_subprocess() -push!(msg, ("Rel performance of simplify: $(round(rel_performance, digits=2))")) +if isnan(rel_performance) + push!(msg, ("Rel performance of simplify: $(round(rel_performance, digits=2))")) +else + push!(msg, ("Error in simplify benchmark: $(err_msg)")) +end @testset "Testing performance of simplify..." begin if ok From 5f80ce4fad2c922e5d9885eeb2674d6440acffb0 Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Mon, 29 Sep 2025 14:18:30 +0200 Subject: [PATCH 7/9] Bugfix --- test/bench_simplify.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bench_simplify.jl b/test/bench_simplify.jl index bf64a58f..8ce7b59b 100644 --- a/test/bench_simplify.jl +++ b/test/bench_simplify.jl @@ -104,7 +104,7 @@ function run_benchmark_subprocess() end ok, time_, rel_performance, err_msg = run_benchmark_subprocess() -if isnan(rel_performance) +if ! isnan(rel_performance) push!(msg, ("Rel performance of simplify: $(round(rel_performance, digits=2))")) else push!(msg, ("Error in simplify benchmark: $(err_msg)")) From 94989fd0b3746459060e2cb41980224887ad477c Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Thu, 9 Oct 2025 06:55:49 +0200 Subject: [PATCH 8/9] Revert changes by AI --- test/bench4.jl | 4 ++-- test/plot_kps3.jl | 4 +++- test/plot_kps4.jl | 4 +++- test/test_kps3_fails.jl | 4 +++- test/test_kps4.jl | 1 + 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/bench4.jl b/test/bench4.jl index 6a2f12c6..0ea9c5d1 100644 --- a/test/bench4.jl +++ b/test/bench4.jl @@ -3,8 +3,8 @@ using Pkg if ! ("PackageCompiler" ∈ keys(Pkg.project().dependencies)) - # Previously attempted to activate a TestEnv fallback. Assume tests run in project. - Pkg.update() # keep update behaviour + using TestEnv; TestEnv.activate() + Pkg.update() end using Test, BenchmarkTools, StaticArrays, LinearAlgebra, KiteUtils using KiteModels, KitePodModels diff --git a/test/plot_kps3.jl b/test/plot_kps3.jl index 5151a700..5776aeb8 100644 --- a/test/plot_kps3.jl +++ b/test/plot_kps3.jl @@ -3,7 +3,9 @@ # activate the test environment if needed using Pkg -# Removed TestEnv fallback; rely on project test target having ControlPlots. +if ! ("ControlPlots" ∈ keys(Pkg.project().dependencies)) + using TestEnv; TestEnv.activate() +end using KiteModels using KitePodModels diff --git a/test/plot_kps4.jl b/test/plot_kps4.jl index 0604c300..8b65facd 100644 --- a/test/plot_kps4.jl +++ b/test/plot_kps4.jl @@ -3,7 +3,9 @@ # activate the test environment if needed using Pkg -# Removed TestEnv fallback; rely on project test target having ControlPlots. +if ! ("ControlPlots" ∈ keys(Pkg.project().dependencies)) + using TestEnv; TestEnv.activate() +end using KiteModels using KitePodModels diff --git a/test/test_kps3_fails.jl b/test/test_kps3_fails.jl index 64a99b7b..8da92236 100644 --- a/test/test_kps3_fails.jl +++ b/test/test_kps3_fails.jl @@ -2,7 +2,9 @@ # SPDX-License-Identifier: MIT using Pkg -# Removed TestEnv activation; assume the test target environment is active. +if ! ("Test" ∈ keys(Pkg.project().dependencies)) + using TestEnv; TestEnv.activate() +end using Test, BenchmarkTools, StaticArrays, LinearAlgebra, KiteUtils diff --git a/test/test_kps4.jl b/test/test_kps4.jl index 167618c8..61003637 100644 --- a/test/test_kps4.jl +++ b/test/test_kps4.jl @@ -3,6 +3,7 @@ using Pkg if ! ("PackageCompiler" ∈ keys(Pkg.project().dependencies)) + using TestEnv; TestEnv.activate() Pkg.update() end using Test, BenchmarkTools, StaticArrays, LinearAlgebra, KiteUtils From 6048705b261e39e18eed993e7005ca23047743db Mon Sep 17 00:00:00 2001 From: Uwe Fechner Date: Thu, 9 Oct 2025 07:00:25 +0200 Subject: [PATCH 9/9] One mor revert --- test/create_sys_image.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/create_sys_image.jl b/test/create_sys_image.jl index eb558037..0b47f76b 100644 --- a/test/create_sys_image.jl +++ b/test/create_sys_image.jl @@ -4,6 +4,7 @@ # activate the test environment if needed using Pkg if ! ("PackageCompiler" ∈ keys(Pkg.project().dependencies)) + using TestEnv; TestEnv.activate() Pkg.update() end @info "Loading packages ..."