Skip to content

Commit 026272c

Browse files
arnavk23ChrisRackauckas
arnavk23
authored and
ChrisRackauckas
committed
Make CUTEst benchmarks extremely conservative to prevent OOM
- Reduce chunk size from 5 to 3 problems per chunk - Lower variable limit from 100 to 50 variables per problem - Reduce maxiters from 1e6 to 1000 iterations - Keep maxtime at 60 seconds per problem - Add aggressive problem size filtering These changes should prevent ProcessSignaled(9) OOM errors in CI while still testing a substantial number of CUTEst problems.
1 parent aa51c1b commit 026272c

File tree

5 files changed

+119
-57
lines changed

5 files changed

+119
-57
lines changed

‎benchmarks/OptimizationCUTEst/CUTEst_bounded.jmd‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
4848
"Ipopt", Symbol(sol.retcode))
4949
end
5050

51-
function run_benchmarks(problems, optimizers; chunk_size=50)
51+
function run_benchmarks(problems, optimizers; chunk_size=3)
5252
problem = String[]
5353
n_vars = Int64[]
5454
secs = Float64[]
@@ -76,12 +76,20 @@ function run_benchmarks(problems, optimizers; chunk_size=50)
7676
nlp_prob = nothing
7777
try
7878
nlp_prob = CUTEstModel(prob_name)
79+
80+
# Skip very large problems to prevent memory issues
81+
if nlp_prob.meta.nvar > 50
82+
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
83+
finalize(nlp_prob)
84+
continue
85+
end
86+
7987
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
8088

8189
for optimizer in optimizers
8290
try
83-
# Set reasonable time limit per problem
84-
sol = solve(prob, optimizer; maxiters = 1e6, maxtime = 300.0)
91+
# Set aggressive time and iteration limits
92+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 60.0)
8593

8694
@info "✓ Solved $(prob_name) with $(optimizer)"
8795
vars, time, alg, code = get_stats(sol, optimizer)

‎benchmarks/OptimizationCUTEst/CUTEst_quadratic.jmd‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
4747
"Ipopt", Symbol(sol.retcode))
4848
end
4949

50-
function run_benchmarks(problems, optimizers; chunk_size=50)
50+
function run_benchmarks(problems, optimizers; chunk_size=3)
5151
problem = String[]
5252
n_vars = Int64[]
5353
secs = Float64[]
@@ -75,12 +75,20 @@ function run_benchmarks(problems, optimizers; chunk_size=50)
7575
nlp_prob = nothing
7676
try
7777
nlp_prob = CUTEstModel(prob_name)
78+
79+
# Skip very large problems to prevent memory issues
80+
if nlp_prob.meta.nvar > 50
81+
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
82+
finalize(nlp_prob)
83+
continue
84+
end
85+
7886
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
7987

8088
for optimizer in optimizers
8189
try
82-
# Set reasonable time limit per problem
83-
sol = solve(prob, optimizer; maxiters = 1e6, maxtime = 300.0)
90+
# Set aggressive time and iteration limits
91+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 60.0)
8492

8593
@info "✓ Solved $(prob_name) with $(optimizer)"
8694
vars, time, alg, code = get_stats(sol, optimizer)

‎benchmarks/OptimizationCUTEst/CUTEst_safe_solvers.jmd‎

Lines changed: 75 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ function get_stats(sol, optimizer_name)
5151
return (length(sol.u), solve_time, optimizer_name, Symbol(sol.retcode))
5252
end
5353

54-
function run_benchmarks(problems, optimizers)
55-
"""Enhanced benchmark loop with better error handling"""
54+
function run_benchmarks(problems, optimizers; chunk_size=3)
55+
"""Enhanced benchmark loop with chunked processing and better error handling"""
5656
problem = String[]
5757
n_vars = Int64[]
5858
secs = Float64[]
@@ -62,64 +62,94 @@ function run_benchmarks(problems, optimizers)
6262
optz = length(optimizers)
6363
n = length(problems)
6464

65+
@info "Processing $(n) problems with $(optz) optimizers in chunks of $(chunk_size)"
66+
6567
broadcast(c -> sizehint!(c, optz * n), [problem, n_vars, secs, solver, retcode])
6668

6769
println("Running comprehensive benchmark:")
6870
println("$(length(problems)) problems × $(length(optimizers)) optimizers = $(length(problems) * length(optimizers)) combinations")
6971

70-
for (i, prob_name) in enumerate(problems)
71-
@printf("Problem %d/%d: %s\n", i, length(problems), prob_name)
72+
# Process problems in chunks to manage memory
73+
for chunk_start in 1:chunk_size:n
74+
chunk_end = min(chunk_start + chunk_size - 1, n)
75+
chunk_problems = problems[chunk_start:chunk_end]
7276

73-
try
74-
nlp_prob = CUTEstModel(prob_name)
75-
76-
# Skip very large problems for computational efficiency
77-
if nlp_prob.meta.nvar > 100
78-
@printf(" Skipping (too large: %d variables)\n", nlp_prob.meta.nvar)
79-
finalize(nlp_prob)
80-
continue
81-
end
82-
83-
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
77+
@info "Processing chunk $(div(chunk_start-1, chunk_size)+1)/$(div(n-1, chunk_size)+1): problems $(chunk_start)-$(chunk_end)"
78+
79+
for (idx, prob_name) in enumerate(chunk_problems)
80+
current_problem = chunk_start + idx - 1
81+
@printf("Problem %d/%d: %s\n", current_problem, n, prob_name)
8482

85-
for (optimizer_name, optimizer) in optimizers
86-
@printf(" Testing %-20s... ", optimizer_name)
83+
nlp_prob = nothing
84+
try
85+
nlp_prob = CUTEstModel(prob_name)
8786

88-
try
89-
sol = solve(prob, optimizer;
90-
maxiters = 5000,
91-
maxtime = 30.0, # 30 seconds timeout per solve
92-
abstol = 1e-6,
93-
reltol = 1e-6)
94-
95-
vars, time, alg, code = get_stats(sol, optimizer_name)
96-
97-
push!(problem, prob_name)
98-
push!(n_vars, vars)
99-
push!(secs, time)
100-
push!(solver, alg)
101-
push!(retcode, code)
102-
103-
success = code == :Success
104-
@printf("%s (%.3fs)\n", success ? "✓" : "✗", time)
87+
# Skip very large problems for computational efficiency
88+
if nlp_prob.meta.nvar > 50
89+
@printf(" Skipping (too large: %d variables)\n", nlp_prob.meta.nvar)
90+
finalize(nlp_prob)
91+
continue
92+
end
93+
94+
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
95+
96+
for (optimizer_name, optimizer) in optimizers
97+
@printf(" Testing %-20s... ", optimizer_name)
10598

106-
catch e
107-
@printf("ERROR: %s\n", string(e))
108-
# Still record failed attempts
99+
try
100+
sol = solve(prob, optimizer;
101+
maxiters = 1000,
102+
maxtime = 30.0, # 30 seconds timeout per solve
103+
abstol = 1e-6,
104+
reltol = 1e-6)
105+
106+
vars, time, alg, code = get_stats(sol, optimizer_name)
107+
108+
push!(problem, prob_name)
109+
push!(n_vars, vars)
110+
push!(secs, time)
111+
push!(solver, alg)
112+
push!(retcode, code)
113+
114+
success = code == :Success
115+
@printf("%s (%.3fs)\n", success ? "✓" : "✗", time)
116+
117+
catch e
118+
@printf("ERROR: %s\n", string(e))
119+
# Still record failed attempts
120+
push!(problem, prob_name)
121+
push!(n_vars, nlp_prob.meta.nvar)
122+
push!(secs, NaN)
123+
push!(solver, optimizer_name)
124+
push!(retcode, :Error)
125+
end
126+
end
127+
128+
catch e
129+
@printf(" Failed to load problem: %s\n", string(e))
130+
# Add failure entries for all optimizers
131+
for (optimizer_name, optimizer) in optimizers
109132
push!(problem, prob_name)
110-
push!(n_vars, nlp_prob.meta.nvar)
133+
push!(n_vars, -1)
111134
push!(secs, NaN)
112135
push!(solver, optimizer_name)
113-
push!(retcode, :Error)
136+
push!(retcode, :LOAD_FAILED)
137+
end
138+
finally
139+
# Clean up resources
140+
if nlp_prob !== nothing
141+
try
142+
finalize(nlp_prob)
143+
catch e
144+
@warn "Failed to finalize $(prob_name): $(e)"
145+
end
114146
end
115147
end
116-
117-
finalize(nlp_prob)
118-
119-
catch e
120-
@printf(" Failed to load problem: %s\n", string(e))
121-
continue
122148
end
149+
150+
# Force garbage collection after each chunk
151+
GC.gc()
152+
@info "Completed chunk, memory usage cleaned up"
123153
end
124154

125155
return DataFrame(problem = problem, n_vars = n_vars, secs = secs, solver = solver,

‎benchmarks/OptimizationCUTEst/CUTEst_unbounded.jmd‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
4747
"Ipopt", Symbol(sol.retcode))
4848
end
4949

50-
function run_benchmarks(problems, optimizers; chunk_size=50)
50+
function run_benchmarks(problems, optimizers; chunk_size=3)
5151
problem = String[]
5252
n_vars = Int64[]
5353
secs = Float64[]
@@ -75,12 +75,20 @@ function run_benchmarks(problems, optimizers; chunk_size=50)
7575
nlp_prob = nothing
7676
try
7777
nlp_prob = CUTEstModel(prob_name)
78+
79+
# Skip very large problems to prevent memory issues
80+
if nlp_prob.meta.nvar > 50
81+
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
82+
finalize(nlp_prob)
83+
continue
84+
end
85+
7886
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
7987

8088
for optimizer in optimizers
8189
try
82-
# Set reasonable time limit per problem
83-
sol = solve(prob, optimizer; maxiters = 1e6, maxtime = 300.0)
90+
# Set aggressive time and iteration limits
91+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 60.0)
8492

8593
@info "✓ Solved $(prob_name) with $(optimizer)"
8694
vars, time, alg, code = get_stats(sol, optimizer)

‎benchmarks/OptimizationCUTEst/CUTEst_unconstrained.jmd‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
4848
"Ipopt", Symbol(sol.retcode))
4949
end
5050

51-
function run_benchmarks(problems, optimizers; chunk_size=50)
51+
function run_benchmarks(problems, optimizers; chunk_size=3)
5252
problem = String[]
5353
n_vars = Int64[]
5454
secs = Float64[]
@@ -76,12 +76,20 @@ function run_benchmarks(problems, optimizers; chunk_size=50)
7676
nlp_prob = nothing
7777
try
7878
nlp_prob = CUTEstModel(prob_name)
79+
80+
# Skip very large problems to prevent memory issues
81+
if nlp_prob.meta.nvar > 50
82+
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
83+
finalize(nlp_prob)
84+
continue
85+
end
86+
7987
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
8088

8189
for optimizer in optimizers
8290
try
83-
# Set reasonable time limit per problem
84-
sol = solve(prob, optimizer; maxiters = 1e6, maxtime = 300.0)
91+
# Set aggressive time and iteration limits
92+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 60.0)
8593

8694
@info "✓ Solved $(prob_name) with $(optimizer)"
8795
vars, time, alg, code = get_stats(sol, optimizer)

0 commit comments

Comments
 (0)