Skip to content

Commit 40bbd90

Browse files
arnavk23ChrisRackauckas
arnavk23
authored and
ChrisRackauckas
committed
changes
1 parent ffb87df commit 40bbd90

File tree

6 files changed

+109
-18
lines changed

6 files changed

+109
-18
lines changed

‎benchmarks/OptimizationCUTEst/CUTEst_bounded.jmd‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ using OptimizationMOI: MOI as MOI
2626
using DataFrames
2727
using Plots
2828
using StatsPlots
29+
using StatsBase: countmap
2930
```
3031

3132
# Benchmarks
@@ -48,7 +49,7 @@ function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
4849
"Ipopt", Symbol(sol.retcode))
4950
end
5051

51-
function run_benchmarks(problems, optimizers; chunk_size=3)
52+
function run_benchmarks(problems, optimizers; chunk_size=1)
5253
problem = String[]
5354
n_vars = Int64[]
5455
secs = Float64[]
@@ -77,7 +78,7 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
7778
try
7879
nlp_prob = CUTEstModel(prob_name)
7980

80-
# Skip extremely large problems to prevent memory issues
81+
# Generous memory limits for 100GB systems - include 5000 var problems
8182
if nlp_prob.meta.nvar > 10000
8283
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
8384
finalize(nlp_prob)
@@ -88,8 +89,8 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
8889

8990
for optimizer in optimizers
9091
try
91-
# Set aggressive time and iteration limits
92-
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 5.0)
92+
# Generous limits for 100GB memory
93+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 30.0)
9394

9495
@info "✓ Solved $(prob_name) with $(optimizer)"
9596
vars, time, alg, code = get_stats(sol, optimizer)
@@ -121,14 +122,16 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
121122
push!(retcode, :LOAD_FAILED)
122123
end
123124
finally
124-
# Clean up resources
125+
# Aggressive cleanup to prevent memory accumulation
125126
if nlp_prob !== nothing
126127
try
127128
finalize(nlp_prob)
128129
catch e
129130
@warn "Failed to finalize $(prob_name): $(e)"
130131
end
131132
end
133+
# Force garbage collection after each problem
134+
GC.gc()
132135
end
133136
end
134137

@@ -155,8 +158,25 @@ problems on this section.
155158
eq_bou_problems = CUTEst.select_sif_problems(min_con=1, only_equ_con=true, only_free_var=false)
156159
@info "after1 - testing $(length(eq_bou_problems)) equality-constrained problems"
157160

161+
# Limit to first 50 problems for 100GB memory systems
162+
eq_bou_problems = eq_bou_problems[1:min(50, length(eq_bou_problems))]
163+
@info "Limited to $(length(eq_bou_problems)) problems for comprehensive testing"
164+
158165
# Analysis
159166
eq_bou_results = run_benchmarks(eq_bou_problems, optimizers)
167+
168+
# Calculate and display success rates
169+
successful_codes = [:Success, :MaxIters, :MaxTime, :FirstOrderOptimal]
170+
successful_results = filter(row -> row.retcode in successful_codes, eq_bou_results)
171+
total_attempts = nrow(eq_bou_results)
172+
successful_attempts = nrow(successful_results)
173+
success_rate = total_attempts > 0 ? round(successful_attempts / total_attempts * 100, digits=1) : 0
174+
175+
@info "SUCCESS RATE ANALYSIS:"
176+
@info "Total attempts: $(total_attempts)"
177+
@info "Successful attempts: $(successful_attempts)"
178+
@info "Success rate: $(success_rate)%"
179+
160180
@info "after2"
161181

162182
@df eq_bou_results scatter(:n_vars, :secs,
@@ -175,8 +195,22 @@ Next, we examine the same relationship for inequality-constrained problems.
175195
neq_bou_problems = CUTEst.select_sif_problems(min_con=1, only_ineq_con=true, only_free_var=false)
176196
@info "after5 - testing $(length(neq_bou_problems)) inequality-constrained problems"
177197

198+
# Limit to first 50 problems for 100GB memory systems
199+
neq_bou_problems = neq_bou_problems[1:min(50, length(neq_bou_problems))]
200+
@info "Limited to $(length(neq_bou_problems)) problems for comprehensive testing"
201+
178202
# Analysis
179203
neq_bou_results = run_benchmarks(neq_bou_problems, optimizers)
204+
205+
# Calculate and display success rates
206+
successful_codes = [:Success, :MaxIters, :MaxTime, :FirstOrderOptimal]
207+
successful_results = filter(row -> row.retcode in successful_codes, neq_bou_results)
208+
total_attempts = nrow(neq_bou_results)
209+
successful_attempts = nrow(successful_results)
210+
success_rate = total_attempts > 0 ? round(successful_attempts / total_attempts * 100, digits=1) : 0
211+
212+
@info "INEQUALITY CONSTRAINED SUCCESS RATE: $(success_rate)% ($(successful_attempts)/$(total_attempts))"
213+
180214
@info "after6"
181215

182216
@df neq_bou_results scatter(:n_vars, :secs,

‎benchmarks/OptimizationCUTEst/CUTEst_quadratic.jmd‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ using OptimizationMOI: MOI as MOI
2626
using DataFrames
2727
using Plots
2828
using StatsPlots
29+
using StatsBase: countmap
2930
```
3031

3132
# Benchmarks
@@ -87,8 +88,8 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
8788

8889
for optimizer in optimizers
8990
try
90-
# Set aggressive time and iteration limits
91-
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 5.0)
91+
# Generous limits for 100GB memory
92+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 30.0)
9293

9394
@info "✓ Solved $(prob_name) with $(optimizer)"
9495
vars, time, alg, code = get_stats(sol, optimizer)
@@ -153,6 +154,15 @@ quad_problems = CUTEst.select_sif_problems(objtype="quadratic", contype="linear"
153154
# Analysis
154155
quad_results = run_benchmarks(quad_problems, optimizers)
155156

157+
# Calculate and display success rates for quadratic problems
158+
successful_codes = [:Success, :MaxIters, :MaxTime, :FirstOrderOptimal]
159+
successful_results = filter(row -> row.retcode in successful_codes, quad_results)
160+
total_attempts = nrow(quad_results)
161+
successful_attempts = nrow(successful_results)
162+
success_rate = total_attempts > 0 ? round(successful_attempts / total_attempts * 100, digits=1) : 0
163+
164+
@info "QUADRATIC PROBLEMS SUCCESS RATE: $(success_rate)% ($(successful_attempts)/$(total_attempts))"
165+
156166
@df quad_results scatter(:n_vars, :secs,
157167
group = :solver,
158168
xlabel = "n. variables",

‎benchmarks/OptimizationCUTEst/CUTEst_safe_solvers.jmd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
9999
try
100100
sol = solve(prob, optimizer;
101101
maxiters = 1000,
102-
maxtime = 5.0, # 10 seconds timeout per solve
102+
maxtime = 30.0, # 30 seconds timeout for 100GB system
103103
abstol = 1e-6,
104104
reltol = 1e-6)
105105

‎benchmarks/OptimizationCUTEst/CUTEst_unbounded.jmd‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ using OptimizationMOI: MOI as MOI
2626
using DataFrames
2727
using Plots
2828
using StatsPlots
29+
using StatsBase: countmap
2930
```
3031

3132
# Benchmarks
@@ -87,8 +88,8 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
8788

8889
for optimizer in optimizers
8990
try
90-
# Set aggressive time and iteration limits
91-
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 5.0)
91+
# Generous limits for 100GB memory
92+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 30.0)
9293

9394
@info "✓ Solved $(prob_name) with $(optimizer)"
9495
vars, time, alg, code = get_stats(sol, optimizer)
@@ -158,6 +159,15 @@ eq_unb_problems = CUTEst.select_sif_problems(min_con=1, only_equ_con=true, only_
158159
# Analysis
159160
eq_unb_results = run_benchmarks(eq_unb_problems, optimizers)
160161

162+
# Calculate and display success rates for equality constrained
163+
successful_codes = [:Success, :MaxIters, :MaxTime, :FirstOrderOptimal]
164+
successful_results = filter(row -> row.retcode in successful_codes, eq_unb_results)
165+
total_attempts = nrow(eq_unb_results)
166+
successful_attempts = nrow(successful_results)
167+
success_rate = total_attempts > 0 ? round(successful_attempts / total_attempts * 100, digits=1) : 0
168+
169+
@info "EQUALITY CONSTRAINED SUCCESS RATE: $(success_rate)% ($(successful_attempts)/$(total_attempts))"
170+
161171
@df eq_unb_results scatter(:n_vars, :secs,
162172
group = :solver,
163173
xlabel = "n. variables",
@@ -175,6 +185,15 @@ neq_unb_problems = CUTEst.select_sif_problems(min_con=1, only_ineq_con=true, onl
175185
# Analysis
176186
neq_unb_results = run_benchmarks(neq_unb_problems, optimizers)
177187

188+
# Calculate and display success rates for inequality constrained
189+
successful_codes = [:Success, :MaxIters, :MaxTime, :FirstOrderOptimal]
190+
successful_results = filter(row -> row.retcode in successful_codes, neq_unb_results)
191+
total_attempts = nrow(neq_unb_results)
192+
successful_attempts = nrow(successful_results)
193+
success_rate = total_attempts > 0 ? round(successful_attempts / total_attempts * 100, digits=1) : 0
194+
195+
@info "INEQUALITY CONSTRAINED SUCCESS RATE: $(success_rate)% ($(successful_attempts)/$(total_attempts))"
196+
178197
@df neq_unb_results scatter(:n_vars, :secs,
179198
group = :solver,
180199
xlabel = "n. variables",

‎benchmarks/OptimizationCUTEst/CUTEst_unconstrained.jmd‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ using OptimizationMOI: MOI as MOI
2626
using DataFrames
2727
using Plots
2828
using StatsPlots
29+
using StatsBase: countmap
2930
```
3031

3132
# Benchmarks
@@ -48,7 +49,7 @@ function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
4849
"Ipopt", Symbol(sol.retcode))
4950
end
5051

51-
function run_benchmarks(problems, optimizers; chunk_size=3)
52+
function run_benchmarks(problems, optimizers; chunk_size=1)
5253
problem = String[]
5354
n_vars = Int64[]
5455
secs = Float64[]
@@ -77,7 +78,7 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
7778
try
7879
nlp_prob = CUTEstModel(prob_name)
7980

80-
# Skip extremely large problems to prevent memory issues
81+
# Generous memory limits for 100GB systems - include 5000 var problems
8182
if nlp_prob.meta.nvar > 10000
8283
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
8384
finalize(nlp_prob)
@@ -88,10 +89,10 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
8889

8990
for optimizer in optimizers
9091
try
91-
# Set aggressive time and iteration limits
92-
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 5.0)
92+
# Generous limits for 100GB memory
93+
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 30.0)
9394

94-
@info "✓ Solved $(prob_name) with $(optimizer)"
95+
@info "✓ Solved $(prob_name) with $(optimizer) - Status: $(sol.retcode)"
9596
vars, time, alg, code = get_stats(sol, optimizer)
9697

9798
push!(problem, prob_name)
@@ -101,11 +102,11 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
101102
push!(retcode, code)
102103
catch e
103104
@warn "✗ Failed to solve $(prob_name) with $(optimizer): $(e)"
104-
# Add failure entry
105+
# Still add entry for failed attempts to maintain data consistency
105106
push!(problem, prob_name)
106-
push!(n_vars, -1)
107+
push!(n_vars, nlp_prob !== nothing ? nlp_prob.meta.nvar : -1)
107108
push!(secs, NaN)
108-
push!(solver, string(optimizer))
109+
push!(solver, string(typeof(optimizer)))
109110
push!(retcode, :FAILED)
110111
end
111112
end
@@ -151,9 +152,35 @@ terms of the time to solution with respect to the number of variables.
151152
unc_problems = collect(CUTEst.select_sif_problems(contype="unc"))
152153
@info "Testing $(length(unc_problems)) unconstrained problems"
153154

155+
# Limit to first 50 problems for 100GB memory systems
156+
unc_problems = unc_problems[1:min(50, length(unc_problems))]
157+
@info "Limited to $(length(unc_problems)) problems for comprehensive testing"
158+
154159
# Analysis
155160
unc_results = run_benchmarks(unc_problems, optimizers)
156161

162+
# Calculate and display success rates
163+
successful_codes = [:Success, :MaxIters, :MaxTime, :FirstOrderOptimal]
164+
successful_results = filter(row -> row.retcode in successful_codes, unc_results)
165+
total_attempts = nrow(unc_results)
166+
successful_attempts = nrow(successful_results)
167+
success_rate = total_attempts > 0 ? round(successful_attempts / total_attempts * 100, digits=1) : 0
168+
169+
@info "SUCCESS RATE ANALYSIS:"
170+
@info "Total attempts: $(total_attempts)"
171+
@info "Successful attempts: $(successful_attempts)"
172+
@info "Success rate: $(success_rate)%"
173+
174+
# Show distribution of return codes
175+
@info "Return code distribution:"
176+
if total_attempts > 0
177+
for (code, count) in sort(collect(pairs(countmap(unc_results.retcode))), by=x->x[2], rev=true)
178+
@info " $(code): $(count) occurrences"
179+
end
180+
else
181+
@info " No results to analyze"
182+
end
183+
157184
@df unc_results scatter(:n_vars, :secs,
158185
group = :solver,
159186
xlabel = "n. variables",

‎benchmarks/OptimizationCUTEst/Project.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,5 @@ Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
1111
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
1212
SciMLBenchmarks = "31c91b34-3c75-11e9-0341-95557aab0344"
1313
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
14+
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
1415
StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd"

0 commit comments

Comments
 (0)