Skip to content

Commit ead71b0

Browse files
arnavk23ChrisRackauckas
arnavk23
authored and
ChrisRackauckas
committed
trying again to commit changes to the CUTEst benchmarks in SciMLBenchmarks.jl.
1 parent 0c9522d commit ead71b0

File tree

9 files changed

+766
-264
lines changed

9 files changed

+766
-264
lines changed

‎benchmarks/OptimizationCUTEst/CUTEst_bounded.jmd‎

Lines changed: 21 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -14,34 +14,31 @@ Optimization.jl.
1414

1515
This benchmark uses the following packages:
1616

17-
```julia
17+
18+
```julia; eval = true; @setup
19+
using Pkg; Pkg.activate("."); Pkg.instantiate()
20+
```
21+
22+
```julia; eval = true
1823
using Optimization
1924
using OptimizationNLPModels
2025
using CUTEst
2126
using OptimizationOptimJL
2227
using Ipopt
2328
using OptimizationMOI
2429
using OptimizationMOI: MOI as MOI
25-
# Analysis and plotting
2630
using DataFrames
2731
using Plots
2832
using StatsPlots
2933
using StatsBase: countmap
30-
```
31-
32-
# Benchmarks
33-
34-
We will be testing the [Ipopt]() and the [LBFGS]() optimizers on these classes of
35-
problems.
34+
using OptimizationOptimJL: LBFGS, ConjugateGradient, NelderMead, SimulatedAnnealing, ParticleSwarm
3635

37-
```julia
3836
optimizers = [
39-
("GradientDescent", Optimization.GradientDescent()),
40-
("LBFGS", Optimization.LBFGS()),
41-
("ConjugateGradient", Optimization.ConjugateGradient()),
42-
("NelderMead", Optimization.NelderMead()),
43-
("SimulatedAnnealing", Optimization.SimulatedAnnealing()),
44-
("ParticleSwarm", Optimization.ParticleSwarm()),
37+
("LBFGS", LBFGS()),
38+
("ConjugateGradient", ConjugateGradient()),
39+
("NelderMead", NelderMead()),
40+
("SimulatedAnnealing", SimulatedAnnealing()),
41+
("ParticleSwarm", ParticleSwarm()),
4542
]
4643

4744
function get_stats(sol, optimizer_name)
@@ -59,38 +56,26 @@ function run_benchmarks(problems, optimizers; chunk_size=1)
5956
secs = Float64[]
6057
solver = String[]
6158
retcode = Symbol[]
62-
6359
optz = length(optimizers)
6460
n = length(problems)
65-
6661
@info "Processing $(n) problems with $(optz) optimizers in chunks of $(chunk_size)"
67-
6862
broadcast(c -> sizehint!(c, optz * n), [problem, n_vars, secs, solver, retcode])
69-
70-
# Process problems in chunks to manage memory
7163
for chunk_start in 1:chunk_size:n
7264
chunk_end = min(chunk_start + chunk_size - 1, n)
7365
chunk_problems = problems[chunk_start:chunk_end]
74-
7566
@info "Processing chunk $(div(chunk_start-1, chunk_size)+1)/$(div(n-1, chunk_size)+1): problems $(chunk_start)-$(chunk_end)"
76-
7767
for (idx, prob_name) in enumerate(chunk_problems)
7868
current_problem = chunk_start + idx - 1
7969
@info "Problem $(current_problem)/$(n): $(prob_name)"
80-
8170
nlp_prob = nothing
8271
try
8372
nlp_prob = CUTEstModel(prob_name)
84-
85-
# Generous memory limits for 100GB systems - include 5000 var problems
8673
if nlp_prob.meta.nvar > 10000
8774
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
8875
finalize(nlp_prob)
8976
continue
9077
end
91-
9278
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
93-
9479
for (optimizer_name, optimizer) in optimizers
9580
try
9681
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 30.0)
@@ -102,18 +87,14 @@ function run_benchmarks(problems, optimizers; chunk_size=1)
10287
push!(solver, alg)
10388
push!(retcode, code)
10489
catch e
105-
@warn "✗ Failed to solve $(prob_name) with $(optimizer_name): $(e)"
10690
push!(problem, prob_name)
107-
push!(n_vars, -1)
91+
push!(n_vars, nlp_prob !== nothing ? nlp_prob.meta.nvar : -1)
10892
push!(secs, NaN)
10993
push!(solver, optimizer_name)
11094
push!(retcode, :FAILED)
11195
end
11296
end
113-
11497
catch e
115-
@warn "✗ Failed to load problem $(prob_name): $(e)"
116-
# Add failure entries for all optimizers
11798
for (optimizer_name, optimizer) in optimizers
11899
push!(problem, prob_name)
119100
push!(n_vars, -1)
@@ -122,29 +103,28 @@ function run_benchmarks(problems, optimizers; chunk_size=1)
122103
push!(retcode, :LOAD_FAILED)
123104
end
124105
finally
125-
# Aggressive cleanup to prevent memory accumulation
126106
if nlp_prob !== nothing
127107
try
128108
finalize(nlp_prob)
129109
catch e
130-
@warn "Failed to finalize $(prob_name): $(e)"
131110
end
132111
end
133-
# Force garbage collection after each problem
134-
GC.gc()
135112
end
136113
end
137-
138-
# Force garbage collection after each chunk
139114
GC.gc()
140115
@info "Completed chunk, memory usage cleaned up"
141116
end
142-
143-
return DataFrame(problem = problem, n_vars = n_vars, secs = secs, solver = solver,
144-
retcode = retcode)
117+
return DataFrame(problem = problem, n_vars = n_vars, secs = secs, solver = solver, retcode = retcode)
145118
end
146119
```
147120

121+
# Benchmarks
122+
123+
We will be testing the [Ipopt]() and the [LBFGS]() optimizers on these classes of
124+
problems.
125+
126+
```
127+
148128
## Equality/Inequality constrained problems with bounded variables
149129

150130
Now we analyze the subset of problems with equality/inequality constraints and whose

‎benchmarks/OptimizationCUTEst/CUTEst_quadratic.jmd‎

Lines changed: 20 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -14,34 +14,31 @@ Optimization.jl.
1414

1515
This benchmark uses the following packages:
1616

17-
```julia
17+
18+
```julia; eval = true; @setup
19+
using Pkg; Pkg.activate("."); Pkg.instantiate()
20+
```
21+
22+
```julia; eval = true
1823
using Optimization
1924
using OptimizationNLPModels
2025
using CUTEst
2126
using OptimizationOptimJL
2227
using Ipopt
2328
using OptimizationMOI
2429
using OptimizationMOI: MOI as MOI
25-
# Analysis and plotting
2630
using DataFrames
2731
using Plots
2832
using StatsPlots
2933
using StatsBase: countmap
30-
```
34+
using OptimizationOptimJL: LBFGS, ConjugateGradient, NelderMead, SimulatedAnnealing, ParticleSwarm
3135

32-
# Benchmarks
33-
34-
We will be testing the [Ipopt]() and the [LBFGS]() optimizers on these classes of
35-
problems.
36-
37-
```julia
3836
optimizers = [
39-
("GradientDescent", Optimization.GradientDescent()),
40-
("LBFGS", Optimization.LBFGS()),
41-
("ConjugateGradient", Optimization.ConjugateGradient()),
42-
("NelderMead", Optimization.NelderMead()),
43-
("SimulatedAnnealing", Optimization.SimulatedAnnealing()),
44-
("ParticleSwarm", Optimization.ParticleSwarm()),
37+
("LBFGS", LBFGS()),
38+
("ConjugateGradient", ConjugateGradient()),
39+
("NelderMead", NelderMead()),
40+
("SimulatedAnnealing", SimulatedAnnealing()),
41+
("ParticleSwarm", ParticleSwarm()),
4542
]
4643

4744
function get_stats(sol, optimizer_name)
@@ -59,38 +56,26 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
5956
secs = Float64[]
6057
solver = String[]
6158
retcode = Symbol[]
62-
6359
optz = length(optimizers)
6460
n = length(problems)
65-
6661
@info "Processing $(n) problems with $(optz) optimizers in chunks of $(chunk_size)"
67-
6862
broadcast(c -> sizehint!(c, optz * n), [problem, n_vars, secs, solver, retcode])
69-
70-
# Process problems in chunks to manage memory
7163
for chunk_start in 1:chunk_size:n
7264
chunk_end = min(chunk_start + chunk_size - 1, n)
7365
chunk_problems = problems[chunk_start:chunk_end]
74-
7566
@info "Processing chunk $(div(chunk_start-1, chunk_size)+1)/$(div(n-1, chunk_size)+1): problems $(chunk_start)-$(chunk_end)"
76-
7767
for (idx, prob_name) in enumerate(chunk_problems)
7868
current_problem = chunk_start + idx - 1
7969
@info "Problem $(current_problem)/$(n): $(prob_name)"
80-
8170
nlp_prob = nothing
8271
try
8372
nlp_prob = CUTEstModel(prob_name)
84-
85-
# Skip extremely large problems to prevent memory issues
8673
if nlp_prob.meta.nvar > 10000
8774
@info " Skipping $(prob_name) (too large: $(nlp_prob.meta.nvar) variables)"
8875
finalize(nlp_prob)
8976
continue
9077
end
91-
9278
prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
93-
9479
for (optimizer_name, optimizer) in optimizers
9580
try
9681
sol = solve(prob, optimizer; maxiters = 1000, maxtime = 30.0)
@@ -102,18 +87,14 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
10287
push!(solver, alg)
10388
push!(retcode, code)
10489
catch e
105-
@warn "✗ Failed to solve $(prob_name) with $(optimizer_name): $(e)"
10690
push!(problem, prob_name)
107-
push!(n_vars, -1)
91+
push!(n_vars, nlp_prob !== nothing ? nlp_prob.meta.nvar : -1)
10892
push!(secs, NaN)
10993
push!(solver, optimizer_name)
11094
push!(retcode, :FAILED)
11195
end
11296
end
113-
11497
catch e
115-
@warn "✗ Failed to load problem $(prob_name): $(e)"
116-
# Add failure entries for all optimizers
11798
for (optimizer_name, optimizer) in optimizers
11899
push!(problem, prob_name)
119100
push!(n_vars, -1)
@@ -122,27 +103,27 @@ function run_benchmarks(problems, optimizers; chunk_size=3)
122103
push!(retcode, :LOAD_FAILED)
123104
end
124105
finally
125-
# Clean up resources
126106
if nlp_prob !== nothing
127107
try
128108
finalize(nlp_prob)
129109
catch e
130-
@warn "Failed to finalize $(prob_name): $(e)"
131110
end
132111
end
133112
end
134113
end
135-
136-
# Force garbage collection after each chunk
137114
GC.gc()
138115
@info "Completed chunk, memory usage cleaned up"
139116
end
140-
141-
return DataFrame(problem = problem, n_vars = n_vars, secs = secs, solver = solver,
142-
retcode = retcode)
117+
return DataFrame(problem = problem, n_vars = n_vars, secs = secs, solver = solver, retcode = retcode)
143118
end
144119
```
145120

121+
# Benchmarks
122+
123+
We will be testing the [Ipopt]() and the [LBFGS]() optimizers on these classes of
124+
problems.
125+
126+
146127
# Quadratic programs with linear constraints
147128

148129
Lastly, we examine the problems with a quadratic objective function and only linear

0 commit comments

Comments
 (0)