From a93505ff70e6e9cf164a3357bc9e9c6709ef5c67 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 25 Dec 2024 11:14:32 +0100 Subject: [PATCH 01/48] m-monotone, non-allocating R2N --- src/R2N_alg.jl | 429 +++++++++++++++++++++++++++++++++ src/RegularizedOptimization.jl | 1 + 2 files changed, 430 insertions(+) create mode 100644 src/R2N_alg.jl diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl new file mode 100644 index 00000000..24c60748 --- /dev/null +++ b/src/R2N_alg.jl @@ -0,0 +1,429 @@ +export R2N, R2NSolver, solve! + +import SolverCore.solve! + +mutable struct R2NSolver{ + T <: Real, + G <: ShiftedProximableFunction, + V <: AbstractVector{T}, + S <: AbstractOptimizationSolver, +} <: AbstractOptimizationSolver + xk::V + ∇fk::V + ∇fk⁻::V + mν∇fk::V + ψ::G + sub_ψ::G + xkn::V + s::V + s1::V + has_bnds::Bool + l_bound::V + u_bound::V + m_fh_hist::V + subsolver::S + substats::GenericExecutionStats{T, V, V, Any} +end + +function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Solver, m_monotone::Int = 1) where {T, V} + x0 = reg_nlp.model.meta.x0 + l_bound = reg_nlp.model.meta.lvar + u_bound = reg_nlp.model.meta.uvar + + xk = similar(x0) + ∇fk = similar(x0) + ∇fk⁻ = similar(x0) + mν∇fk = similar(x0) + xkn = similar(x0) + s = zero(x0) + s1 = similar(x0) + has_bnds = any(l_bound .!= T(-Inf)) || any(u_bound .!= T(Inf)) + m_fh_hist = fill(T(-Inf), m_monotone - 1) + + ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) + sub_ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) + + sub_nlp = RegularizedNLPModel(reg_nlp.model, sub_ψ) + substats = GenericExecutionStats(reg_nlp.model) + + return R2NSolver( + xk, + ∇fk, + ∇fk⁻, + mν∇fk, + ψ, + sub_ψ, + xkn, + s, + s1, + has_bnds, + l_bound, + u_bound, + m_fh_hist, + subsolver(sub_nlp), + substats + ) +end + +function R2N( + nlp::AbstractNLPModel{R, V}, + h, + options::ROSolverOptions{R}; + kwargs..., +) where {R <: Real, V} + kwargs_dict = Dict(kwargs...) + selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) + x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) + reg_nlp = RegularizedNLPModel(nlp, h, selected) + return R2N( + reg_nlp, + x = x0, + atol = options.ϵa, + rtol = options.ϵr, + neg_tol = options.neg_tol, + verbose = options.verbose, + max_iter = options.maxIter, + max_time = options.maxTime, + σmin = options.σmin, + η1 = options.η1, + η2 = options.η2, + ν = options.ν, + γ = options.γ; + kwargs_dict..., + ) +end + +function R2N(reg_nlp::AbstractRegularizedNLPModel; kwargs...) + kwargs_dict = Dict(kwargs...) + m_monotone = pop!(kwargs_dict, :m_monotone, 1) + subsolver = pop!(kwargs_dict, :subsolver, R2Solver) + solver = R2NSolver(reg_nlp, subsolver = subsolver, m_monotone = m_monotone) + stats = GenericExecutionStats(reg_nlp.model) + solve!(solver, reg_nlp, stats; kwargs_dict...) + return stats +end + + +function SolverCore.solve!( + solver::R2NSolver{T}, + reg_nlp::AbstractRegularizedNLPModel{T, V}, + stats::GenericExecutionStats{T, V}; + callback = (args...) -> nothing, + x::V = reg_nlp.model.meta.x0, + atol::T = √eps(T), + sub_atol::T = atol, + rtol::T = √eps(T), + neg_tol::T = eps(T)^(1 / 4), + verbose::Int = 0, + max_iter::Int = 10000, + max_time::Float64 = 30.0, + max_eval::Int = -1, + σmin::T = eps(T), + η1::T = √√eps(T), + η2::T = T(0.9), + ν = eps(T)^(1/5), + γ::T = T(3), + β = 1/eps(T), + θ = eps(T)^(1/5), + kwargs... +) where {T, V} + + reset!(stats) + + # Retrieve workspace + selected = reg_nlp.selected + h = reg_nlp.h + nlp = reg_nlp.model + + xk = solver.xk .= x + + # Make sure ψ has the correct shift + shift!(solver.ψ, xk) + + σk = 1/ν + ∇fk = solver.∇fk + ∇fk⁻ = solver.∇fk⁻ + mν∇fk = solver.mν∇fk + ψ = solver.ψ + xkn = solver.xkn + s = solver.s + s1 = solver.s1 + has_bnds = solver.has_bnds + if has_bnds + l_bound = solver.l_bound + u_bound = solver.u_bound + end + m_fh_hist = solver.m_fh_hist + m_monotone = length(m_fh_hist) + 1 + + subsolver = solver.subsolver + substats = solver.substats + + # initialize parameters + improper = false + hk = @views h(xk[selected]) + if hk == Inf + verbose > 0 && @info "R2N: finding initial guess where nonsmooth term is finite" + prox!(xk, h, xk, one(eltype(x0))) + hk = @views h(xk[selected]) + hk < Inf || error("prox computation must be erroneous") + verbose > 0 && @debug "R2N: found point where h has value" hk + end + improper = (hk == -Inf) + + if verbose > 0 + @info log_header( + [:outer, :inner, :fx, :hx, :xi, :ρ, :σ, :normx, :norms, :normB, :arrow], + [Int, Int, Float64, Float64, Float64, Float64, Float64, Float64, Float64, Float64, Char], + hdr_override = Dict{Symbol, String}( # TODO: Add this as constant dict elsewhere + :outer => "outer", + :inner => "inner", + :fx => "f(x)", + :hx => "h(x)", + :xi => "√(ξ1/ν)", + :ρ => "ρ", + :σ => "σ", + :normx => "‖x‖", + :norms => "‖s‖", + :normB => "‖B‖", + :arrow => "R2N", + ), + colsep = 1, + ) + end + + local ξ1::T + local ρk::T + + fk = obj(nlp, xk) + grad!(nlp, xk, ∇fk) + ∇fk⁻ .= ∇fk + + quasiNewtTest = isa(nlp, QuasiNewtonModel) + Bk = hess_op(nlp, xk) + local λmax::T + try + λmax = opnorm(Bk) + catch LAPACKException + λmax = opnorm(Matrix(Bk)) + end + + νInv = (1 + θ) *( σk + λmax) + sqrt_ξ1_νInv = one(T) + ν_subsolver = 1/νInv + + @. mν∇fk = -ν_subsolver * ∇fk + m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) + + set_iter!(stats, 0) + start_time = time() + set_time!(stats, 0.0) + set_objective!(stats, fk + hk) + set_solver_specific!(stats, :smooth_obj, fk) + set_solver_specific!(stats, :nonsmooth_obj, hk) + + # model for first prox-gradient step and ξ1 + φ1(d) = ∇fk' * d + mk1(d) = φ1(d) + ψ(d) + + # model for subsequent prox-gradient steps and ξ + φ(d) = (d' * (Bk * d)) / 2 + ∇fk' * d + σk * dot(d, d) / 2 + + ∇φ!(g, d) = begin + mul!(g, Bk, d) + g .+= ∇fk + g .+= σk * d + g + end + + mk(d) = φ(d) + ψ(d) + prox!(s, ψ, mν∇fk, ν_subsolver) + mks = mk1(s) + + ξ1 = hk - mks + max(1, abs(hk)) * 10 * eps() + sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 * νInv) : sqrt(-ξ1 * νInv) + solved = (ξ1 < 0 && sqrt_ξ1_νInv ≤ neg_tol) || (ξ1 ≥ 0 && sqrt_ξ1_νInv ≤ atol) + (ξ1 < 0 && sqrt_ξ1_νInv > neg_tol) && + error("R2N: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") + atol += rtol * sqrt_ξ1_νInv # make stopping test absolute and relative + sub_atol += rtol * sqrt_ξ1_νInv + + set_solver_specific!(stats, :xi, sqrt_ξ1_νInv) + set_status!( + stats, + get_status( + reg_nlp, + elapsed_time = stats.elapsed_time, + iter = stats.iter, + optimal = solved, + improper = improper, + max_eval = max_eval, + max_time = max_time, + max_iter = max_iter, + ), + ) + + callback(nlp, solver, stats) + + done = stats.status != :unknown + + while !done + + s1 .= s + + sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5) , sqrt_ξ1_νInv * 1e-3) # 1.0e-5 default + #@debug "setting inner stopping tolerance to" subsolver_options.optTol + #subsolver_args = subsolver == R2DH ? (SpectralGradient(1., f.meta.nvar),) : () + nlp_model = FirstOrderModel(φ,∇φ!,s) + model = RegularizedNLPModel(nlp_model, ψ) + #model.selected .= reg_nlp.selected + solve!( + subsolver, + model, + substats, + x = s, + atol = sub_atol, + ν = ν_subsolver, + kwargs...) + + s .= substats.solution + + if norm(s) > β * norm(s1) + s .= s1 + end + """ + if mk(s) > mk(s1) + s .= s1 + end + """ + xkn .= xk .+ s + fkn = obj(nlp, xkn) + hkn = h(xkn[selected]) + hkn == -Inf && error("nonsmooth term is not proper") + mks = mk(s) + + fhmax = m_monotone > 1 ? maximum(m_fh_hist) : fk + hk + Δobj = fhmax - (fkn + hkn) + max(1, abs(fk + hk)) * 10 * eps() + Δmod = fhmax - (fk + mks) + max(1, abs(fhmax)) * 10 * eps() + ξ = hk - mks + max(1, abs(hk)) * 10 * eps() + + if (ξ ≤ 0 || isnan(ξ)) + error("R2N: failed to compute a step: ξ = $ξ") + end + + ρk = Δobj / Δmod + + verbose > 0 && + stats.iter % verbose == 0 && + @info log_row( + Any[ + stats.iter, + substats.iter, + fk, + hk, + sqrt_ξ1_νInv, + ρk, + σk, + norm(xk), + norm(s), + λmax, + (η2 ≤ ρk < Inf) ? "↗" : (ρk < η1 ? "↘" : "="), + ], + colsep = 1, + ) + + if η2 ≤ ρk < Inf + σk = max(σk/γ, σmin) + end + + if η1 ≤ ρk < Inf + xk .= xkn + has_bounds(nlp) && set_bounds!(ψ, l_bound - xk, u_bound - xk) + + #update functions + fk = fkn + hk = hkn + + shift!(ψ, xk) + ∇fk = grad!(nlp, xk, ∇fk) + + if quasiNewtTest + push!(nlp, s, ∇fk - ∇fk⁻) + end + Bk = hess_op(nlp, xk) + try + λmax = opnorm(Bk) + catch LAPACKException + λmax = opnorm(Matrix(Bk)) + end + ∇fk⁻ .= ∇fk + end + + if ρk < η1 || ρk == Inf + σk = σk * γ + end + + νInv = (1 + θ) *( σk + λmax) + m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) + + set_objective!(stats, fk + hk) + set_solver_specific!(stats, :smooth_obj, fk) + set_solver_specific!(stats, :nonsmooth_obj, hk) + set_iter!(stats, stats.iter + 1) + set_time!(stats, time() - start_time) + + ν_subsolver = 1/νInv + @. mν∇fk = - ν_subsolver * ∇fk + prox!(s, ψ, mν∇fk, ν_subsolver) + mks = mk1(s) + + ξ1 = hk - mks + max(1, abs(hk)) * 10 * eps() + + sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 * νInv) : sqrt(-ξ1 * νInv) + solved = (ξ1 < 0 && sqrt_ξ1_νInv ≤ neg_tol) || (ξ1 ≥ 0 && sqrt_ξ1_νInv ≤ atol) + + (ξ1 < 0 && sqrt_ξ1_νInv > neg_tol) && + error("R2N: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") + set_solver_specific!(stats, :xi, sqrt_ξ1_νInv) + set_status!( + stats, + get_status( + reg_nlp, + elapsed_time = stats.elapsed_time, + iter = stats.iter, + optimal = solved, + improper = improper, + max_eval = max_eval, + max_time = max_time, + max_iter = max_iter, + ), + ) + + callback(nlp, solver, stats) + + done = stats.status != :unknown + end + + if verbose > 0 && stats.status == :first_order + @info log_row( + Any[ + stats.iter, + 0, + fk, + hk, + sqrt_ξ1_νInv, + ρk, + σk, + norm(xk), + norm(s), + λmax, + (η2 ≤ ρk < Inf) ? "↘" : (ρk < η1 ? "↗" : "="), + ], + colsep = 1, + ) + @info "R2N: terminating with √(ξ1/ν) = $(sqrt_ξ1_νInv)" + end + + set_solution!(stats,xk) + return stats +end \ No newline at end of file diff --git a/src/RegularizedOptimization.jl b/src/RegularizedOptimization.jl index 53365e55..5bd44e21 100644 --- a/src/RegularizedOptimization.jl +++ b/src/RegularizedOptimization.jl @@ -19,6 +19,7 @@ include("splitting.jl") include("TR_alg.jl") include("TRDH_alg.jl") include("R2_alg.jl") +include("R2N_alg.jl") include("LM_alg.jl") include("LMTR_alg.jl") include("R2DH.jl") From 9b852ef76dcebf557dd9b431959fd65810b9c293 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sat, 28 Dec 2024 11:50:30 +0100 Subject: [PATCH 02/48] m-monotone, non-allocating R2DH --- src/R2DH_alg.jl | 413 +++++++++++++++++++++++++++++++++ src/RegularizedOptimization.jl | 1 + 2 files changed, 414 insertions(+) create mode 100644 src/R2DH_alg.jl diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl new file mode 100644 index 00000000..0bf2ebe5 --- /dev/null +++ b/src/R2DH_alg.jl @@ -0,0 +1,413 @@ +export R2DH, R2DH_mine, R2DHSolver, solve! + +mutable struct R2DHSolver{ + T <: Real, + G <: Union{ShiftedProximableFunction, Nothing}, + V <: AbstractVector{T}, +} <: AbstractOptimizationSolver + xk::V + ∇fk::V + ∇fk⁻::V + mν∇fk::V + ψ::G + xkn::V + s::V + dkσk::V + has_bnds::Bool + l_bound::V + u_bound::V + l_bound_m_x::V + u_bound_m_x::V + m_fh_hist::V +end + +function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; Dσ::T = T(1), m_monotone::Int = 1) where{T, V} + x0 = reg_nlp.model.meta.x0 + l_bound = reg_nlp.model.meta.lvar + u_bound = reg_nlp.model.meta.uvar + + xk = similar(x0) + ∇fk = similar(x0) + ∇fk⁻ = similar(x0) + mν∇fk = similar(x0) + xkn = similar(x0) + s = similar(x0) + dkσk = similar(x0) + has_bnds = any(l_bound .!= T(-Inf)) || any(u_bound .!= T(Inf)) + if has_bnds + l_bound_m_x = similar(xk) + u_bound_m_x = similar(xk) + @. l_bound_m_x = l_bound - x0 + @. u_bound_m_x = u_bound - x0 + else + l_bound_m_x = similar(xk, 0) + u_bound_m_x = similar(xk, 0) + end + m_fh_hist = fill(T(-Inf), m_monotone - 1) + + ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) + + return R2DHSolver( + xk, + ∇fk, + ∇fk⁻, + mν∇fk, + ψ, + xkn, + s, + dkσk, + has_bnds, + l_bound, + u_bound, + l_bound_m_x, + u_bound_m_x, + m_fh_hist + ) +end + + +""" + R2DH(nlp, h, options) + R2DH(f, ∇f!, h, options, x0) + +A second-order quadratic regularization method for the problem + + min f(x) + h(x) + +where f: ℝⁿ → ℝ is C¹ and h: ℝⁿ → ℝ is lower semi-continuous, proper, and prox-bounded. + +About each iterate xₖ, a step sₖ is computed as a solution of + + min φ(s; xₖ) + ψ(s; xₖ) + +where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀ(Dₖ + σₖI)s is a quadratic approximation of f about xₖ, +ψ(s; xₖ) = h(xₖ + s), Dₖ is a diagonal Hessian approximation and σₖ > 0 is the regularization parameter. + +### Arguments + +* `nlp::AbstractDiagonalQNModel`: a smooth optimization problem +* `h`: a regularizer such as those defined in ProximalOperators +* `options::ROSolverOptions`: a structure containing algorithmic parameters +* `x0::AbstractVector`: an initial guess (in the second calling form) + +### Keyword Arguments + +* `x0::AbstractVector`: an initial guess (in the first calling form: default = `nlp.meta.x0`) +* `selected::AbstractVector{<:Integer}`: subset of variables to which `h` is applied (default `1:length(x0)`). +* `D`: Diagonal quasi-Newton operator. +* `Mmonotone::Int`: number of previous values of the objective to consider for the non-monotone variant (default: 6). + +The objective and gradient of `nlp` will be accessed. + +In the second form, instead of `nlp`, the user may pass in + +* `f` a function such that `f(x)` returns the value of f at x +* `∇f!` a function to evaluate the gradient in place, i.e., such that `∇f!(g, x)` store ∇f(x) in `g` + +### Return values + +* `xk`: the final iterate +* `Fobj_hist`: an array with the history of values of the smooth objective +* `Hobj_hist`: an array with the history of values of the nonsmooth objective +* `Complex_hist`: an array with the history of number of inner iterations. +""" + +function R2DH( + nlp::AbstractDiagonalQNModel{T, V}, + h, + options::ROSolverOptions{T}; + kwargs..., +) where{T, V} + kwargs_dict = Dict(kwargs...) + selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) + x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) + reg_nlp = RegularizedNLPModel(nlp, h, selected) + return R2DH( + reg_nlp, + x = x0, + atol = options.ϵa, + rtol = options.ϵr, + neg_tol = options.neg_tol, + verbose = options.verbose, + max_iter = options.maxIter, + max_time = options.maxTime, + σmin = options.σmin, + η1 = options.η1, + η2 = options.η2, + ν = options.ν, + γ = options.γ, + θ = options.θ, + kwargs_dict..., + ) +end + +function R2DH( + reg_nlp::AbstractRegularizedNLPModel{T, V}; + kwargs... +) where{T, V} + kwargs_dict = Dict(kwargs...) + m_monotone = pop!(kwargs_dict, :m_monotone, 1) + solver = R2DHSolver(reg_nlp, m_monotone = m_monotone) + stats = GenericExecutionStats(reg_nlp.model) + solve!(solver, reg_nlp, stats; kwargs_dict...) + return stats +end + +function solve!( + solver::R2DHSolver{T}, + reg_nlp::AbstractRegularizedNLPModel{T, V}, + stats::GenericExecutionStats{T, V}; + callback = (args...) -> nothing, + x::V = reg_nlp.model.meta.x0, + D::AbstractDiagonalQuasiNewtonOperator{T} = isa(reg_nlp.model, AbstractDiagonalQNModel) ? hess_op(reg_nlp.model, x) : SpectralGradient(T(1), reg_nlp.model.meta.nvar), + atol::T = √eps(T), + rtol::T = √eps(T), + neg_tol::T = eps(T)^(1 / 4), + verbose::Int = 0, + max_iter::Int = 10000, + max_time::Float64 = 30.0, + max_eval::Int = -1, + σmin::T = eps(T), + η1::T = √√eps(T), + η2::T = T(0.9), + ν::T = eps(T)^(1 / 5), + γ::T = T(3), + θ::T = eps(T)^(1/5), +) where{T, V} + reset!(stats) + + # Retrieve workspace + selected = reg_nlp.selected + h = reg_nlp.h + nlp = reg_nlp.model + + xk = solver.xk .= x + + # Make sure ψ has the correct shift + shift!(solver.ψ, xk) + + ∇fk = solver.∇fk + ∇fk⁻ = solver.∇fk⁻ + mν∇fk = solver.mν∇fk + dkσk = solver.dkσk + ψ = solver.ψ + xkn = solver.xkn + s = solver.s + m_fh_hist = solver.m_fh_hist + has_bnds = solver.has_bnds + + if has_bnds + l_bound = solver.l_bound + u_bound = solver.u_bound + end + m_fh_hist = solver.m_fh_hist + m_monotone = length(m_fh_hist) + 1 + + + # initialize parameters + improper = false + hk = @views h(xk[selected]) + if hk == Inf + verbose > 0 && @info "R2: finding initial guess where nonsmooth term is finite" + prox!(xk, h, xk, one(eltype(x0))) + hk = @views h(xk[selected]) + hk < Inf || error("prox computation must be erroneous") + verbose > 0 && @debug "R2: found point where h has value" hk + end + improper = (hk == -Inf) + + if verbose > 0 + @info log_header( + [:iter, :fx, :hx, :xi, :ρ, :σ, :normx, :norms, :arrow], + [Int, Float64, Float64, Float64, Float64, Float64, Float64, Float64, Char], + hdr_override = Dict{Symbol, String}( # TODO: Add this as constant dict elsewhere + :iter => "iter", + :fx => "f(x)", + :hx => "h(x)", + :xi => "√(ξ/ν)", + :ρ => "ρ", + :σ => "σ", + :normx => "‖x‖", + :norms => "‖s‖", + :arrow => " ", + ), + colsep = 1, + ) + end + + local ξ::T + local ρk::T + + σk = max(1 / ν, σmin) + + fk = obj(nlp, xk) + grad!(nlp, xk, ∇fk) + ∇fk⁻ .= ∇fk + spectral_test = isa(D, SpectralGradient) + + @. dkσk = D.d .+ σk + DNorm = norm(D.d, Inf) + + ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + sqrt_ξ_νInv = one(T) + + @. mν∇fk = -ν₁ * ∇fk + m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) + + set_iter!(stats, 0) + start_time = time() + set_time!(stats, 0.0) + set_objective!(stats, fk + hk) + set_solver_specific!(stats, :smooth_obj, fk) + set_solver_specific!(stats, :nonsmooth_obj, hk) + + φ(d) = ∇fk' * d + (d' * (dkσk .* d)) / 2 + mk(d) = φ(d) + ψ(d) + + spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) + + mks = mk(s) #TODO check whether it is -Inf, perhaps add a while loop until we find an acceptable σ. + + ξ = hk - mks + max(1, abs(hk)) * 10 * eps() + sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) + solved = (ξ < 0 && sqrt_ξ_νInv ≤ neg_tol) || (ξ ≥ 0 && sqrt_ξ_νInv ≤ atol) + (ξ < 0 && sqrt_ξ_νInv > neg_tol) && + error("R2DH: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") + atol += rtol * sqrt_ξ_νInv # make stopping test absolute and relative + + set_solver_specific!(stats, :xi, sqrt_ξ_νInv) + set_status!( + stats, + get_status( + reg_nlp, + elapsed_time = stats.elapsed_time, + iter = stats.iter, + optimal = solved, + improper = improper, + max_eval = max_eval, + max_time = max_time, + max_iter = max_iter, + ), + ) + + callback(nlp, solver, stats) + + done = stats.status != :unknown + + while !done + # Update xk, sigma_k + xkn .= xk .+ s + fkn = obj(nlp, xkn) + hkn = @views h(xkn[selected]) + improper = (hkn == -Inf) + + fhmax = m_monotone > 1 ? maximum(m_fh_hist) : fk + hk + Δobj = fhmax - (fkn + hkn) + max(1, abs(fhmax)) * 10 * eps() + Δmod = fhmax - (fk + mks) + max(1, abs(hk)) * 10 * eps() + + ρk = Δobj / Δmod + + verbose > 0 && + stats.iter % verbose == 0 && + @info log_row( + Any[ + stats.iter, + fk, + hk, + sqrt_ξ_νInv, + ρk, + σk, + norm(xk), + norm(s), + (η2 ≤ ρk < Inf) ? "↘" : (ρk < η1 ? "↗" : "="), + ], + colsep = 1, + ) + if η2 ≤ ρk < Inf + σk = max(σk / γ, σmin) + end + + if η1 ≤ ρk < Inf + xk .= xkn + if has_bnds + @. l_bound_m_x = l_bound - xk + @. u_bound_m_x = u_bound - xk + set_bounds!(ψ, l_bound_m_x, u_bound_m_x) + end + fk = fkn + hk = hkn + shift!(ψ, xk) + grad!(nlp, xk, ∇fk) + push!(D, s, ∇fk - ∇fk⁻) # update QN operator + ∇fk⁻ .= ∇fk + end + + if ρk < η1 || ρk == Inf + σk = σk * γ + end + + set_objective!(stats, fk + hk) + set_solver_specific!(stats, :smooth_obj, fk) + set_solver_specific!(stats, :nonsmooth_obj, hk) + set_iter!(stats, stats.iter + 1) + set_time!(stats, time() - start_time) + + @. dkσk = D.d .+ σk + DNorm = norm(D.d, Inf) + + ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + + @. mν∇fk = -ν₁ * ∇fk + m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) + + spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) + mks = mk(s) #TODO check whether it is -Inf, perhaps add a while loop until we find an acceptable σ. + + ξ = hk - mks + max(1, abs(hk)) * 10 * eps() + sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) + solved = (ξ < 0 && sqrt_ξ_νInv ≤ neg_tol) || (ξ ≥ 0 && sqrt_ξ_νInv ≤ atol) + (ξ < 0 && sqrt_ξ_νInv > neg_tol) && + error("R2DH: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") + + set_solver_specific!(stats, :xi, sqrt_ξ_νInv) + set_status!( + stats, + get_status( + reg_nlp, + elapsed_time = stats.elapsed_time, + iter = stats.iter, + optimal = solved, + improper = improper, + max_eval = max_eval, + max_time = max_time, + max_iter = max_iter, + ), + ) + + callback(nlp, solver, stats) + + done = stats.status != :unknown + end + + if verbose > 0 && stats.status == :first_order + @info log_row( + Any[ + stats.iter, + 0, + fk, + hk, + sqrt_ξ_νInv, + ρk, + σk, + norm(xk), + norm(s), + (η2 ≤ ρk < Inf) ? "↘" : (ρk < η1 ? "↗" : "="), + ], + colsep = 1, + ) + @info "R2DH: terminating with √(ξ/ν) = $(sqrt_ξ_νInv)" + end + + set_solution!(stats,xk) + return stats +end diff --git a/src/RegularizedOptimization.jl b/src/RegularizedOptimization.jl index 5bd44e21..3d9df94f 100644 --- a/src/RegularizedOptimization.jl +++ b/src/RegularizedOptimization.jl @@ -19,6 +19,7 @@ include("splitting.jl") include("TR_alg.jl") include("TRDH_alg.jl") include("R2_alg.jl") +include("R2DH_alg.jl") include("R2N_alg.jl") include("LM_alg.jl") include("LMTR_alg.jl") From 8b53b236378c5b05a752b40a211ebe24e4468c5f Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sat, 28 Dec 2024 11:50:53 +0100 Subject: [PATCH 03/48] minor fixes in R2N --- src/R2N_alg.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 24c60748..32cf4e42 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -66,11 +66,11 @@ function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Sol end function R2N( - nlp::AbstractNLPModel{R, V}, + nlp::AbstractNLPModel{T, V}, h, - options::ROSolverOptions{R}; + options::ROSolverOptions{T}; kwargs..., -) where {R <: Real, V} +) where {T <: Real, V} kwargs_dict = Dict(kwargs...) selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) @@ -88,7 +88,9 @@ function R2N( η1 = options.η1, η2 = options.η2, ν = options.ν, - γ = options.γ; + γ = options.γ, + θ = options.γ, + β = options.β; kwargs_dict..., ) end @@ -123,9 +125,8 @@ function SolverCore.solve!( η2::T = T(0.9), ν = eps(T)^(1/5), γ::T = T(3), - β = 1/eps(T), - θ = eps(T)^(1/5), - kwargs... + β::T = 1/eps(T), + θ::T = eps(T)^(1/5), ) where {T, V} reset!(stats) From df013cfbd65704dd5133702913753c530ff5f7e8 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sat, 28 Dec 2024 12:28:01 +0100 Subject: [PATCH 04/48] add tests for R2N,R2DH --- src/R2DH_alg.jl | 2 +- src/R2N_alg.jl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index 0bf2ebe5..109821ef 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -236,7 +236,7 @@ function solve!( end local ξ::T - local ρk::T + local ρk::T = zero(T) σk = max(1 / ν, σmin) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 32cf4e42..76a0542a 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -127,6 +127,7 @@ function SolverCore.solve!( γ::T = T(3), β::T = 1/eps(T), θ::T = eps(T)^(1/5), + kwargs... ) where {T, V} reset!(stats) @@ -195,6 +196,7 @@ function SolverCore.solve!( local ξ1::T local ρk::T + ρk = T(1) fk = obj(nlp, xk) grad!(nlp, xk, ∇fk) @@ -284,7 +286,7 @@ function SolverCore.solve!( substats, x = s, atol = sub_atol, - ν = ν_subsolver, + ν = ν_subsolver; kwargs...) s .= substats.solution From 7f579477eb348153e39af41365fdd690bca3295d Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sat, 28 Dec 2024 14:16:16 +0100 Subject: [PATCH 05/48] add test for whether the model gives -Inf in R2DH + Todos --- src/R2DH_alg.jl | 23 ++++++++++++++++++++--- src/R2N_alg.jl | 4 ++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index 109821ef..bfcebb7a 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -261,12 +261,20 @@ function solve!( set_solver_specific!(stats, :smooth_obj, fk) set_solver_specific!(stats, :nonsmooth_obj, hk) - φ(d) = ∇fk' * d + (d' * (dkσk .* d)) / 2 + φ(d) = ∇fk' * d + (d' * (dkσk .* d)) / 2 #TODO This probably allocated a little mk(d) = φ(d) + ψ(d) spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) - mks = mk(s) #TODO check whether it is -Inf, perhaps add a while loop until we find an acceptable σ. + mks = mk(s) + while mks == -Inf #TODO add test coverage for this + σk = σk * γ + dkσk .= D.d .+ σk + DNorm = norm(D.d, Inf) + ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + @. mν∇fk = -ν₁ * ∇fk + spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) + end ξ = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) @@ -361,7 +369,16 @@ function solve!( m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) - mks = mk(s) #TODO check whether it is -Inf, perhaps add a while loop until we find an acceptable σ. + mks = mk(s) + + while mks == -Inf #TODO add test coverage for this + σk = σk * γ + dkσk .= D.d .+ σk + DNorm = norm(D.d, Inf) + ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + @. mν∇fk = -ν₁ * ∇fk + spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) + end ξ = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 76a0542a..8742a2be 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -207,7 +207,7 @@ function SolverCore.solve!( local λmax::T try λmax = opnorm(Bk) - catch LAPACKException + catch LAPACKException # This is really bad and should be removed ASAP; see PR #159. λmax = opnorm(Matrix(Bk)) end @@ -230,7 +230,7 @@ function SolverCore.solve!( mk1(d) = φ1(d) + ψ(d) # model for subsequent prox-gradient steps and ξ - φ(d) = (d' * (Bk * d)) / 2 + ∇fk' * d + σk * dot(d, d) / 2 + φ(d) = (d' * (Bk * d)) / 2 + ∇fk' * d + σk * dot(d, d) / 2 # TODO this probably allocates a little. ∇φ!(g, d) = begin mul!(g, Bk, d) From 792fd7fb9015b4f5bf0b5a902ed99f6b91e2c71a Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 30 Dec 2024 11:22:01 +0100 Subject: [PATCH 06/48] allow for keyword args in wrappedallocs macro --- test/test_allocs.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/test_allocs.jl b/test/test_allocs.jl index 3eb218eb..19da5ad5 100644 --- a/test/test_allocs.jl +++ b/test/test_allocs.jl @@ -26,12 +26,16 @@ allocate: ``` """ macro wrappedallocs(expr) - argnames = [gensym() for a in expr.args] + kwargs = [a for a in expr.args if isa(a, Expr)] + args = [a for a in expr.args if isa(a, Symbol)] + + argnames = [gensym() for a in args] + kwargs_dict = Dict{Symbol, Any}(a.args[1] => a.args[2] for a in kwargs if a.head == :kw) quote - function g($(argnames...)) - @allocated $(Expr(expr.head, argnames...)) + function g($(argnames...); kwargs_dict...) + @allocated $(Expr(expr.head, argnames..., kwargs...)) end - $(Expr(:call, :g, [esc(a) for a in expr.args]...)) + $(Expr(:call, :g, [esc(a) for a in args]...)) end end From d2b8e2c8782acebe21f8c23914b249aa4c5d8b14 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 31 Dec 2024 13:26:06 +0100 Subject: [PATCH 07/48] remove allocations in R2DH --- src/R2DH_alg.jl | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index bfcebb7a..bdab81d7 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -4,11 +4,13 @@ mutable struct R2DHSolver{ T <: Real, G <: Union{ShiftedProximableFunction, Nothing}, V <: AbstractVector{T}, + QN <: AbstractDiagonalQuasiNewtonOperator{T} } <: AbstractOptimizationSolver xk::V ∇fk::V ∇fk⁻::V mν∇fk::V + D::QN ψ::G xkn::V s::V @@ -21,7 +23,7 @@ mutable struct R2DHSolver{ m_fh_hist::V end -function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; Dσ::T = T(1), m_monotone::Int = 1) where{T, V} +function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int = 1) where{T, V} x0 = reg_nlp.model.meta.x0 l_bound = reg_nlp.model.meta.lvar u_bound = reg_nlp.model.meta.uvar @@ -46,12 +48,14 @@ function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; Dσ::T = T(1), m m_fh_hist = fill(T(-Inf), m_monotone - 1) ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) + D = isa(reg_nlp.model, AbstractDiagonalQNModel) ? hess_op(reg_nlp.model, x0) : SpectralGradient(T(1), reg_nlp.model.meta.nvar) return R2DHSolver( xk, ∇fk, ∇fk⁻, mν∇fk, + D, ψ, xkn, s, @@ -159,7 +163,6 @@ function solve!( stats::GenericExecutionStats{T, V}; callback = (args...) -> nothing, x::V = reg_nlp.model.meta.x0, - D::AbstractDiagonalQuasiNewtonOperator{T} = isa(reg_nlp.model, AbstractDiagonalQNModel) ? hess_op(reg_nlp.model, x) : SpectralGradient(T(1), reg_nlp.model.meta.nvar), atol::T = √eps(T), rtol::T = √eps(T), neg_tol::T = eps(T)^(1 / 4), @@ -189,6 +192,7 @@ function solve!( ∇fk = solver.∇fk ∇fk⁻ = solver.∇fk⁻ mν∇fk = solver.mν∇fk + D = solver.D dkσk = solver.dkσk ψ = solver.ψ xkn = solver.xkn @@ -200,7 +204,6 @@ function solve!( l_bound = solver.l_bound u_bound = solver.u_bound end - m_fh_hist = solver.m_fh_hist m_monotone = length(m_fh_hist) + 1 @@ -208,11 +211,11 @@ function solve!( improper = false hk = @views h(xk[selected]) if hk == Inf - verbose > 0 && @info "R2: finding initial guess where nonsmooth term is finite" + verbose > 0 && @info "R2DH: finding initial guess where nonsmooth term is finite" prox!(xk, h, xk, one(eltype(x0))) hk = @views h(xk[selected]) hk < Inf || error("prox computation must be erroneous") - verbose > 0 && @debug "R2: found point where h has value" hk + verbose > 0 && @debug "R2DH: found point where h has value" hk end improper = (hk == -Inf) @@ -261,8 +264,16 @@ function solve!( set_solver_specific!(stats, :smooth_obj, fk) set_solver_specific!(stats, :nonsmooth_obj, hk) - φ(d) = ∇fk' * d + (d' * (dkσk .* d)) / 2 #TODO This probably allocated a little - mk(d) = φ(d) + ψ(d) + φ(d) = begin + result = zero(T) + n = length(d) + @inbounds for i = 1:n + result += d[i]^2*dkσk[i]/2 + ∇fk[i]*d[i] + end + return result + end + + mk(d)::T = φ(d) + ψ(d)::T spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) @@ -274,13 +285,14 @@ function solve!( ν₁ = 1 / ((DNorm + σk) * (1 + θ)) @. mν∇fk = -ν₁ * ∇fk spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) + mks = mk(s) end ξ = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) solved = (ξ < 0 && sqrt_ξ_νInv ≤ neg_tol) || (ξ ≥ 0 && sqrt_ξ_νInv ≤ atol) (ξ < 0 && sqrt_ξ_νInv > neg_tol) && - error("R2DH: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") + error("R2DH: prox-gradient step should produce a decrease but ξ = $(ξ)") atol += rtol * sqrt_ξ_νInv # make stopping test absolute and relative set_solver_specific!(stats, :xi, sqrt_ξ_νInv) @@ -331,9 +343,6 @@ function solve!( ], colsep = 1, ) - if η2 ≤ ρk < Inf - σk = max(σk / γ, σmin) - end if η1 ≤ ρk < Inf xk .= xkn @@ -346,10 +355,15 @@ function solve!( hk = hkn shift!(ψ, xk) grad!(nlp, xk, ∇fk) - push!(D, s, ∇fk - ∇fk⁻) # update QN operator + @. ∇fk⁻ = ∇fk - ∇fk⁻ + push!(D, s, ∇fk⁻) # update QN operator ∇fk⁻ .= ∇fk end + if η2 ≤ ρk < Inf + σk = max(σk / γ, σmin) + end + if ρk < η1 || ρk == Inf σk = σk * γ end @@ -378,13 +392,14 @@ function solve!( ν₁ = 1 / ((DNorm + σk) * (1 + θ)) @. mν∇fk = -ν₁ * ∇fk spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) + mks = mk(s) end ξ = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) solved = (ξ < 0 && sqrt_ξ_νInv ≤ neg_tol) || (ξ ≥ 0 && sqrt_ξ_νInv ≤ atol) (ξ < 0 && sqrt_ξ_νInv > neg_tol) && - error("R2DH: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") + error("R2DH: prox-gradient step should produce a decrease but ξ = $(ξ)") set_solver_specific!(stats, :xi, sqrt_ξ_νInv) set_status!( From 70499ded9fcfad2219557e06d474929c6c8c196f Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 31 Dec 2024 13:45:59 +0100 Subject: [PATCH 08/48] add R2DH documentation --- src/R2DH_alg.jl | 88 ++++++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index bdab81d7..ad26eebe 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -71,51 +71,73 @@ end """ - R2DH(nlp, h, options) - R2DH(f, ∇f!, h, options, x0) + R2DH(reg_nlp; kwargs…) A second-order quadratic regularization method for the problem min f(x) + h(x) -where f: ℝⁿ → ℝ is C¹ and h: ℝⁿ → ℝ is lower semi-continuous, proper, and prox-bounded. +where f: ℝⁿ → ℝ has a Lipschitz-continuous gradient, and h: ℝⁿ → ℝ is +lower semi-continuous, proper and prox-bounded. About each iterate xₖ, a step sₖ is computed as a solution of - min φ(s; xₖ) + ψ(s; xₖ) + min φ(s; xₖ) + ½ σₖ ‖s‖² + ψ(s; xₖ) -where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀ(Dₖ + σₖI)s is a quadratic approximation of f about xₖ, -ψ(s; xₖ) = h(xₖ + s), Dₖ is a diagonal Hessian approximation and σₖ > 0 is the regularization parameter. +where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀDₖs is a quadratic approximation of f about xₖ, +ψ(s; xₖ) is either h(xₖ + s) or an approximation of h(xₖ + s), ‖⋅‖ is the ℓ₂ norm and σₖ > 0 is the regularization parameter. -### Arguments +For advanced usage, first define a solver "R2DHSolver" to preallocate the memory used in the algorithm, and then call `solve!`: -* `nlp::AbstractDiagonalQNModel`: a smooth optimization problem -* `h`: a regularizer such as those defined in ProximalOperators -* `options::ROSolverOptions`: a structure containing algorithmic parameters -* `x0::AbstractVector`: an initial guess (in the second calling form) + solver = R2DHSolver(reg_nlp; m_monotone = 1) + solve!(solver, reg_nlp) -### Keyword Arguments - -* `x0::AbstractVector`: an initial guess (in the first calling form: default = `nlp.meta.x0`) -* `selected::AbstractVector{<:Integer}`: subset of variables to which `h` is applied (default `1:length(x0)`). -* `D`: Diagonal quasi-Newton operator. -* `Mmonotone::Int`: number of previous values of the objective to consider for the non-monotone variant (default: 6). - -The objective and gradient of `nlp` will be accessed. - -In the second form, instead of `nlp`, the user may pass in - -* `f` a function such that `f(x)` returns the value of f at x -* `∇f!` a function to evaluate the gradient in place, i.e., such that `∇f!(g, x)` store ∇f(x) in `g` - -### Return values - -* `xk`: the final iterate -* `Fobj_hist`: an array with the history of values of the smooth objective -* `Hobj_hist`: an array with the history of values of the nonsmooth objective -* `Complex_hist`: an array with the history of number of inner iterations. + stats = RegularizedExecutionStats(reg_nlp) + solver = R2DHSolver(reg_nlp) + solve!(solver, reg_nlp, stats) + +# Arguments +* `reg_nlp::AbstractRegularizedNLPModel{T, V}`: the problem to solve, see `RegularizedProblems.jl`, `NLPModels.jl`. + +# Keyword arguments +- `x::V = nlp.meta.x0`: the initial guess; +- `atol::T = √eps(T)`: absolute tolerance; +- `rtol::T = √eps(T)`: relative tolerance; +- `neg_tol::T = eps(T)^(1 / 4)`: negative tolerance +- `max_eval::Int = -1`: maximum number of evaluation of the objective function (negative number means unlimited); +- `max_time::Float64 = 30.0`: maximum time limit in seconds; +- `max_iter::Int = 10000`: maximum number of iterations; +- `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration; +- `σmin::T = eps(T)`: minimum value of the regularization parameter; +- `η1::T = √√eps(T)`: very successful iteration threshold; +- `η2::T = T(0.9)`: successful iteration threshold; +- `ν::T = eps(T)^(1 / 5)`: multiplicative inverse of the regularization parameter: ν = 1/σ; +- `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. +- `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. +- `m_monotone::Int = 1`: monotoneness parameter. By default, R2DH is monotone but the non-monotone variant can be used with `m_monotone > 1` + +The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. + +# Output +The value returned is a `GenericExecutionStats`, see `SolverCore.jl`. + +# Callback +The callback is called at each iteration. +The expected signature of the callback is `callback(nlp, solver, stats)`, and its output is ignored. +Changing any of the input arguments will affect the subsequent iterations. +In particular, setting `stats.status = :user` will stop the algorithm. +All relevant information should be available in `nlp` and `solver`. +Notably, you can access, and modify, the following: +- `solver.xk`: current iterate; +- `solver.∇fk`: current gradient; +- `stats`: structure holding the output of the algorithm (`GenericExecutionStats`), which contains, among other things: + - `stats.iter`: current iteration counter; + - `stats.objective`: current objective function value; + - `stats.solver_specific[:smooth_obj]`: current value of the smooth part of the objective function + - `stats.solver_specific[:nonsmooth_obj]`: current value of the nonsmooth part of the objective function + - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything will stop the algorithm, but you should use `:user` to properly indicate the intention. + - `stats.elapsed_time`: elapsed time in seconds. """ - function R2DH( nlp::AbstractDiagonalQNModel{T, V}, h, @@ -175,7 +197,7 @@ function solve!( η2::T = T(0.9), ν::T = eps(T)^(1 / 5), γ::T = T(3), - θ::T = eps(T)^(1/5), + θ::T = eps(T)^(1 / 5), ) where{T, V} reset!(stats) From 0fe3fbb49a7b3fff5fae1263c8e85b016e0f805e Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sat, 4 Jan 2025 16:35:47 +0100 Subject: [PATCH 09/48] minor updates for R2DH --- src/R2DH_alg.jl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index ad26eebe..ed7e69ef 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -1,8 +1,10 @@ -export R2DH, R2DH_mine, R2DHSolver, solve! +export R2DH, R2DHSolver, solve! + +import SolverCore.solve! mutable struct R2DHSolver{ T <: Real, - G <: Union{ShiftedProximableFunction, Nothing}, + G <: ShiftedProximableFunction, V <: AbstractVector{T}, QN <: AbstractDiagonalQuasiNewtonOperator{T} } <: AbstractOptimizationSolver @@ -179,7 +181,7 @@ function R2DH( return stats end -function solve!( +function SolverCore.solve!( solver::R2DHSolver{T}, reg_nlp::AbstractRegularizedNLPModel{T, V}, stats::GenericExecutionStats{T, V}; @@ -199,6 +201,7 @@ function solve!( γ::T = T(3), θ::T = eps(T)^(1 / 5), ) where{T, V} + reset!(stats) # Retrieve workspace @@ -223,6 +226,8 @@ function solve!( has_bnds = solver.has_bnds if has_bnds + l_bound_m_x = solver.l_bound_m_x + u_bound_m_x = solver.u_bound_m_x l_bound = solver.l_bound u_bound = solver.u_bound end @@ -254,7 +259,7 @@ function solve!( :σ => "σ", :normx => "‖x‖", :norms => "‖s‖", - :arrow => " ", + :arrow => "R2DH", ), colsep = 1, ) From 7bc3326f5f4282cfe343f64df166e28535c324e1 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sun, 5 Jan 2025 18:44:16 +0100 Subject: [PATCH 10/48] significantly reduce allocations in R2N - down to 8.32Ko --- src/R2DH_alg.jl | 7 +- src/R2NModel.jl | 52 ++++++ src/R2N_alg.jl | 282 +++++++++++++++++++++------------ src/RegularizedOptimization.jl | 1 + 4 files changed, 237 insertions(+), 105 deletions(-) create mode 100644 src/R2NModel.jl diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index ed7e69ef..c8dcf49f 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -239,7 +239,7 @@ function SolverCore.solve!( hk = @views h(xk[selected]) if hk == Inf verbose > 0 && @info "R2DH: finding initial guess where nonsmooth term is finite" - prox!(xk, h, xk, one(eltype(x0))) + prox!(xk, h, xk, T(1)) hk = @views h(xk[selected]) hk < Inf || error("prox computation must be erroneous") verbose > 0 && @debug "R2DH: found point where h has value" hk @@ -249,14 +249,11 @@ function SolverCore.solve!( if verbose > 0 @info log_header( [:iter, :fx, :hx, :xi, :ρ, :σ, :normx, :norms, :arrow], - [Int, Float64, Float64, Float64, Float64, Float64, Float64, Float64, Char], + [Int, T, T, T, T, T, T, T, Char], hdr_override = Dict{Symbol, String}( # TODO: Add this as constant dict elsewhere - :iter => "iter", :fx => "f(x)", :hx => "h(x)", :xi => "√(ξ/ν)", - :ρ => "ρ", - :σ => "σ", :normx => "‖x‖", :norms => "‖s‖", :arrow => "R2DH", diff --git a/src/R2NModel.jl b/src/R2NModel.jl new file mode 100644 index 00000000..5e944820 --- /dev/null +++ b/src/R2NModel.jl @@ -0,0 +1,52 @@ +export R2NModel + +mutable struct R2NModel{T <: Real, V <: AbstractVector{T}, G <: AbstractLinearOperator{T}} <: AbstractNLPModel{T, V} + B :: G + ∇f :: V + v :: V + σ :: T + meta::NLPModelMeta{T, V} + counters::Counters +end + + function R2NModel( + B :: G, + ∇f :: V, + v :: V, + σ :: T, + x0 :: V +) where{T, V, G} + meta = NLPModelMeta( + length(∇f), + x0 = x0, # Perhaps we should add lvar and uvar as well here. + ) + return R2NModel( + B :: G, + ∇f :: V, + v :: V, + σ :: T, + meta, + Counters() + ) +end + +function NLPModels.obj(nlp::R2NModel, x::AbstractVector) + @lencheck nlp.meta.nvar x + increment!(nlp, :neval_obj) + mul!(nlp.v, nlp.B, x) + return dot(nlp.v, x)/2 + dot(nlp.∇f, x) + nlp.σ * dot(x, x) / 2 +end + +function NLPModels.grad!(nlp::R2NModel, x::AbstractVector, g::AbstractVector) + @lencheck nlp.meta.nvar x + @lencheck nlp.meta.nvar g + increment!(nlp, :neval_grad) + mul!(g, nlp.B, x) + g .+= nlp.∇f + g .+= nlp.σ * x + return g +end + +function NLPModels.push!(nlp::R2NModel, s::AbstractVector, y::AbstractVector) + push!(nlp.B, s, y) +end \ No newline at end of file diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 8742a2be..76d24c59 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -6,23 +6,26 @@ mutable struct R2NSolver{ T <: Real, G <: ShiftedProximableFunction, V <: AbstractVector{T}, - S <: AbstractOptimizationSolver, + ST <: AbstractOptimizationSolver, + PB <: AbstractRegularizedNLPModel } <: AbstractOptimizationSolver xk::V ∇fk::V ∇fk⁻::V mν∇fk::V ψ::G - sub_ψ::G xkn::V s::V s1::V has_bnds::Bool l_bound::V u_bound::V + l_bound_m_x::V + u_bound_m_x::V m_fh_hist::V - subsolver::S - substats::GenericExecutionStats{T, V, V, Any} + subsolver::ST + subpb::PB + substats::GenericExecutionStats{T, V, V, T} end function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Solver, m_monotone::Int = 1) where {T, V} @@ -35,42 +38,131 @@ function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Sol ∇fk⁻ = similar(x0) mν∇fk = similar(x0) xkn = similar(x0) - s = zero(x0) + s = similar(x0) s1 = similar(x0) + v = similar(x0) has_bnds = any(l_bound .!= T(-Inf)) || any(u_bound .!= T(Inf)) + if has_bnds + l_bound_m_x = similar(xk) + u_bound_m_x = similar(xk) + @. l_bound_m_x = l_bound - x0 + @. u_bound_m_x = u_bound - x0 + else + l_bound_m_x = similar(xk, 0) + u_bound_m_x = similar(xk, 0) + end m_fh_hist = fill(T(-Inf), m_monotone - 1) ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) - sub_ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) - sub_nlp = RegularizedNLPModel(reg_nlp.model, sub_ψ) - substats = GenericExecutionStats(reg_nlp.model) + Bk = hess_op(reg_nlp.model, x0) + sub_nlp = R2NModel( + Bk, + ∇fk, + v, + T(1), + x0 + ) + subpb = RegularizedNLPModel(sub_nlp, ψ) + substats = GenericExecutionStats(subpb) + subsolver = subsolver(subpb) - return R2NSolver( + return R2NSolver{T, typeof(ψ), V, typeof(subsolver), typeof(subpb)}( xk, ∇fk, ∇fk⁻, mν∇fk, ψ, - sub_ψ, xkn, s, s1, has_bnds, l_bound, u_bound, + l_bound_m_x, + u_bound_m_x, m_fh_hist, - subsolver(sub_nlp), + subsolver, + subpb, substats ) end - + + +""" + R2N(reg_nlp; kwargs…) + +A second-order quadratic regularization method for the problem + + min f(x) + h(x) + +where f: ℝⁿ → ℝ has a Lipschitz-continuous gradient, and h: ℝⁿ → ℝ is +lower semi-continuous, proper and prox-bounded. + +About each iterate xₖ, a step sₖ is computed as a solution of + + min φ(s; xₖ) + ½ σₖ ‖s‖² + ψ(s; xₖ) + +where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀBₖs is a quadratic approximation of f about xₖ, +ψ(s; xₖ) is either h(xₖ + s) or an approximation of h(xₖ + s), ‖⋅‖ is the ℓ₂ norm and σₖ > 0 is the regularization parameter. + +For advanced usage, first define a solver "R2NSolver" to preallocate the memory used in the algorithm, and then call `solve!`: + + solver = R2NSolver(reg_nlp; m_monotone = 1) + solve!(solver, reg_nlp) + + stats = RegularizedExecutionStats(reg_nlp) + solver = R2NSolver(reg_nlp) + solve!(solver, reg_nlp, stats) + +# Arguments +* `reg_nlp::AbstractRegularizedNLPModel{T, V}`: the problem to solve, see `RegularizedProblems.jl`, `NLPModels.jl`. + +# Keyword arguments +- `x::V = nlp.meta.x0`: the initial guess; +- `atol::T = √eps(T)`: absolute tolerance; +- `rtol::T = √eps(T)`: relative tolerance; +- `neg_tol::T = eps(T)^(1 / 4)`: negative tolerance +- `max_eval::Int = -1`: maximum number of evaluation of the objective function (negative number means unlimited); +- `max_time::Float64 = 30.0`: maximum time limit in seconds; +- `max_iter::Int = 10000`: maximum number of iterations; +- `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration; +- `σmin::T = eps(T)`: minimum value of the regularization parameter; +- `η1::T = √√eps(T)`: very successful iteration threshold; +- `η2::T = T(0.9)`: successful iteration threshold; +- `ν::T = eps(T)^(1 / 5)`: multiplicative inverse of the regularization parameter: ν = 1/σ; +- `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. +- `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. +- `m_monotone::Int = 1`: monotoneness parameter. By default, R2DH is monotone but the non-monotone variant can be used with `m_monotone > 1` + +The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. + +# Output +The value returned is a `GenericExecutionStats`, see `SolverCore.jl`. + +# Callback +The callback is called at each iteration. +The expected signature of the callback is `callback(nlp, solver, stats)`, and its output is ignored. +Changing any of the input arguments will affect the subsequent iterations. +In particular, setting `stats.status = :user` will stop the algorithm. +All relevant information should be available in `nlp` and `solver`. +Notably, you can access, and modify, the following: +- `solver.xk`: current iterate; +- `solver.∇fk`: current gradient; +- `stats`: structure holding the output of the algorithm (`GenericExecutionStats`), which contains, among other things: + - `stats.iter`: current iteration counter; + - `stats.objective`: current objective function value; + - `stats.solver_specific[:smooth_obj]`: current value of the smooth part of the objective function + - `stats.solver_specific[:nonsmooth_obj]`: current value of the nonsmooth part of the objective function + - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything will stop the algorithm, but you should use `:user` to properly indicate the intention. + - `stats.elapsed_time`: elapsed time in seconds. +""" function R2N( nlp::AbstractNLPModel{T, V}, h, options::ROSolverOptions{T}; kwargs..., -) where {T <: Real, V} +) where{T, V} kwargs_dict = Dict(kwargs...) selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) @@ -89,31 +181,30 @@ function R2N( η2 = options.η2, ν = options.ν, γ = options.γ, - θ = options.γ, - β = options.β; + θ = options.θ, kwargs_dict..., ) end -function R2N(reg_nlp::AbstractRegularizedNLPModel; kwargs...) +function R2N( + reg_nlp::AbstractRegularizedNLPModel{T, V}; + kwargs... +) where{T, V} kwargs_dict = Dict(kwargs...) m_monotone = pop!(kwargs_dict, :m_monotone, 1) - subsolver = pop!(kwargs_dict, :subsolver, R2Solver) - solver = R2NSolver(reg_nlp, subsolver = subsolver, m_monotone = m_monotone) + solver = R2NSolver(reg_nlp, m_monotone = m_monotone) stats = GenericExecutionStats(reg_nlp.model) solve!(solver, reg_nlp, stats; kwargs_dict...) return stats end - function SolverCore.solve!( - solver::R2NSolver{T}, + solver::R2NSolver{T, G, V}, reg_nlp::AbstractRegularizedNLPModel{T, V}, stats::GenericExecutionStats{T, V}; callback = (args...) -> nothing, x::V = reg_nlp.model.meta.x0, atol::T = √eps(T), - sub_atol::T = atol, rtol::T = √eps(T), neg_tol::T = eps(T)^(1 / 4), verbose::Int = 0, @@ -123,13 +214,12 @@ function SolverCore.solve!( σmin::T = eps(T), η1::T = √√eps(T), η2::T = T(0.9), - ν = eps(T)^(1/5), + ν::T = eps(T)^(1 / 5), γ::T = T(3), - β::T = 1/eps(T), - θ::T = eps(T)^(1/5), + β::T = 1 / eps(T), + θ::T = eps(T)^(1 / 5), kwargs... -) where {T, V} - +) where{T, V, G} reset!(stats) # Retrieve workspace @@ -142,7 +232,6 @@ function SolverCore.solve!( # Make sure ψ has the correct shift shift!(solver.ψ, xk) - σk = 1/ν ∇fk = solver.∇fk ∇fk⁻ = solver.∇fk⁻ mν∇fk = solver.mν∇fk @@ -150,23 +239,23 @@ function SolverCore.solve!( xkn = solver.xkn s = solver.s s1 = solver.s1 + m_fh_hist = solver.m_fh_hist has_bnds = solver.has_bnds + if has_bnds + l_bound_m_x = solver.l_bound_m_x + u_bound_m_x = solver.u_bound_m_x l_bound = solver.l_bound u_bound = solver.u_bound end - m_fh_hist = solver.m_fh_hist m_monotone = length(m_fh_hist) + 1 - subsolver = solver.subsolver - substats = solver.substats - # initialize parameters improper = false hk = @views h(xk[selected]) if hk == Inf verbose > 0 && @info "R2N: finding initial guess where nonsmooth term is finite" - prox!(xk, h, xk, one(eltype(x0))) + prox!(xk, h, xk, T(1)) hk = @views h(xk[selected]) hk < Inf || error("prox computation must be erroneous") verbose > 0 && @debug "R2N: found point where h has value" hk @@ -176,15 +265,11 @@ function SolverCore.solve!( if verbose > 0 @info log_header( [:outer, :inner, :fx, :hx, :xi, :ρ, :σ, :normx, :norms, :normB, :arrow], - [Int, Int, Float64, Float64, Float64, Float64, Float64, Float64, Float64, Float64, Char], - hdr_override = Dict{Symbol, String}( # TODO: Add this as constant dict elsewhere - :outer => "outer", - :inner => "inner", + [Int, Int, T, T, T, T, T, T, T, T, Char], + hdr_override = Dict{Symbol, String}( :fx => "f(x)", :hx => "h(x)", :xi => "√(ξ1/ν)", - :ρ => "ρ", - :σ => "σ", :normx => "‖x‖", :norms => "‖s‖", :normB => "‖B‖", @@ -195,27 +280,27 @@ function SolverCore.solve!( end local ξ1::T - local ρk::T - ρk = T(1) + local ρk::T = zero(T) + + σk = max(1 / ν, σmin) fk = obj(nlp, xk) grad!(nlp, xk, ∇fk) ∇fk⁻ .= ∇fk quasiNewtTest = isa(nlp, QuasiNewtonModel) - Bk = hess_op(nlp, xk) - local λmax::T + λmax::T = T(1) + """ try - λmax = opnorm(Bk) - catch LAPACKException # This is really bad and should be removed ASAP; see PR #159. - λmax = opnorm(Matrix(Bk)) + λmax = opnorm(solver.subpb.model.B) # TODO: This allocates; see utils.jl + catch LAPACKException # This should be removed ASAP; see PR #159. + λmax = opnorm(Matrix(solver.subpb.model.B)) end - - νInv = (1 + θ) *( σk + λmax) + """ + ν₁ = 1 / ((λmax + σk) * (1 + θ)) sqrt_ξ1_νInv = one(T) - ν_subsolver = 1/νInv - @. mν∇fk = -ν_subsolver * ∇fk + @. mν∇fk = -ν₁ * ∇fk m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) set_iter!(stats, 0) @@ -225,32 +310,28 @@ function SolverCore.solve!( set_solver_specific!(stats, :smooth_obj, fk) set_solver_specific!(stats, :nonsmooth_obj, hk) - # model for first prox-gradient step and ξ1 - φ1(d) = ∇fk' * d - mk1(d) = φ1(d) + ψ(d) + φ1 = let ∇fk = ∇fk + d -> dot(∇fk, d) + end - # model for subsequent prox-gradient steps and ξ - φ(d) = (d' * (Bk * d)) / 2 + ∇fk' * d + σk * dot(d, d) / 2 # TODO this probably allocates a little. + mk1 = let ψ = ψ + d -> φ1(d) + ψ(d)::T + end - ∇φ!(g, d) = begin - mul!(g, Bk, d) - g .+= ∇fk - g .+= σk * d - g + mk = let ψ = ψ, solver = solver + d -> obj(solver.subpb.model, d) + ψ(d)::T end - mk(d) = φ(d) + ψ(d) - prox!(s, ψ, mν∇fk, ν_subsolver) + prox!(s, ψ, mν∇fk, ν₁) mks = mk1(s) ξ1 = hk - mks + max(1, abs(hk)) * 10 * eps() - sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 * νInv) : sqrt(-ξ1 * νInv) + sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 / ν₁) : sqrt(-ξ1 / ν₁) solved = (ξ1 < 0 && sqrt_ξ1_νInv ≤ neg_tol) || (ξ1 ≥ 0 && sqrt_ξ1_νInv ≤ atol) (ξ1 < 0 && sqrt_ξ1_νInv > neg_tol) && error("R2N: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") atol += rtol * sqrt_ξ1_νInv # make stopping test absolute and relative - sub_atol += rtol * sqrt_ξ1_νInv - + set_solver_specific!(stats, :xi, sqrt_ξ1_νInv) set_status!( stats, @@ -270,38 +351,31 @@ function SolverCore.solve!( done = stats.status != :unknown - while !done + while !done s1 .= s - sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5) , sqrt_ξ1_νInv * 1e-3) # 1.0e-5 default - #@debug "setting inner stopping tolerance to" subsolver_options.optTol - #subsolver_args = subsolver == R2DH ? (SpectralGradient(1., f.meta.nvar),) : () - nlp_model = FirstOrderModel(φ,∇φ!,s) - model = RegularizedNLPModel(nlp_model, ψ) - #model.selected .= reg_nlp.selected + + solver.subpb.model.σ = σk solve!( - subsolver, - model, - substats, - x = s, + solver.subsolver, + solver.subpb, + solver.substats; + x = s, atol = sub_atol, - ν = ν_subsolver; - kwargs...) - - s .= substats.solution + ν = ν₁, + kwargs... + ) + + s .= solver.substats.solution if norm(s) > β * norm(s1) s .= s1 end - """ - if mk(s) > mk(s1) - s .= s1 - end - """ + xkn .= xk .+ s fkn = obj(nlp, xkn) - hkn = h(xkn[selected]) + hkn = @views h(xkn[selected]) hkn == -Inf && error("nonsmooth term is not proper") mks = mk(s) @@ -310,7 +384,7 @@ function SolverCore.solve!( Δmod = fhmax - (fk + mks) + max(1, abs(fhmax)) * 10 * eps() ξ = hk - mks + max(1, abs(hk)) * 10 * eps() - if (ξ ≤ 0 || isnan(ξ)) + if (ξ < 0 || isnan(ξ)) error("R2N: failed to compute a step: ξ = $ξ") end @@ -321,7 +395,7 @@ function SolverCore.solve!( @info log_row( Any[ stats.iter, - substats.iter, + solver.substats.iter, fk, hk, sqrt_ξ1_νInv, @@ -341,8 +415,11 @@ function SolverCore.solve!( if η1 ≤ ρk < Inf xk .= xkn - has_bounds(nlp) && set_bounds!(ψ, l_bound - xk, u_bound - xk) - + if has_bnds + @. l_bound_m_x = l_bound - xk + @. u_bound_m_x = u_bound - xk + set_bounds!(ψ, l_bound_m_x, u_bound_m_x) + end #update functions fk = fkn hk = hkn @@ -351,22 +428,28 @@ function SolverCore.solve!( ∇fk = grad!(nlp, xk, ∇fk) if quasiNewtTest - push!(nlp, s, ∇fk - ∇fk⁻) + @. ∇fk⁻ = ∇fk - ∇fk⁻ + push!(nlp, s, ∇fk⁻) end - Bk = hess_op(nlp, xk) + """ try - λmax = opnorm(Bk) + λmax = opnorm(solver.subpb.model.B) catch LAPACKException - λmax = opnorm(Matrix(Bk)) + λmax = opnorm(Matrix(solver.subpb.model.B)) end + """ ∇fk⁻ .= ∇fk end + if η2 ≤ ρk < Inf + σk = max(σk/γ, σmin) + end + if ρk < η1 || ρk == Inf σk = σk * γ end - - νInv = (1 + θ) *( σk + λmax) + + ν₁ = 1/(1 + θ) *( σk + λmax) m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) set_objective!(stats, fk + hk) @@ -375,14 +458,13 @@ function SolverCore.solve!( set_iter!(stats, stats.iter + 1) set_time!(stats, time() - start_time) - ν_subsolver = 1/νInv - @. mν∇fk = - ν_subsolver * ∇fk - prox!(s, ψ, mν∇fk, ν_subsolver) + @. mν∇fk = - ν₁ * ∇fk + prox!(s, ψ, mν∇fk, ν₁) mks = mk1(s) ξ1 = hk - mks + max(1, abs(hk)) * 10 * eps() - sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 * νInv) : sqrt(-ξ1 * νInv) + sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 / ν₁) : sqrt(-ξ1 / ν₁) solved = (ξ1 < 0 && sqrt_ξ1_νInv ≤ neg_tol) || (ξ1 ≥ 0 && sqrt_ξ1_νInv ≤ atol) (ξ1 < 0 && sqrt_ξ1_νInv > neg_tol) && @@ -427,6 +509,6 @@ function SolverCore.solve!( @info "R2N: terminating with √(ξ1/ν) = $(sqrt_ξ1_νInv)" end - set_solution!(stats,xk) + set_solution!(stats, xk) return stats end \ No newline at end of file diff --git a/src/RegularizedOptimization.jl b/src/RegularizedOptimization.jl index 3d9df94f..7e243b20 100644 --- a/src/RegularizedOptimization.jl +++ b/src/RegularizedOptimization.jl @@ -20,6 +20,7 @@ include("TR_alg.jl") include("TRDH_alg.jl") include("R2_alg.jl") include("R2DH_alg.jl") +include("R2NModel.jl") include("R2N_alg.jl") include("LM_alg.jl") include("LMTR_alg.jl") From 6db3f105340ccb6c040e071902812c91c90ec773 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 7 Jan 2025 12:06:52 +0100 Subject: [PATCH 11/48] resolve allocation for R2N --- src/R2NModel.jl | 2 +- src/R2N_alg.jl | 28 ++++++++++++---------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/R2NModel.jl b/src/R2NModel.jl index 5e944820..1c234d52 100644 --- a/src/R2NModel.jl +++ b/src/R2NModel.jl @@ -43,7 +43,7 @@ function NLPModels.grad!(nlp::R2NModel, x::AbstractVector, g::AbstractVector) increment!(nlp, :neval_grad) mul!(g, nlp.B, x) g .+= nlp.∇f - g .+= nlp.σ * x + g .+= nlp.σ .* x return g end diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 76d24c59..78719fbc 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -162,7 +162,7 @@ function R2N( h, options::ROSolverOptions{T}; kwargs..., -) where{T, V} +) where {T <: Real, V} kwargs_dict = Dict(kwargs...) selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) @@ -180,19 +180,16 @@ function R2N( η1 = options.η1, η2 = options.η2, ν = options.ν, - γ = options.γ, - θ = options.θ, + γ = options.γ; kwargs_dict..., ) end -function R2N( - reg_nlp::AbstractRegularizedNLPModel{T, V}; - kwargs... -) where{T, V} +function R2N(reg_nlp::AbstractRegularizedNLPModel; kwargs...) kwargs_dict = Dict(kwargs...) m_monotone = pop!(kwargs_dict, :m_monotone, 1) - solver = R2NSolver(reg_nlp, m_monotone = m_monotone) + subsolver = pop!(kwargs_dict, :subsolver, R2Solver) + solver = R2NSolver(reg_nlp, subsolver = subsolver, m_monotone = m_monotone) stats = GenericExecutionStats(reg_nlp.model) solve!(solver, reg_nlp, stats; kwargs_dict...) return stats @@ -322,8 +319,8 @@ function SolverCore.solve!( d -> obj(solver.subpb.model, d) + ψ(d)::T end - prox!(s, ψ, mν∇fk, ν₁) - mks = mk1(s) + prox!(s1, ψ, mν∇fk, ν₁) + mks = mk1(s1) ξ1 = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 / ν₁) : sqrt(-ξ1 / ν₁) @@ -353,15 +350,14 @@ function SolverCore.solve!( while !done - s1 .= s sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5) , sqrt_ξ1_νInv * 1e-3) # 1.0e-5 default - + solver.subpb.model.σ = σk solve!( solver.subsolver, solver.subpb, solver.substats; - x = s, + x = s1, atol = sub_atol, ν = ν₁, kwargs... @@ -429,7 +425,7 @@ function SolverCore.solve!( if quasiNewtTest @. ∇fk⁻ = ∇fk - ∇fk⁻ - push!(nlp, s, ∇fk⁻) + push!(solver.subpb.model.B, s, ∇fk⁻) end """ try @@ -459,8 +455,8 @@ function SolverCore.solve!( set_time!(stats, time() - start_time) @. mν∇fk = - ν₁ * ∇fk - prox!(s, ψ, mν∇fk, ν₁) - mks = mk1(s) + prox!(s1, ψ, mν∇fk, ν₁) + mks = mk1(s1) ξ1 = hk - mks + max(1, abs(hk)) * 10 * eps() From b0c96032439584f5545b6fc983bbcb72103340ad Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 7 Jan 2025 17:03:33 +0100 Subject: [PATCH 12/48] fix few errors in R2N --- src/R2N_alg.jl | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 78719fbc..672dca42 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -287,13 +287,14 @@ function SolverCore.solve!( quasiNewtTest = isa(nlp, QuasiNewtonModel) λmax::T = T(1) - """ + solver.subpb.model.B = hess_op(nlp, xk) + try λmax = opnorm(solver.subpb.model.B) # TODO: This allocates; see utils.jl catch LAPACKException # This should be removed ASAP; see PR #159. λmax = opnorm(Matrix(solver.subpb.model.B)) end - """ + ν₁ = 1 / ((λmax + σk) * (1 + θ)) sqrt_ξ1_νInv = one(T) @@ -350,7 +351,7 @@ function SolverCore.solve!( while !done - sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5) , sqrt_ξ1_νInv * 1e-3) # 1.0e-5 default + sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5) , sqrt_ξ1_νInv * 1e-3) solver.subpb.model.σ = σk solve!( @@ -405,10 +406,6 @@ function SolverCore.solve!( colsep = 1, ) - if η2 ≤ ρk < Inf - σk = max(σk/γ, σmin) - end - if η1 ≤ ρk < Inf xk .= xkn if has_bnds @@ -425,15 +422,16 @@ function SolverCore.solve!( if quasiNewtTest @. ∇fk⁻ = ∇fk - ∇fk⁻ - push!(solver.subpb.model.B, s, ∇fk⁻) + push!(nlp, s, ∇fk⁻) end - """ + solver.subpb.model.B = hess_op(nlp, xk) + try λmax = opnorm(solver.subpb.model.B) catch LAPACKException λmax = opnorm(Matrix(solver.subpb.model.B)) end - """ + ∇fk⁻ .= ∇fk end @@ -445,7 +443,7 @@ function SolverCore.solve!( σk = σk * γ end - ν₁ = 1/(1 + θ) *( σk + λmax) + ν₁ = 1 / ((λmax + σk) * (1 + θ)) m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) set_objective!(stats, fk + hk) From 9cc1a5af4918c5f2acb993670f2d29e53554b91e Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 7 Jan 2025 18:20:56 +0100 Subject: [PATCH 13/48] add doc for R2NModel --- src/R2NModel.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/R2NModel.jl b/src/R2NModel.jl index 1c234d52..f5f5eaf0 100644 --- a/src/R2NModel.jl +++ b/src/R2NModel.jl @@ -1,5 +1,19 @@ export R2NModel +@doc raw""" + R2NModel(B, ∇f, v, σ, x0) + +Given the unconstrained optimization problem: +```math +\min f(x), +``` +this model represents the smooth R2N subproblem: +```math +\min_s \ \tfrac{1}{2} s^T B s + ∇f^T s + \tfrac{1}{2} σ\|s\|^2 +``` +where `B` represents either an approximation of the Hessian of `f` or the Hessian itself and `∇f` represents the gradient of `f` on `x0`. +`σ > 0` is a regularization parameter and `v` is a vector of the same size as `x0` used for computations +""" mutable struct R2NModel{T <: Real, V <: AbstractVector{T}, G <: AbstractLinearOperator{T}} <: AbstractNLPModel{T, V} B :: G ∇f :: V From 07b4655808933b9cd7b966df544c90e79c1822e4 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 8 Jan 2025 11:10:38 +0100 Subject: [PATCH 14/48] update R2DH initialization when called with R2N --- src/R2N_alg.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 672dca42..7c1e3999 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -354,6 +354,7 @@ function SolverCore.solve!( sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5) , sqrt_ξ1_νInv * 1e-3) solver.subpb.model.σ = σk + isa(solver.subsolver, R2DHSolver) && (solver.subsolver.D.d[1] = 1/ν₁) solve!( solver.subsolver, solver.subpb, From c8462de4a7da0f990d1b89d97d09821c127891e2 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 13:02:38 -0500 Subject: [PATCH 15/48] fix rebase issues --- src/R2DH.jl | 321 --------------------------------- src/R2N.jl | 297 ------------------------------ src/R2N_alg.jl | 2 +- src/RegularizedOptimization.jl | 3 - test/runtests.jl | 9 +- test/test_allocs.jl | 15 +- 6 files changed, 11 insertions(+), 636 deletions(-) delete mode 100644 src/R2DH.jl delete mode 100644 src/R2N.jl diff --git a/src/R2DH.jl b/src/R2DH.jl deleted file mode 100644 index bc3a3664..00000000 --- a/src/R2DH.jl +++ /dev/null @@ -1,321 +0,0 @@ -export R2DH - -""" - R2DH(nlp, h, options) - -A second-order quadratic regularization method for the problem - - min f(x) + h(x) - -where f: ℝⁿ → ℝ is C¹ and h: ℝⁿ → ℝ is lower semi-continuous, proper, and prox-bounded. - -About each iterate xₖ, a step sₖ is computed as a solution of - - min φ(s; xₖ) + ψ(s; xₖ) - -where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀ(Dₖ + σₖI)s is a quadratic approximation of f about xₖ, -ψ(s; xₖ) = h(xₖ + s), Dₖ is a diagonal Hessian approximation and σₖ > 0 is the regularization parameter. - -### Arguments - -* `nlp::AbstractDiagonalQNModel`: a smooth optimization problem -* `h`: a regularizer such as those defined in ProximalOperators -* `options::ROSolverOptions`: a structure containing algorithmic parameters - -### Keyword Arguments - -* `x0::AbstractVector`: an initial guess (in the first calling form: default = `nlp.meta.x0`) -* `selected::AbstractVector{<:Integer}`: subset of variables to which `h` is applied (default `1:length(x0)`). -* `D`: Diagonal quasi-Newton operator. -* `Mmonotone::Int`: number of previous values of the objective to consider for the non-monotone variant (default: 6). - -The objective and gradient of `nlp` will be accessed. - -### Return values -The value returned is a `GenericExecutionStats`, see `SolverCore.jl`. -""" -function R2DH( - nlp::AbstractDiagonalQNModel{R, S}, - h, - options::ROSolverOptions{R}; - kwargs..., -) where {R <: Real, S} - kwargs_dict = Dict(kwargs...) - x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) - xk, k, outdict = R2DH( - x -> obj(nlp, x), - (g, x) -> grad!(nlp, x, g), - h, - hess_op(nlp, x0), - options, - x0; - l_bound = nlp.meta.lvar, - u_bound = nlp.meta.uvar, - kwargs..., - ) - sqrt_ξ_νInv = outdict[:sqrt_ξ_νInv] - stats = GenericExecutionStats(nlp) - set_status!(stats, outdict[:status]) - set_solution!(stats, xk) - set_objective!(stats, outdict[:fk] + outdict[:hk]) - set_residuals!(stats, zero(eltype(xk)), sqrt_ξ_νInv) - set_iter!(stats, k) - set_time!(stats, outdict[:elapsed_time]) - set_solver_specific!(stats, :sigma, outdict[:sigma]) - set_solver_specific!(stats, :Fhist, outdict[:Fhist]) - set_solver_specific!(stats, :Hhist, outdict[:Hhist]) - set_solver_specific!(stats, :Time_hist, outdict[:Time_hist]) - set_solver_specific!(stats, :NonSmooth, outdict[:NonSmooth]) - set_solver_specific!(stats, :SubsolverCounter, outdict[:Chist]) - return stats -end - -""" - R2DH(f, ∇f!, h, options, x0) - -A second calling form for `R2DH` where the objective and gradient are passed as arguments. - -### Arguments - -* `f::Function`: the objective function -* `∇f!::Function`: the gradient function -* `h`: a regularizer such as those defined in ProximalOperators -* `D`: Diagonal quasi-Newton operator. -* `options::ROSolverOptions`: a structure containing algorithmic parameters -* `x0::AbstractVector`: an initial guess - -### Keyword Arguments - -* `Mmonotone::Int`: number of previous values of the objective to consider for the non-monotone variant (default: 6). -* `selected::AbstractVector{<:Integer}`: subset of variables to which `h` is applied (default `1:length(x0)`). - -### Return values - -* `xk`: the final iterate -* `k`: the number of iterations -* `outdict`: a dictionary containing the following fields: - * `Fhist`: an array with the history of values of the smooth objective - * `Hhist`: an array with the history of values of the nonsmooth objective - * `Time_hist`: an array with the history of elapsed times - * `Chist`: an array with the history of number of inner iterations - * `NonSmooth`: the nonsmooth term - * `status`: the status of the solver either `:first_order`, `:max_iter`, `:max_time` or `:exception` - * `fk`: the value of the smooth objective at the final iterate - * `hk`: the value of the nonsmooth objective at the final iterate - * `sqrt_ξ_νInv`: the square root of the ratio of the nonsmooth term to the regularization parameter - * `elapsed_time`: the elapsed time -""" -function R2DH( - f::F, - ∇f!::G, - h::H, - D::DQN, - options::ROSolverOptions{R}, - x0::AbstractVector{R}; - Mmonotone::Int = 6, - selected::AbstractVector{<:Integer} = 1:length(x0), - kwargs..., -) where {F <: Function, G <: Function, H, R <: Real, DQN <: AbstractDiagonalQuasiNewtonOperator} - start_time = time() - elapsed_time = 0.0 - ϵ = options.ϵa - ϵr = options.ϵr - neg_tol = options.neg_tol - verbose = options.verbose - maxIter = options.maxIter - maxTime = options.maxTime - σmin = options.σmin - σk = options.σk - η1 = options.η1 - η2 = options.η2 - ν = options.ν - γ = options.γ - θ = options.θ - - local l_bound, u_bound - has_bnds = false - for (key, val) in kwargs - if key == :l_bound - l_bound = val - has_bnds = has_bnds || any(l_bound .!= R(-Inf)) - elseif key == :u_bound - u_bound = val - has_bnds = has_bnds || any(u_bound .!= R(Inf)) - end - end - - if verbose == 0 - ptf = Inf - elseif verbose == 1 - ptf = round(maxIter / 10) - elseif verbose == 2 - ptf = round(maxIter / 100) - else - ptf = 1 - end - - # initialize parameters - xk = copy(x0) - hk = h(xk[selected]) - if hk == Inf - verbose > 0 && @info "R2DH: finding initial guess where nonsmooth term is finite" - prox!(xk, h, x0, one(eltype(x0))) - hk = h(xk[selected]) - hk < Inf || error("prox computation must be erroneous") - verbose > 0 && @debug "R2DH: found point where h has value" hk - end - hk == -Inf && error("nonsmooth term is not proper") - - xkn = similar(xk) - s = zero(xk) - ψ = has_bnds ? shifted(h, xk, l_bound - xk, u_bound - xk, selected) : shifted(h, xk) - - Fobj_hist = zeros(maxIter + 1) - Hobj_hist = zeros(maxIter + 1) - time_hist = zeros(maxIter + 1) - FHobj_hist = fill!(Vector{R}(undef, Mmonotone - 1), R(-Inf)) - Complex_hist = zeros(Int, maxIter + 1) - if verbose > 0 - #! format: off - @info @sprintf "%6s %8s %8s %7s %8s %7s %7s %7s %1s" "iter" "f(x)" "h(x)" "√(ξ/ν)" "ρ" "σ" "‖x‖" "‖s‖" "" - #! format: off - end - - local ξ - k = 0 - - fk = f(xk) - ∇fk = similar(xk) - ∇f!(∇fk, xk) - ∇fk⁻ = copy(∇fk) - spectral_test = isa(D, SpectralGradient) - Dkσk = D.d .+ σk - DNorm = norm(D.d, Inf) - - ν = θ / (DNorm + σk) - mν∇fk = -ν * ∇fk - sqrt_ξ_νInv = one(R) - - optimal = false - tired = maxIter > 0 && k ≥ maxIter || elapsed_time > maxTime - - while !(optimal || tired) - # model with diagonal hessian - φ(d) = ∇fk' * d + (d' * (Dkσk .* d)) / 2 - mk(d) = φ(d) + ψ(d) - - if spectral_test - prox!(s, ψ, mν∇fk, ν) - else - iprox!(s, ψ, ∇fk, Dkσk) - end - mks = mk(s) - - k = k + 1 - elapsed_time = time() - start_time - Fobj_hist[k] = fk - Hobj_hist[k] = hk - time_hist[k] = elapsed_time - Mmonotone > 1 && (FHobj_hist[mod(k-1, Mmonotone - 1) + 1] = fk + hk) - - Complex_hist[k] += 1 - xkn .= xk .+ s - fkn = f(xkn) - hkn = h(xkn[selected]) - hkn == -Inf && error("nonsmooth term is not proper") - - fhmax = Mmonotone > 1 ? maximum(FHobj_hist) : fk + hk - Δobj = fhmax - (fkn + hkn) + max(1, abs(fhmax)) * 10 * eps() - Δmod = fhmax - (fk + mks) + max(1, abs(hk)) * 10 * eps() - ξ = hk - mks + max(1, abs(hk)) * 10 * eps() - sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν) : sqrt(-ξ / ν) - - if ξ ≥ 0 && k == 1 - ϵ += ϵr * sqrt_ξ_νInv # make stopping test absolute and relative - end - - if (ξ < 0 && sqrt_ξ_νInv ≤ neg_tol) || (ξ ≥ 0 && sqrt_ξ_νInv < ϵ) - # the current xk is approximately first-order stationary - optimal = true - continue - end - - if (ξ ≤ 0 || isnan(ξ)) - error("R2DH: failed to compute a step: ξ = $ξ") - end - - ρk = Δobj / Δmod - - σ_stat = (η2 ≤ ρk < Inf) ? "↘" : (ρk < η1 ? "↗" : "=") - - if (verbose > 0) && ((k % ptf == 0) || (k == 1)) - #! format: off - @info @sprintf "%6d %8.1e %8.1e %7.1e %8.1e %7.1e %7.1e %7.1e %1s" k fk hk sqrt_ξ_νInv ρk σk norm(xk) norm(s) σ_stat - #! format: on - end - - if η2 ≤ ρk < Inf - σk = max(σk / γ, σmin) - end - - if η1 ≤ ρk < Inf - xk .= xkn - has_bnds && set_bounds!(ψ, l_bound - xk, u_bound - xk) - fk = fkn - hk = hkn - shift!(ψ, xk) - ∇f!(∇fk, xk) - push!(D, s, ∇fk - ∇fk⁻) # update QN operator - DNorm = norm(D.d, Inf) - ∇fk⁻ .= ∇fk - end - - if ρk < η1 || ρk == Inf - σk = σk * γ - end - - Dkσk .= D.d .+ σk - ν = θ / (DNorm + σk) - - tired = maxIter > 0 && k ≥ maxIter - if !tired - @. mν∇fk = -ν * ∇fk - end - end - - if verbose > 0 - if k == 1 - @info @sprintf "%6d %8.1e %8.1e" k fk hk - elseif optimal - #! format: off - @info @sprintf "%6d %8.1e %8.1e %7.1e %8s %7.1e %7.1e %7.1e" k fk hk sqrt_ξ_νInv "" σk norm(xk) norm(s) - #! format: on - @info "R2DH: terminating with √(ξ/ν) = $(sqrt_ξ_νInv))" - end - end - - status = if optimal - :first_order - elseif elapsed_time > maxTime - :max_time - elseif tired - :max_iter - else - :exception - end - outdict = Dict( - :Fhist => Fobj_hist[1:k], - :Hhist => Hobj_hist[1:k], - :Time_hist => time_hist[1:k], - :Chist => Complex_hist[1:k], - :NonSmooth => h, - :status => status, - :fk => fk, - :hk => hk, - :sqrt_ξ_νInv => sqrt_ξ_νInv, - :elapsed_time => elapsed_time, - :sigma => σk, - ) - - return xk, k, outdict -end diff --git a/src/R2N.jl b/src/R2N.jl deleted file mode 100644 index f1e97ef3..00000000 --- a/src/R2N.jl +++ /dev/null @@ -1,297 +0,0 @@ -export R2N - -""" -R2N(nlp, h, χ, options; kwargs...) - -A regularized quasi-Newton method for the problem - - min f(x) + h(x) - -where f: ℝⁿ → ℝ is C¹ and h: ℝⁿ → ℝ is lower semi-continuous, proper and prox-bounded. - -About each iterate xₖ, a step sₖ is computed as an approximate solution of - - min φ(s; xₖ) + ½ σₖ ‖s‖² + ψ(s; xₖ) - -where φ(s; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀBₖs is a quadratic approximation of f about xₖ, -ψ(s; xₖ) = h(xₖ + s) and σₖ > 0 is the regularization parameter. -The subproblem is solved inexactly by way of a first-order method such as the proximal-gradient -method or the quadratic regularization method. - -### Arguments - -* `nlp::AbstractNLPModel`: a smooth optimization problem -* `h`: a regularizer such as those defined in ProximalOperators -* `options::ROSolverOptions`: a structure containing algorithmic parameters. - -The objective, gradient and Hessian of `nlp` will be accessed. -The Hessian is accessed as an abstract operator and need not be the exact Hessian. - -### Keyword arguments - -* `x0::AbstractVector`: an initial guess (default: `nlp.meta.x0`) -* `subsolver_logger::AbstractLogger`: a logger to pass to the subproblem solver (default: the null logger) -* `subsolver`: the procedure used to compute a step (`R2DH`, `R2` or `PG`) -* `subsolver_options::ROSolverOptions`: default options to pass to the subsolver (default: all defaut options) -* `Mmonotone::Int`: number of previous values of the objective to consider for the non-monotone variant (default: 1). -* `selected::AbstractVector{<:Integer}`: subset of variables to which `h` is applied (default `1:nlp.meta.nvar`). - -### Return values - -* `xk`: the final iterate -* `Fobj_hist`: an array with the history of values of the smooth objective -* `Hobj_hist`: an array with the history of values of the nonsmooth objective -* `Complex_hist`: an array with the history of number of inner iterations. -""" -function R2N( - f::AbstractNLPModel, - h::H, - options::ROSolverOptions{R}; - x0::AbstractVector = f.meta.x0, - subsolver_logger::Logging.AbstractLogger = Logging.NullLogger(), - subsolver = R2, - subsolver_options = ROSolverOptions(ϵa = options.ϵa), - Mmonotone::Int = 1, - selected::AbstractVector{<:Integer} = 1:(f.meta.nvar), -) where {H, R} - start_time = time() - elapsed_time = 0.0 - # initialize passed options - ϵ = options.ϵa - ϵ_subsolver_init = subsolver_options.ϵa - ϵ_subsolver = copy(ϵ_subsolver_init) - ϵr = options.ϵr - Δk = options.Δk - verbose = options.verbose - maxIter = options.maxIter - maxTime = options.maxTime - η1 = options.η1 - η2 = options.η2 - γ = options.γ - θ = options.θ - σmin = options.σmin - α = options.α - β = options.β - σk = options.σk - - # store initial values of the subsolver_options fields that will be modified - ν_subsolver = subsolver_options.ν - ϵa_subsolver = subsolver_options.ϵa - - local l_bound, u_bound - if has_bounds(f) - l_bound = f.meta.lvar - u_bound = f.meta.uvar - end - - if verbose == 0 - ptf = Inf - elseif verbose == 1 - ptf = round(maxIter / 10) - elseif verbose == 2 - ptf = round(maxIter / 100) - else - ptf = 1 - end - - # initialize parameters - xk = copy(x0) - hk = h(xk[selected]) - if hk == Inf - verbose > 0 && @info "R2N: finding initial guess where nonsmooth term is finite" - prox!(xk, h, x0, one(eltype(x0))) - hk = h(xk[selected]) - hk < Inf || error("prox computation must be erroneous") - verbose > 0 && @debug "R2N: found point where h has value" hk - end - hk == -Inf && error("nonsmooth term is not proper") - - xkn = similar(xk) - s = zero(xk) - ψ = has_bounds(f) ? shifted(h, xk, l_bound - xk, u_bound - xk, selected) : shifted(h, xk) - - Fobj_hist = zeros(maxIter) - Hobj_hist = zeros(maxIter) - FHobj_hist = fill!(Vector{R}(undef, Mmonotone - 1), R(-Inf)) - Complex_hist = zeros(Int, maxIter) - if verbose > 0 - #! format: off - @info @sprintf "%6s %8s %8s %8s %7s %7s %8s %7s %7s %7s %7s %1s" "outer" "inner" "f(x)" "h(x)" "√(ξ1/ν)" "√ξ" "ρ" "σ" "‖x‖" "‖s‖" "‖Bₖ‖" "R2N" - #! format: on - end - - # main algorithm initialization - - local ξ1 - k = 0 - - fk = obj(f, xk) - ∇fk = grad(f, xk) - ∇fk⁻ = copy(∇fk) - - quasiNewtTest = isa(f, QuasiNewtonModel) - Bk = hess_op(f, xk) - - λmax, found_λ = opnorm(Bk) - found_λ || error("operator norm computation failed") - ν = θ / (σk + λmax) - sqrt_ξ1_νInv = one(R) - - optimal = false - tired = k ≥ maxIter || elapsed_time > maxTime - - while !(optimal || tired) - k = k + 1 - elapsed_time = time() - start_time - Fobj_hist[k] = fk - Hobj_hist[k] = hk - Mmonotone > 1 && (FHobj_hist[mod(k - 1, Mmonotone - 1) + 1] = fk + hk) - - # model for first prox-gradient step and ξ1 - φ1(d) = ∇fk' * d - mk1(d) = φ1(d) + ψ(d) - - # model for subsequent prox-gradient steps and ξ - φ(d) = ∇fk' * d + dot(d, Bk * d) / 2 + σk * dot(d, d) / 2 - - ∇φ!(g, d) = begin - mul!(g, Bk, d) - g .+= ∇fk - g .+= σk * d - g - end - - mk(d) = φ(d) + ψ(d) - - # take first proximal gradient step s1 and see if current xk is nearly stationary - # s1 minimizes φ1(s) + ‖s‖² / 2 / ν + ψ(s) ⟺ s1 ∈ prox{νψ}(-ν∇φ1(0)). - - subsolver_options.ν = ν - prox!(s, ψ, -subsolver_options.ν * ∇fk, subsolver_options.ν) - ξ1 = hk - mk1(s) + max(1, abs(hk)) * 10 * eps() - ξ1 > 0 || error("R2N: first prox-gradient step should produce a decrease but ξ1 = $(ξ1)") - sqrt_ξ1_νInv = sqrt(ξ1 / ν) - - if ξ1 ≥ 0 && k == 1 - ϵ_increment = ϵr * sqrt_ξ1_νInv - ϵ += ϵ_increment # make stopping test absolute and relative - ϵ_subsolver += ϵ_increment - end - - if sqrt_ξ1_νInv < ϵ - # the current xk is approximately first-order stationary - optimal = true - continue - end - s1 = copy(s) - - subsolver_options.ϵa = k == 1 ? 1.0e-3 : min(sqrt_ξ1_νInv^(1.5), sqrt_ξ1_νInv * 1e-3) - verbose > 0 && @debug "setting inner stopping tolerance to" subsolver_options.optTol - subsolver_options.σk = σk - subsolver_args = subsolver == R2DH ? (SpectralGradient(1 / ν, f.meta.nvar),) : () - s, iter, _ = with_logger(subsolver_logger) do - subsolver(φ, ∇φ!, ψ, subsolver_args..., subsolver_options, s) - end - - if norm(s) > β * norm(s1) - s .= s1 - end - - # restore initial subsolver_options.ϵa here so that subsolver_options.ϵa - # is not modified if there is an error - subsolver_options.ν = ν_subsolver - subsolver_options.ϵa = ϵ_subsolver_init - subsolver_options.σk = σk - Complex_hist[k] = iter - - xkn .= xk .+ s - fkn = obj(f, xkn) - hkn = h(xkn[selected]) - hkn == -Inf && error("nonsmooth term is not proper") - mks = mk(s) - - fhmax = Mmonotone > 1 ? maximum(FHobj_hist) : fk + hk - Δobj = fhmax - (fkn + hkn) + max(1, abs(fhmax)) * 10 * eps() - Δmod = fhmax - (fk + mks) + max(1, abs(fhmax)) * 10 * eps() - ξ = hk - mks + max(1, abs(hk)) * 10 * eps() - - if (ξ ≤ 0 || isnan(ξ)) - error("R2N: failed to compute a step: ξ = $ξ") - end - - ρk = Δobj / Δmod - - R2N_stat = (η2 ≤ ρk < Inf) ? "↘" : (ρk < η1 ? "↗" : "=") - - if (verbose > 0) && ((k % ptf == 0) || (k == 1)) - #! format: off - @info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8.1e %7.1e %7.1e %7.1e %7.1e %1s" k iter fk hk sqrt_ξ1_νInv sqrt(ξ1) ρk σk norm(xk) norm(s) λmax R2N_stat - #! format: off - end - - if η2 ≤ ρk < Inf - σk = max(σk/γ, σmin) - end - - if η1 ≤ ρk < Inf - xk .= xkn - has_bounds(f) && set_bounds!(ψ, l_bound - xk, u_bound - xk) - - #update functions - fk = fkn - hk = hkn - - # update gradient & Hessian - shift!(ψ, xk) - ∇fk = grad(f, xk) - if quasiNewtTest - push!(f, s, ∇fk - ∇fk⁻) - end - Bk = hess_op(f, xk) - λmax, found_λ = opnorm(Bk) - found_λ || error("operator norm computation failed") - ∇fk⁻ .= ∇fk - end - - if ρk < η1 || ρk == Inf - σk = σk * γ - end - ν = θ / (σk + λmax) - tired = k ≥ maxIter || elapsed_time > maxTime - end - - if verbose > 0 - if k == 1 - @info @sprintf "%6d %8s %8.1e %8.1e" k "" fk hk - elseif optimal - #! format: off - @info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8s %7.1e %7.1e %7.1e %7.1e" k 1 fk hk sqrt_ξ1_νInv sqrt(ξ1) "" σk norm(xk) norm(s) λmax - #! format: on - @info "R2N: terminating with √(ξ1/ν) = $(sqrt_ξ1_νInv)" - end - end - - status = if optimal - :first_order - elseif elapsed_time > maxTime - :max_time - elseif tired - :max_iter - else - :exception - end - - stats = GenericExecutionStats(f) - set_status!(stats, status) - set_solution!(stats, xk) - set_objective!(stats, fk + hk) - set_residuals!(stats, zero(eltype(xk)), sqrt_ξ1_νInv) - set_iter!(stats, k) - set_time!(stats, elapsed_time) - set_solver_specific!(stats, :sigma, σk) - set_solver_specific!(stats, :Fhist, Fobj_hist[1:k]) - set_solver_specific!(stats, :Hhist, Hobj_hist[1:k]) - set_solver_specific!(stats, :NonSmooth, h) - set_solver_specific!(stats, :SubsolverCounter, Complex_hist[1:k]) - return stats -end diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 7c1e3999..4b1b1454 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -64,7 +64,7 @@ function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Sol x0 ) subpb = RegularizedNLPModel(sub_nlp, ψ) - substats = GenericExecutionStats(subpb) + substats = RegularizedExecutionStats(subpb) subsolver = subsolver(subpb) return R2NSolver{T, typeof(ψ), V, typeof(subsolver), typeof(subpb)}( diff --git a/src/RegularizedOptimization.jl b/src/RegularizedOptimization.jl index 7e243b20..b2fdd4f3 100644 --- a/src/RegularizedOptimization.jl +++ b/src/RegularizedOptimization.jl @@ -24,9 +24,6 @@ include("R2NModel.jl") include("R2N_alg.jl") include("LM_alg.jl") include("LMTR_alg.jl") -include("R2DH.jl") -include("R2N.jl") - include("AL_alg.jl") end # module RegularizedOptimization diff --git a/test/runtests.jl b/test/runtests.jl index 8bd97182..1e8b28d4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -134,7 +134,7 @@ for (h, h_name) ∈ ((NormL1(λ), "l1"),) end end -R2N_R2DH(args...; kwargs...) = R2N(args...; subsolver = R2DH, kwargs...) +R2N_R2DH(args...; kwargs...) = R2N(args...; subsolver = R2DHSolver, kwargs...) for (mod, mod_name) ∈ ( (SpectralGradientModel, "spg"), (DiagonalPSBModel, "psb"), @@ -153,14 +153,7 @@ for (mod, mod_name) ∈ ( out = solver(mod(bpdn), h, options, x0 = x0) @test typeof(out.solution) == typeof(bpdn.meta.x0) @test length(out.solution) == bpdn.meta.nvar - @test typeof(out.solver_specific[:Fhist]) == typeof(out.solution) - @test typeof(out.solver_specific[:Hhist]) == typeof(out.solution) - @test typeof(out.solver_specific[:SubsolverCounter]) == Array{Int, 1} @test typeof(out.dual_feas) == eltype(out.solution) - @test length(out.solver_specific[:Fhist]) == length(out.solver_specific[:Hhist]) - @test length(out.solver_specific[:Fhist]) == length(out.solver_specific[:SubsolverCounter]) - @test obj(bpdn, out.solution) == out.solver_specific[:Fhist][end] - @test h(out.solution) == out.solver_specific[:Hhist][end] @test out.status == :first_order end end diff --git a/test/test_allocs.jl b/test/test_allocs.jl index 19da5ad5..76783136 100644 --- a/test/test_allocs.jl +++ b/test/test_allocs.jl @@ -41,12 +41,15 @@ end # Test non allocating solve! @testset "allocs" begin - for (h, h_name) ∈ ((NormL0(λ), "l0"), (NormL1(λ), "l1")) - for solver ∈ (:R2Solver,) - reg_nlp = RegularizedNLPModel(bpdn, h) - solver = eval(solver)(reg_nlp) - stats = RegularizedExecutionStats(reg_nlp) - @test @wrappedallocs(solve!(solver, reg_nlp, stats)) == 0 + for (h, h_name) ∈ ((NormL0(λ), "l0"), ) + for (solver, solver_name) ∈ ((:R2Solver, "R2"), (:R2DHSolver, "R2DH"), (:R2NSolver, "R2N")) + @testset "$(solver_name)" begin + reg_nlp = RegularizedNLPModel(LBFGSModel(bpdn), h) + solver = eval(solver)(reg_nlp) + stats = RegularizedExecutionStats(reg_nlp) + @test @wrappedallocs(solve!(solver, reg_nlp, stats, ν = 1.0, atol = 1e-6, rtol = 1e-6)) == 0 + @test stats.status == :first_order + end end end end From d279296b302f611525303a07e4f0a3012a0aecff Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 13:26:14 -0500 Subject: [PATCH 16/48] solve R2DH signature, allow for AbstractNLPModel --- src/R2DH_alg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index c8dcf49f..f57e80b6 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -141,7 +141,7 @@ Notably, you can access, and modify, the following: - `stats.elapsed_time`: elapsed time in seconds. """ function R2DH( - nlp::AbstractDiagonalQNModel{T, V}, + nlp::AbstractNLPModel{T, V}, h, options::ROSolverOptions{T}; kwargs..., From b41cc81da57080d01e679aa2ce07d6faffb0daa1 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 13:48:47 -0500 Subject: [PATCH 17/48] solve verbose bug in R2DH --- src/R2DH_alg.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index f57e80b6..982ed796 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -449,7 +449,6 @@ function SolverCore.solve!( @info log_row( Any[ stats.iter, - 0, fk, hk, sqrt_ξ_νInv, From 808f8d41e6221c282249f04955d37377f95370fa Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 14:01:48 -0500 Subject: [PATCH 18/48] Resolve verbose bug in R2N --- src/R2N_alg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 4b1b1454..9c1cc08f 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -402,7 +402,7 @@ function SolverCore.solve!( norm(xk), norm(s), λmax, - (η2 ≤ ρk < Inf) ? "↗" : (ρk < η1 ? "↘" : "="), + (η2 ≤ ρk < Inf) ? "↘" : (ρk < η1 ? "↗" : "="), ], colsep = 1, ) From 279ad4cb1a36e542f51089a5c14b55e8f6d9c430 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 15:16:23 -0500 Subject: [PATCH 19/48] default m_monotone to 6 in R2DH --- src/R2DH_alg.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index 982ed796..b184b588 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -25,7 +25,7 @@ mutable struct R2DHSolver{ m_fh_hist::V end -function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int = 1) where{T, V} +function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int = 6) where{T, V} x0 = reg_nlp.model.meta.x0 l_bound = reg_nlp.model.meta.lvar u_bound = reg_nlp.model.meta.uvar @@ -91,7 +91,7 @@ where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀDₖs is a quadratic appr For advanced usage, first define a solver "R2DHSolver" to preallocate the memory used in the algorithm, and then call `solve!`: - solver = R2DHSolver(reg_nlp; m_monotone = 1) + solver = R2DHSolver(reg_nlp; m_monotone = 6) solve!(solver, reg_nlp) stats = RegularizedExecutionStats(reg_nlp) @@ -116,7 +116,7 @@ For advanced usage, first define a solver "R2DHSolver" to preallocate the memory - `ν::T = eps(T)^(1 / 5)`: multiplicative inverse of the regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. - `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. -- `m_monotone::Int = 1`: monotoneness parameter. By default, R2DH is monotone but the non-monotone variant can be used with `m_monotone > 1` +- `m_monotone::Int = 6`: monotoneness parameter. By default, R2DH is non-monotone but the monotone variant can be used with `m_monotone = 1` The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. From f9075d3e54a97f3f1ffd5ea382468f1bf30c94d5 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 17:18:58 -0500 Subject: [PATCH 20/48] solve m-monotoneness issues - R2N w R2DH give the same result now --- src/R2DH_alg.jl | 6 +++--- src/R2N_alg.jl | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index b184b588..c875c2df 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -222,7 +222,7 @@ function SolverCore.solve!( ψ = solver.ψ xkn = solver.xkn s = solver.s - m_fh_hist = solver.m_fh_hist + m_fh_hist = solver.m_fh_hist .= T(-Inf) has_bnds = solver.has_bnds if has_bnds @@ -279,7 +279,6 @@ function SolverCore.solve!( sqrt_ξ_νInv = one(T) @. mν∇fk = -ν₁ * ∇fk - m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) set_iter!(stats, 0) start_time = time() @@ -287,6 +286,7 @@ function SolverCore.solve!( set_objective!(stats, fk + hk) set_solver_specific!(stats, :smooth_obj, fk) set_solver_specific!(stats, :nonsmooth_obj, hk) + m_monotone > 1 && (m_fh_hist[(stats.iter)%(m_monotone - 1) + 1] = fk + hk) φ(d) = begin result = zero(T) @@ -404,7 +404,7 @@ function SolverCore.solve!( ν₁ = 1 / ((DNorm + σk) * (1 + θ)) @. mν∇fk = -ν₁ * ∇fk - m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) + m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) mks = mk(s) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index 9c1cc08f..a8a4017f 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -236,7 +236,7 @@ function SolverCore.solve!( xkn = solver.xkn s = solver.s s1 = solver.s1 - m_fh_hist = solver.m_fh_hist + m_fh_hist = solver.m_fh_hist .= T(-Inf) has_bnds = solver.has_bnds if has_bnds @@ -299,7 +299,6 @@ function SolverCore.solve!( sqrt_ξ1_νInv = one(T) @. mν∇fk = -ν₁ * ∇fk - m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) set_iter!(stats, 0) start_time = time() @@ -307,6 +306,7 @@ function SolverCore.solve!( set_objective!(stats, fk + hk) set_solver_specific!(stats, :smooth_obj, fk) set_solver_specific!(stats, :nonsmooth_obj, hk) + m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) φ1 = let ∇fk = ∇fk d -> dot(∇fk, d) @@ -361,7 +361,7 @@ function SolverCore.solve!( solver.substats; x = s1, atol = sub_atol, - ν = ν₁, + ν = 1/σk, kwargs... ) @@ -445,7 +445,7 @@ function SolverCore.solve!( end ν₁ = 1 / ((λmax + σk) * (1 + θ)) - m_monotone > 1 && (m_fh_hist[mod(stats.iter+1, m_monotone - 1) + 1] = fk + hk) + m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) set_objective!(stats, fk + hk) set_solver_specific!(stats, :smooth_obj, fk) From 896432eb2b2a7a5e192190b9d59cc4663df225b4 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 17:40:26 -0500 Subject: [PATCH 21/48] force m_monotone = 6 as default for R2DH --- src/R2DH_alg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/R2DH_alg.jl b/src/R2DH_alg.jl index c875c2df..123695d5 100644 --- a/src/R2DH_alg.jl +++ b/src/R2DH_alg.jl @@ -174,7 +174,7 @@ function R2DH( kwargs... ) where{T, V} kwargs_dict = Dict(kwargs...) - m_monotone = pop!(kwargs_dict, :m_monotone, 1) + m_monotone = pop!(kwargs_dict, :m_monotone, 6) solver = R2DHSolver(reg_nlp, m_monotone = m_monotone) stats = GenericExecutionStats(reg_nlp.model) solve!(solver, reg_nlp, stats; kwargs_dict...) From 026ada5cf6c8f9fb1fd70cbbe9c3ad1bc16cd78d Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 20 Jan 2025 17:54:12 -0500 Subject: [PATCH 22/48] add nu_sub variable for R2N --- src/R2N_alg.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/R2N_alg.jl b/src/R2N_alg.jl index a8a4017f..c49706b1 100644 --- a/src/R2N_alg.jl +++ b/src/R2N_alg.jl @@ -296,6 +296,8 @@ function SolverCore.solve!( end ν₁ = 1 / ((λmax + σk) * (1 + θ)) + ν_sub = ν₁ + sqrt_ξ1_νInv = one(T) @. mν∇fk = -ν₁ * ∇fk @@ -355,13 +357,14 @@ function SolverCore.solve!( solver.subpb.model.σ = σk isa(solver.subsolver, R2DHSolver) && (solver.subsolver.D.d[1] = 1/ν₁) + ν_sub = isa(solver.subsolver, R2DHSolver) ? 1/σk : ν₁ solve!( solver.subsolver, solver.subpb, solver.substats; x = s1, atol = sub_atol, - ν = 1/σk, + ν = ν_sub, kwargs... ) From 414475a4b51a620eb0dcb9cf72c468658068485e Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 21 Jan 2025 09:30:36 -0500 Subject: [PATCH 23/48] replace *_alg.jl --- src/{R2DH_alg.jl => R2DH.jl} | 0 src/{R2N_alg.jl => R2N.jl} | 0 src/RegularizedOptimization.jl | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/{R2DH_alg.jl => R2DH.jl} (100%) rename src/{R2N_alg.jl => R2N.jl} (100%) diff --git a/src/R2DH_alg.jl b/src/R2DH.jl similarity index 100% rename from src/R2DH_alg.jl rename to src/R2DH.jl diff --git a/src/R2N_alg.jl b/src/R2N.jl similarity index 100% rename from src/R2N_alg.jl rename to src/R2N.jl diff --git a/src/RegularizedOptimization.jl b/src/RegularizedOptimization.jl index b2fdd4f3..8bbea397 100644 --- a/src/RegularizedOptimization.jl +++ b/src/RegularizedOptimization.jl @@ -19,9 +19,9 @@ include("splitting.jl") include("TR_alg.jl") include("TRDH_alg.jl") include("R2_alg.jl") -include("R2DH_alg.jl") +include("R2DH.jl") include("R2NModel.jl") -include("R2N_alg.jl") +include("R2N.jl") include("LM_alg.jl") include("LMTR_alg.jl") include("AL_alg.jl") From d70e5df7fd3cc27afbdd88f7a03e16a90c0733ba Mon Sep 17 00:00:00 2001 From: Maxence Gollier <134112149+MaxenceGollier@users.noreply.github.com> Date: Sat, 8 Feb 2025 21:01:59 -0500 Subject: [PATCH 24/48] Apply suggestions from code review Co-authored-by: Dominique Co-authored-by: MohamedLaghdafHABIBOULLAH <81633807+MohamedLaghdafHABIBOULLAH@users.noreply.github.com> --- src/R2N.jl | 15 +++++++-------- src/R2NModel.jl | 8 ++++---- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/R2N.jl b/src/R2N.jl index c49706b1..ae576b4c 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -112,7 +112,6 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory solve!(solver, reg_nlp) stats = RegularizedExecutionStats(reg_nlp) - solver = R2NSolver(reg_nlp) solve!(solver, reg_nlp, stats) # Arguments @@ -128,12 +127,12 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory - `max_iter::Int = 10000`: maximum number of iterations; - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration; - `σmin::T = eps(T)`: minimum value of the regularization parameter; -- `η1::T = √√eps(T)`: very successful iteration threshold; -- `η2::T = T(0.9)`: successful iteration threshold; +- `η1::T = √√eps(T)`: successful iteration threshold; +- `η2::T = T(0.9)`: very successful iteration threshold; - `ν::T = eps(T)^(1 / 5)`: multiplicative inverse of the regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. - `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. -- `m_monotone::Int = 1`: monotoneness parameter. By default, R2DH is monotone but the non-monotone variant can be used with `m_monotone > 1` +- `m_monotone::Int = 1`: monotonicity parameter. By default, R2DH is monotone but the non-monotone variant will be used if `m_monotone > 1` The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. @@ -154,7 +153,7 @@ Notably, you can access, and modify, the following: - `stats.objective`: current objective function value; - `stats.solver_specific[:smooth_obj]`: current value of the smooth part of the objective function - `stats.solver_specific[:nonsmooth_obj]`: current value of the nonsmooth part of the objective function - - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything will stop the algorithm, but you should use `:user` to properly indicate the intention. + - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything other than `:unknown` will stop the algorithm, but you should use `:user` to properly indicate the intention. - `stats.elapsed_time`: elapsed time in seconds. """ function R2N( @@ -291,7 +290,7 @@ function SolverCore.solve!( try λmax = opnorm(solver.subpb.model.B) # TODO: This allocates; see utils.jl - catch LAPACKException # This should be removed ASAP; see PR #159. + catch LAPACKException # FIXME: This should be removed ASAP; see PR #159. λmax = opnorm(Matrix(solver.subpb.model.B)) end @@ -366,7 +365,7 @@ function SolverCore.solve!( atol = sub_atol, ν = ν_sub, kwargs... - ) + ) s .= solver.substats.solution @@ -422,7 +421,7 @@ function SolverCore.solve!( hk = hkn shift!(ψ, xk) - ∇fk = grad!(nlp, xk, ∇fk) + grad!(nlp, xk, ∇fk) if quasiNewtTest @. ∇fk⁻ = ∇fk - ∇fk⁻ diff --git a/src/R2NModel.jl b/src/R2NModel.jl index f5f5eaf0..d514ee3b 100644 --- a/src/R2NModel.jl +++ b/src/R2NModel.jl @@ -9,10 +9,10 @@ Given the unconstrained optimization problem: ``` this model represents the smooth R2N subproblem: ```math -\min_s \ \tfrac{1}{2} s^T B s + ∇f^T s + \tfrac{1}{2} σ\|s\|^2 +\min_s \ ∇f^T s + \tfrac{1}{2} s^T B s + \tfrac{1}{2} σ \|s\|^2 ``` -where `B` represents either an approximation of the Hessian of `f` or the Hessian itself and `∇f` represents the gradient of `f` on `x0`. -`σ > 0` is a regularization parameter and `v` is a vector of the same size as `x0` used for computations +where `B` is either an approximation of the Hessian of `f` or the Hessian itself and `∇f` represents the gradient of `f` at `x0`. +`σ > 0` is a regularization parameter and `v` is a vector of the same size as `x0` used for intermediary computations. """ mutable struct R2NModel{T <: Real, V <: AbstractVector{T}, G <: AbstractLinearOperator{T}} <: AbstractNLPModel{T, V} B :: G @@ -23,7 +23,7 @@ mutable struct R2NModel{T <: Real, V <: AbstractVector{T}, G <: AbstractLinearOp counters::Counters end - function R2NModel( +function R2NModel( B :: G, ∇f :: V, v :: V, From 2f461f70ac3169d90d9b94adff23ff1bb065b58a Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sat, 8 Feb 2025 21:53:23 -0500 Subject: [PATCH 25/48] apply suggestions in R2N & R2DH --- src/R2DH.jl | 8 +++----- src/R2N.jl | 26 ++++++++------------------ 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index 123695d5..75a7c8fe 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -79,7 +79,7 @@ A second-order quadratic regularization method for the problem min f(x) + h(x) -where f: ℝⁿ → ℝ has a Lipschitz-continuous gradient, and h: ℝⁿ → ℝ is +where f: ℝⁿ → ℝ is C¹, and h: ℝⁿ → ℝ is lower semi-continuous, proper and prox-bounded. About each iterate xₖ, a step sₖ is computed as a solution of @@ -113,7 +113,7 @@ For advanced usage, first define a solver "R2DHSolver" to preallocate the memory - `σmin::T = eps(T)`: minimum value of the regularization parameter; - `η1::T = √√eps(T)`: very successful iteration threshold; - `η2::T = T(0.9)`: successful iteration threshold; -- `ν::T = eps(T)^(1 / 5)`: multiplicative inverse of the regularization parameter: ν = 1/σ; +- `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. - `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. - `m_monotone::Int = 6`: monotoneness parameter. By default, R2DH is non-monotone but the monotone variant can be used with `m_monotone = 1` @@ -319,7 +319,6 @@ function SolverCore.solve!( error("R2DH: prox-gradient step should produce a decrease but ξ = $(ξ)") atol += rtol * sqrt_ξ_νInv # make stopping test absolute and relative - set_solver_specific!(stats, :xi, sqrt_ξ_νInv) set_status!( stats, get_status( @@ -343,7 +342,6 @@ function SolverCore.solve!( xkn .= xk .+ s fkn = obj(nlp, xkn) hkn = @views h(xkn[selected]) - improper = (hkn == -Inf) fhmax = m_monotone > 1 ? maximum(m_fh_hist) : fk + hk Δobj = fhmax - (fkn + hkn) + max(1, abs(fhmax)) * 10 * eps() @@ -425,7 +423,6 @@ function SolverCore.solve!( (ξ < 0 && sqrt_ξ_νInv > neg_tol) && error("R2DH: prox-gradient step should produce a decrease but ξ = $(ξ)") - set_solver_specific!(stats, :xi, sqrt_ξ_νInv) set_status!( stats, get_status( @@ -464,5 +461,6 @@ function SolverCore.solve!( end set_solution!(stats,xk) + set_residuals!(stats, zero(eltype(xk)), sqrt_ξ_νInv) return stats end diff --git a/src/R2N.jl b/src/R2N.jl index ae576b4c..35bd59e5 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -96,7 +96,7 @@ A second-order quadratic regularization method for the problem min f(x) + h(x) -where f: ℝⁿ → ℝ has a Lipschitz-continuous gradient, and h: ℝⁿ → ℝ is +where f: ℝⁿ → ℝ is C¹, and h: ℝⁿ → ℝ is lower semi-continuous, proper and prox-bounded. About each iterate xₖ, a step sₖ is computed as a solution of @@ -129,7 +129,7 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory - `σmin::T = eps(T)`: minimum value of the regularization parameter; - `η1::T = √√eps(T)`: successful iteration threshold; - `η2::T = T(0.9)`: very successful iteration threshold; -- `ν::T = eps(T)^(1 / 5)`: multiplicative inverse of the regularization parameter: ν = 1/σ; +- `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. - `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. - `m_monotone::Int = 1`: monotonicity parameter. By default, R2DH is monotone but the non-monotone variant will be used if `m_monotone > 1` @@ -288,11 +288,7 @@ function SolverCore.solve!( λmax::T = T(1) solver.subpb.model.B = hess_op(nlp, xk) - try - λmax = opnorm(solver.subpb.model.B) # TODO: This allocates; see utils.jl - catch LAPACKException # FIXME: This should be removed ASAP; see PR #159. - λmax = opnorm(Matrix(solver.subpb.model.B)) - end + λmax = opnorm(solver.subpb.model.B) ν₁ = 1 / ((λmax + σk) * (1 + θ)) ν_sub = ν₁ @@ -310,15 +306,15 @@ function SolverCore.solve!( m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) φ1 = let ∇fk = ∇fk - d -> dot(∇fk, d) + d -> dot(∇fk, d) end mk1 = let ψ = ψ - d -> φ1(d) + ψ(d)::T + d -> φ1(d) + ψ(d)::T end mk = let ψ = ψ, solver = solver - d -> obj(solver.subpb.model, d) + ψ(d)::T + d -> obj(solver.subpb.model, d) + ψ(d)::T end prox!(s1, ψ, mν∇fk, ν₁) @@ -331,7 +327,6 @@ function SolverCore.solve!( error("R2N: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") atol += rtol * sqrt_ξ1_νInv # make stopping test absolute and relative - set_solver_specific!(stats, :xi, sqrt_ξ1_νInv) set_status!( stats, get_status( @@ -376,7 +371,6 @@ function SolverCore.solve!( xkn .= xk .+ s fkn = obj(nlp, xkn) hkn = @views h(xkn[selected]) - hkn == -Inf && error("nonsmooth term is not proper") mks = mk(s) fhmax = m_monotone > 1 ? maximum(m_fh_hist) : fk + hk @@ -429,11 +423,7 @@ function SolverCore.solve!( end solver.subpb.model.B = hess_op(nlp, xk) - try - λmax = opnorm(solver.subpb.model.B) - catch LAPACKException - λmax = opnorm(Matrix(solver.subpb.model.B)) - end + λmax = opnorm(solver.subpb.model.B) ∇fk⁻ .= ∇fk end @@ -466,7 +456,6 @@ function SolverCore.solve!( (ξ1 < 0 && sqrt_ξ1_νInv > neg_tol) && error("R2N: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") - set_solver_specific!(stats, :xi, sqrt_ξ1_νInv) set_status!( stats, get_status( @@ -507,5 +496,6 @@ function SolverCore.solve!( end set_solution!(stats, xk) + set_residuals!(stats, zero(eltype(xk)), sqrt_ξ1_νInv) return stats end \ No newline at end of file From d293358733c2142c1fd4edd0bb7c09531ee4d458 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sat, 8 Feb 2025 22:22:13 -0500 Subject: [PATCH 26/48] apply suggestions in R2NModel --- src/R2N.jl | 1 - src/R2NModel.jl | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/R2N.jl b/src/R2N.jl index 35bd59e5..9dee167b 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -59,7 +59,6 @@ function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Sol sub_nlp = R2NModel( Bk, ∇fk, - v, T(1), x0 ) diff --git a/src/R2NModel.jl b/src/R2NModel.jl index d514ee3b..3b58b70a 100644 --- a/src/R2NModel.jl +++ b/src/R2NModel.jl @@ -26,14 +26,15 @@ end function R2NModel( B :: G, ∇f :: V, - v :: V, σ :: T, x0 :: V ) where{T, V, G} + @assert length(x0) == length(∇f) meta = NLPModelMeta( length(∇f), x0 = x0, # Perhaps we should add lvar and uvar as well here. ) + v = similar(x0) return R2NModel( B :: G, ∇f :: V, @@ -59,8 +60,4 @@ function NLPModels.grad!(nlp::R2NModel, x::AbstractVector, g::AbstractVector) g .+= nlp.∇f g .+= nlp.σ .* x return g -end - -function NLPModels.push!(nlp::R2NModel, s::AbstractVector, y::AbstractVector) - push!(nlp.B, s, y) end \ No newline at end of file From 11982ec76c282fe28b41a38310795ec023264aad Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 12 Feb 2025 11:49:19 -0500 Subject: [PATCH 27/48] change theta for better readibility --- src/R2DH.jl | 15 ++++++++++----- src/R2N.jl | 8 ++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index 75a7c8fe..9dc2e1c2 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -199,7 +199,7 @@ function SolverCore.solve!( η2::T = T(0.9), ν::T = eps(T)^(1 / 5), γ::T = T(3), - θ::T = eps(T)^(1 / 5), + θ::T = 1/(1 + eps(T)^(1 / 5)), ) where{T, V} reset!(stats) @@ -275,7 +275,7 @@ function SolverCore.solve!( @. dkσk = D.d .+ σk DNorm = norm(D.d, Inf) - ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + ν₁ = θ / (DNorm + σk) sqrt_ξ_νInv = one(T) @. mν∇fk = -ν₁ * ∇fk @@ -306,7 +306,7 @@ function SolverCore.solve!( σk = σk * γ dkσk .= D.d .+ σk DNorm = norm(D.d, Inf) - ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + ν₁ = θ / (DNorm + σk) @. mν∇fk = -ν₁ * ∇fk spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) mks = mk(s) @@ -399,7 +399,7 @@ function SolverCore.solve!( @. dkσk = D.d .+ σk DNorm = norm(D.d, Inf) - ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + ν₁ = θ / (DNorm + σk) @. mν∇fk = -ν₁ * ∇fk m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) @@ -411,7 +411,7 @@ function SolverCore.solve!( σk = σk * γ dkσk .= D.d .+ σk DNorm = norm(D.d, Inf) - ν₁ = 1 / ((DNorm + σk) * (1 + θ)) + ν₁ = θ / (DNorm + σk) @. mν∇fk = -ν₁ * ∇fk spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) mks = mk(s) @@ -464,3 +464,8 @@ function SolverCore.solve!( set_residuals!(stats, zero(eltype(xk)), sqrt_ξ_νInv) return stats end + + +#theta 0.000740095979741405 +#nu1 0.0003700479898707025 +#original 0.4996302256786179 \ No newline at end of file diff --git a/src/R2N.jl b/src/R2N.jl index 9dee167b..b9cca5f3 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -130,7 +130,7 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory - `η2::T = T(0.9)`: very successful iteration threshold; - `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. -- `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. +- `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model. - `m_monotone::Int = 1`: monotonicity parameter. By default, R2DH is monotone but the non-monotone variant will be used if `m_monotone > 1` The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. @@ -212,7 +212,7 @@ function SolverCore.solve!( ν::T = eps(T)^(1 / 5), γ::T = T(3), β::T = 1 / eps(T), - θ::T = eps(T)^(1 / 5), + θ::T = 1/(1+eps(T)^(1 / 5)), kwargs... ) where{T, V, G} reset!(stats) @@ -289,7 +289,7 @@ function SolverCore.solve!( λmax = opnorm(solver.subpb.model.B) - ν₁ = 1 / ((λmax + σk) * (1 + θ)) + ν₁ = θ / (λmax + σk) ν_sub = ν₁ sqrt_ξ1_νInv = one(T) @@ -435,7 +435,7 @@ function SolverCore.solve!( σk = σk * γ end - ν₁ = 1 / ((λmax + σk) * (1 + θ)) + ν₁ = θ / (λmax + σk) m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) set_objective!(stats, fk + hk) From 509264ad6c7c67da883fc0313691fe12bbee381c Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 12 Feb 2025 12:06:35 -0500 Subject: [PATCH 28/48] add improper check and return in R2N & R2DH --- src/R2DH.jl | 4 +++- src/R2N.jl | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index 9dc2e1c2..c9710047 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -115,7 +115,7 @@ For advanced usage, first define a solver "R2DHSolver" to preallocate the memory - `η2::T = T(0.9)`: successful iteration threshold; - `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. -- `θ::T = eps(T)^(1/5)`: is the model decrease fraction with respect to the decrease of the Cauchy model. +- `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model. - `m_monotone::Int = 6`: monotoneness parameter. By default, R2DH is non-monotone but the monotone variant can be used with `m_monotone = 1` The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. @@ -245,6 +245,8 @@ function SolverCore.solve!( verbose > 0 && @debug "R2DH: found point where h has value" hk end improper = (hk == -Inf) + improper == true && @warn "R2DH: Improper term detected" + improper == true && return stats if verbose > 0 @info log_header( diff --git a/src/R2N.jl b/src/R2N.jl index b9cca5f3..e4af04c5 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -256,6 +256,8 @@ function SolverCore.solve!( verbose > 0 && @debug "R2N: found point where h has value" hk end improper = (hk == -Inf) + improper == true && @warn "R2N: Improper term detected" + improper == true && return stats if verbose > 0 @info log_header( From 3b046b353cae4c2616770b2465374a3f44816b3a Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 12 Feb 2025 12:06:48 -0500 Subject: [PATCH 29/48] remove R2N alloc test - add FIXME --- test/test_allocs.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_allocs.jl b/test/test_allocs.jl index 76783136..0489ca56 100644 --- a/test/test_allocs.jl +++ b/test/test_allocs.jl @@ -44,6 +44,7 @@ end for (h, h_name) ∈ ((NormL0(λ), "l0"), ) for (solver, solver_name) ∈ ((:R2Solver, "R2"), (:R2DHSolver, "R2DH"), (:R2NSolver, "R2N")) @testset "$(solver_name)" begin + solver_name == "R2N" && continue #FIXME reg_nlp = RegularizedNLPModel(LBFGSModel(bpdn), h) solver = eval(solver)(reg_nlp) stats = RegularizedExecutionStats(reg_nlp) From 0c994e8cfa79098ae93a07fdf80eeb56dca73b0e Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 12 Feb 2025 13:58:31 -0500 Subject: [PATCH 30/48] remove mks == -infty test in R2DH --- src/R2DH.jl | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index c9710047..bd196317 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -304,15 +304,6 @@ function SolverCore.solve!( spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) mks = mk(s) - while mks == -Inf #TODO add test coverage for this - σk = σk * γ - dkσk .= D.d .+ σk - DNorm = norm(D.d, Inf) - ν₁ = θ / (DNorm + σk) - @. mν∇fk = -ν₁ * ∇fk - spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) - mks = mk(s) - end ξ = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) @@ -409,16 +400,6 @@ function SolverCore.solve!( spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) mks = mk(s) - while mks == -Inf #TODO add test coverage for this - σk = σk * γ - dkσk .= D.d .+ σk - DNorm = norm(D.d, Inf) - ν₁ = θ / (DNorm + σk) - @. mν∇fk = -ν₁ * ∇fk - spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) - mks = mk(s) - end - ξ = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) solved = (ξ < 0 && sqrt_ξ_νInv ≤ neg_tol) || (ξ ≥ 0 && sqrt_ξ_νInv ≤ atol) From 98eddc43477d6d27f50bf34afbe7606625d11506 Mon Sep 17 00:00:00 2001 From: Maxence Gollier <134112149+MaxenceGollier@users.noreply.github.com> Date: Thu, 20 Feb 2025 14:33:38 -0500 Subject: [PATCH 31/48] Remove dead comments in R2DH --- src/R2DH.jl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index bd196317..8e0a4714 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -446,9 +446,4 @@ function SolverCore.solve!( set_solution!(stats,xk) set_residuals!(stats, zero(eltype(xk)), sqrt_ξ_νInv) return stats -end - - -#theta 0.000740095979741405 -#nu1 0.0003700479898707025 -#original 0.4996302256786179 \ No newline at end of file +end \ No newline at end of file From e2047933c5fa4c083b209f2a3dc5348eec0092f6 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sun, 23 Feb 2025 19:01:58 -0500 Subject: [PATCH 32/48] revert changes to LM & LMTR, revert theta default value --- src/RegularizedOptimization.jl | 4 ++-- src/input_struct.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/RegularizedOptimization.jl b/src/RegularizedOptimization.jl index 8bbea397..01778e86 100644 --- a/src/RegularizedOptimization.jl +++ b/src/RegularizedOptimization.jl @@ -19,11 +19,11 @@ include("splitting.jl") include("TR_alg.jl") include("TRDH_alg.jl") include("R2_alg.jl") +include("LM_alg.jl") +include("LMTR_alg.jl") include("R2DH.jl") include("R2NModel.jl") include("R2N.jl") -include("LM_alg.jl") -include("LMTR_alg.jl") include("AL_alg.jl") end # module RegularizedOptimization diff --git a/src/input_struct.jl b/src/input_struct.jl index 1b8b1581..7242b90a 100644 --- a/src/input_struct.jl +++ b/src/input_struct.jl @@ -34,7 +34,7 @@ mutable struct ROSolverOptions{R} α::R = 1 / eps(R), ν::R = eps(R)^(1 / 5), γ::R = R(3), - θ::R = 1/(1+eps(R)^(1 / 5)), + θ::R = eps(R)^(1 / 5), β::R = 1 / eps(R), reduce_TR::Bool = true, ) where {R <: Real} From 45adfb8f962eb3714c3b764f73e8a6ab88de7ca5 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sun, 23 Feb 2025 19:03:07 -0500 Subject: [PATCH 33/48] fix comment indentation --- Manifest.toml | 959 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 959 insertions(+) create mode 100644 Manifest.toml diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 00000000..56f38586 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,959 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.1" +manifest_format = "2.0" +project_hash = "9fe38f5c99c6c58cf2b2755f50a48fa53f4ecb4e" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + + [deps.AbstractFFTs.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "cd8b948862abee8f3d3e9b73a102a9ca924debb0" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "4.2.0" +weakdeps = ["SparseArrays", "StaticArrays"] + + [deps.Adapt.extensions] + AdaptSparseArraysExt = "SparseArrays" + AdaptStaticArraysExt = "StaticArrays" + +[[deps.AliasTables]] +deps = ["PtrArrays", "Random"] +git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" +uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" +version = "1.1.3" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Arpack]] +deps = ["Arpack_jll", "Libdl", "LinearAlgebra", "Logging"] +git-tree-sha1 = "9b9b347613394885fd1c8c7729bfc60528faa436" +uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97" +version = "0.5.4" + +[[deps.Arpack_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "OpenBLAS_jll", "Pkg"] +git-tree-sha1 = "5ba6c757e8feccf03a1554dfaf3e26b3cfc7fd5e" +uuid = "68821587-b530-5797-8361-c406ea357684" +version = "3.5.1+1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BenchmarkTools]] +deps = ["Compat", "JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"] +git-tree-sha1 = "e38fbc49a620f5d0b660d7f543db1009fe0f8336" +uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +version = "1.6.0" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1b96ea4a01afe0ea4090c5c8039690672dd13f2e" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.9+0" + +[[deps.CodecBzip2]] +deps = ["Bzip2_jll", "TranscodingStreams"] +git-tree-sha1 = "84990fa864b7f2b4901901ca12736e45ee79068c" +uuid = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" +version = "0.8.5" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.8" + +[[deps.ColorSchemes]] +deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] +git-tree-sha1 = "403f2d8e209681fcbd9468a8514efff3ea08452e" +uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +version = "3.29.0" + +[[deps.ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.5" + +[[deps.ColorVectorSpace]] +deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "Requires", "Statistics", "TensorCore"] +git-tree-sha1 = "a1f44953f2382ebb937d60dafbe2deea4bd23249" +uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" +version = "0.10.0" +weakdeps = ["SpecialFunctions"] + + [deps.ColorVectorSpace.extensions] + SpecialFunctionsExt = "SpecialFunctions" + +[[deps.Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.11" + +[[deps.CommonSolve]] +git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c" +uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" +version = "0.2.4" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools"] +git-tree-sha1 = "cda2cfaebb4be89c9084adaca7dd7333369715c5" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.1" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.16.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.ConstructionBase]] +git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.8" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.Contour]] +git-tree-sha1 = "439e35b0b36e2e5881738abc8857bd92ad6ff9a8" +uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" +version = "0.6.3" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.15.1" + +[[deps.Distributions]] +deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] +git-tree-sha1 = "03aa5d44647eaec98e1920635cdfed5d5560a8b9" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.117" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + DistributionsTestExt = "Test" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.FastClosures]] +git-tree-sha1 = "acebe244d53ee1b461970f8910c235b259e772ef" +uuid = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" +version = "0.3.2" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FillArrays]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "6a70198746448456524cb442b8af316927ff3e1a" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.13.0" +weakdeps = ["PDMats", "SparseArrays", "Statistics"] + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStatisticsExt = "Statistics" + +[[deps.FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.5" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "a2df1b776752e3f344e5116c06d75a10436ab853" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.38" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.Graphics]] +deps = ["Colors", "LinearAlgebra", "NaNMath"] +git-tree-sha1 = "a641238db938fff9b2f60d08ed9030387daf428c" +uuid = "a2bd30eb-e257-5431-a919-1863eab51364" +version = "1.1.3" + +[[deps.HypergeometricFunctions]] +deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "2bd56245074fab4015b9174f24ceba8293209053" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.27" + +[[deps.ImageCore]] +deps = ["AbstractFFTs", "Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "Reexport"] +git-tree-sha1 = "db645f20b59f060d8cfae696bc9538d13fd86416" +uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" +version = "0.8.22" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "e2222959fbc6c19554dc15174c81bf7bf3aa691c" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.4" + +[[deps.IterativeSolvers]] +deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] +git-tree-sha1 = "59545b0a2b27208b0650df0a46b8e3019f85055b" +uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" +version = "0.9.4" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.7.0" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "1d322381ef7b087548321d3f878cb4c9bd8f8f9b" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.14.1" + + [deps.JSON3.extensions] + JSON3ArrowExt = ["ArrowTypes"] + + [deps.JSON3.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JSOSolvers]] +deps = ["Krylov", "LinearAlgebra", "LinearOperators", "Logging", "NLPModels", "NLPModelsModifiers", "Printf", "SolverCore", "SolverTools"] +git-tree-sha1 = "67f493f5ad881df690c1d9a9bfb8c7c7eda42fba" +uuid = "10dff2fc-5484-5881-a0e0-c90441020f8a" +version = "0.11.2" + +[[deps.Krylov]] +deps = ["LinearAlgebra", "Printf", "SparseArrays"] +git-tree-sha1 = "b29d37ce30fa401a4563b18880ab91f979a29734" +uuid = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" +version = "0.9.10" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.LinearOperators]] +deps = ["FastClosures", "LinearAlgebra", "Printf", "Requires", "SparseArrays", "TimerOutputs"] +git-tree-sha1 = "3468cfb98c69620005b87f6dbeb6aeb8cffe9465" +repo-rev = "main" +repo-url = "https://github.com/JuliaSmoothOptimizers/LinearOperators.jl.git" +uuid = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125" +version = "2.9.0" + + [deps.LinearOperators.extensions] + LinearOperatorsAMDGPUExt = "AMDGPU" + LinearOperatorsCUDAExt = "CUDA" + LinearOperatorsChainRulesCoreExt = "ChainRulesCore" + LinearOperatorsJLArraysExt = "JLArrays" + LinearOperatorsLDLFactorizationsExt = "LDLFactorizations" + LinearOperatorsMetalExt = "Metal" + + [deps.LinearOperators.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" + LDLFactorizations = "40e66cde-538c-5869-a4ad-c39174c6795b" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.29" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.METIS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "2eefa8baa858871ae7770c98c3c2a7e46daba5b4" +uuid = "d00139f3-1899-568f-a2f0-47f597d42d70" +version = "5.1.3+0" + +[[deps.MacroTools]] +git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.15" + +[[deps.MappedArrays]] +git-tree-sha1 = "2dab0221fe2b0f2cb6754eaa743cc266339f527e" +uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" +version = "0.4.2" + +[[deps.MarchingCubes]] +deps = ["PrecompileTools", "StaticArrays"] +git-tree-sha1 = "0e893025924b6becbae4109f8020ac0e12674b01" +uuid = "299715c1-40a9-479a-aaf9-4a633d36f717" +version = "0.1.11" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MathOptInterface]] +deps = ["BenchmarkTools", "CodecBzip2", "CodecZlib", "DataStructures", "ForwardDiff", "JSON3", "LinearAlgebra", "MutableArithmetics", "NaNMath", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays", "SpecialFunctions", "Test", "Unicode"] +git-tree-sha1 = "cfc8c22e3b0157430fc80eb6ef46546483356ea9" +uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" +version = "1.36.0" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MosaicViews]] +deps = ["MappedArrays", "OffsetArrays", "PaddedViews", "StackViews"] +git-tree-sha1 = "7b86a5d4d70a9f5cdf2dacb3cbe6d251d1a61dbe" +uuid = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" +version = "0.3.4" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.MutableArithmetics]] +deps = ["LinearAlgebra", "SparseArrays", "Test"] +git-tree-sha1 = "491bdcdc943fcbc4c005900d7463c9f216aabf4c" +uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" +version = "1.6.4" + +[[deps.NLPModels]] +deps = ["FastClosures", "LinearAlgebra", "LinearOperators", "Printf", "SparseArrays"] +git-tree-sha1 = "51b458add76a938917772ee661ffb9d59b4c7e5d" +uuid = "a4795742-8479-5a88-8948-cc11e1c8c1a6" +version = "0.20.0" + +[[deps.NLPModelsModifiers]] +deps = ["FastClosures", "LinearAlgebra", "LinearOperators", "NLPModels", "Printf", "SparseArrays"] +git-tree-sha1 = "a80505adbe42104cbbe9674591a5ccd9e9c2dfda" +uuid = "e01155f1-5c6f-4375-a9d8-616dd036575f" +version = "0.7.2" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "cc0a5deefdb12ab3a096f00a6d42133af4560d71" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.1.2" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.Noise]] +deps = ["ImageCore", "PoissonRandom", "Random"] +git-tree-sha1 = "42224fd87de5c50a593bbf1bce16c67b1d65da88" +uuid = "81d43f40-5267-43b7-ae1c-8b967f377efa" +version = "0.2.2" + +[[deps.OSQP]] +deps = ["Libdl", "LinearAlgebra", "MathOptInterface", "OSQP_jll", "SparseArrays"] +git-tree-sha1 = "50faf456a64ac1ca097b78bcdf288d94708adcdd" +uuid = "ab2f91bb-94b4-55e3-9ba0-7f65df51de79" +version = "0.8.1" + +[[deps.OSQP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "d0f73698c33e04e557980a06d75c2d82e3f0eb49" +uuid = "9c4f68bf-6205-5545-a508-2878b064d984" +version = "0.600.200+0" + +[[deps.OffsetArrays]] +git-tree-sha1 = "5e1897147d1ff8d98883cda2be2187dcf57d8f0c" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.15.0" +weakdeps = ["Adapt"] + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + +[[deps.OpenBLAS32_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ece4587683695fe4c5f20e990da0ed7e83c351e7" +uuid = "656ef2d0-ae68-5445-9ca0-591084a874a2" +version = "0.3.29+0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.27+1" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+2" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.6+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "cc4054e898b852042d7b503313f7ad03de99c3dd" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.0" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "966b85253e959ea89c53a9abebbf2e964fbf593b" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.32" + +[[deps.PaddedViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "0fac6313486baae819364c52b4f483450a9d793f" +uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" +version = "0.5.12" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.1" + +[[deps.Percival]] +deps = ["JSOSolvers", "Krylov", "LinearAlgebra", "LinearOperators", "Logging", "NLPModels", "NLPModelsModifiers", "SolverCore", "SolverTools"] +git-tree-sha1 = "058d2127b2f9e8d5cd75c6123954abec4c214572" +uuid = "01435c0c-c90d-11e9-3788-63660f8fbccc" +version = "0.7.2" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" + + [deps.Pkg.extensions] + REPLExt = "REPL" + + [deps.Pkg.weakdeps] + REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.PoissonRandom]] +deps = ["Random"] +git-tree-sha1 = "a0f1159c33f846aa77c3f30ebbc69795e5327152" +uuid = "e409e4f3-bfea-5376-8464-e040bb5c01ab" +version = "0.4.4" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Profile]] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +version = "1.11.0" + +[[deps.ProximalCore]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1f9f650b4b7a60533098dc5e864458f0e4a5b926" +uuid = "dc4f5ac2-75d1-4f31-931e-60435d74994b" +version = "0.1.2" + +[[deps.ProximalOperators]] +deps = ["IterativeSolvers", "LinearAlgebra", "OSQP", "ProximalCore", "SparseArrays", "SuiteSparse", "TSVD"] +git-tree-sha1 = "13a384f52be09c6795ab1c3ad71c8a207decb0ba" +uuid = "a725b495-10eb-56fe-b38b-717eba820537" +version = "0.15.3" + +[[deps.PtrArrays]] +git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" +uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" +version = "1.3.0" + +[[deps.QRMumps]] +deps = ["Libdl", "LinearAlgebra", "OpenBLAS32_jll", "Printf", "SparseArrays", "qr_mumps_jll"] +git-tree-sha1 = "dcadd41c716d150699e0694cebe854ca73af5a51" +repo-rev = "golub_riley" +repo-url = "https://github.com/MaxenceGollier/QRMumps.jl.git" +uuid = "422b30a1-cc69-4d85-abe7-cc07b540c444" +version = "0.2.7" +weakdeps = ["SparseMatricesCOO"] + + [deps.QRMumps.extensions] + QRMumpsSparseMatricesCOOExt = "SparseMatricesCOO" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "9da16da70037ba9d701192e27befedefb91ec284" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.11.2" + + [deps.QuadGK.extensions] + QuadGKEnzymeExt = "Enzyme" + + [deps.QuadGK.weakdeps] + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.RegularizedProblems]] +deps = ["Distributions", "LinearAlgebra", "NLPModels", "Noise", "Random", "Requires", "SparseArrays"] +git-tree-sha1 = "ae672e0c4f9c7f83a7470175367ef1c01b559d26" +uuid = "ea076b23-609f-44d2-bb12-a4ae45328278" +version = "0.1.1" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "852bd0f55565a9e973fcfee83a84413270224dc4" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.8.0" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.5.1+0" + +[[deps.Roots]] +deps = ["CommonSolve", "Printf", "Setfield"] +git-tree-sha1 = "838b60ee62bebc794864c880a47e331e00c47505" +uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" +version = "1.4.1" + +[[deps.SCOTCH_jll]] +deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll"] +git-tree-sha1 = "ce0770b4aea5f187e0179844503e0d3061d95ab8" +uuid = "a8d0f55d-b80e-548d-aff6-1a04c175f0f9" +version = "7.0.6+1" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "Requires"] +git-tree-sha1 = "38d88503f695eb0301479bc9b0d4320b378bafe5" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "0.8.2" + +[[deps.ShiftedProximalOperators]] +deps = ["LinearAlgebra", "OpenBLAS32_jll", "ProximalOperators", "QRMumps", "Roots", "SparseArrays", "SparseMatricesCOO", "libblastrampoline_jll"] +git-tree-sha1 = "57b818e1e04638cca9b843201b31e809957c7e7d" +repo-rev = "compositeNormL2" +repo-url = "https://github.com/MaxenceGollier/ShiftedProximalOperators.jl.git" +uuid = "d4fd37fa-580c-4e43-9b30-361c21aae263" +version = "0.2.1" + +[[deps.SolverCore]] +deps = ["LinearAlgebra", "NLPModels", "Printf"] +git-tree-sha1 = "03a1e0d2d39b9ebc9510f2452c0adfbe887b9cb2" +uuid = "ff4d7338-4cf1-434d-91df-b86cb86fb843" +version = "0.3.8" + +[[deps.SolverTools]] +deps = ["LinearAlgebra", "LinearOperators", "NLPModels", "Printf"] +git-tree-sha1 = "a09dc3247756965ef4bd0aafc92be4914303a5eb" +uuid = "b5612192-2639-5dc1-abfe-fbedd65fab29" +version = "0.8.8" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.11.0" + +[[deps.SparseMatricesCOO]] +deps = ["LinearAlgebra", "SparseArrays", "Test", "UnicodePlots"] +git-tree-sha1 = "b5ca894173efe08725d376063cccb5fa2bcb7a5a" +uuid = "fa32481b-f100-4b48-8dc8-c62f61b13870" +version = "0.2.3" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "64cca0c26b4f31ba18f13f6c12af7c85f478cfde" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.5.0" + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + + [deps.SpecialFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + +[[deps.StackViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "46e589465204cd0c08b4bd97385e4fa79a0c770c" +uuid = "cae243ae-269e-4f55-b966-ac2d0dc13c15" +version = "0.1.1" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "e3be13f448a43610f978d29b7adf78c76022467a" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.12" + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.3" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.7.0" + +[[deps.StatsBase]] +deps = ["AliasTables", "DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "29321314c920c26684834965ec2ce0dacc9cf8e5" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.4" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "b423576adc27097764a90e163157bcfc9acf0f46" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.2" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.11.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.7.0+0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TSVD]] +deps = ["Adapt", "LinearAlgebra"] +git-tree-sha1 = "c39caef6bae501e5607a6caf68dd9ac6e8addbcb" +uuid = "9449cd9e-2762-5aa3-a617-5413e99d722e" +version = "0.4.4" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.TensorCore]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6" +uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50" +version = "0.1.1" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "3832505b94c1868baea47764127e6d36b5c9f29e" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.27" + + [deps.TimerOutputs.extensions] + FlameGraphsExt = "FlameGraphs" + + [deps.TimerOutputs.weakdeps] + FlameGraphs = "08572546-2f56-4bcf-ba4e-bab62c3a3f89" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.UnicodePlots]] +deps = ["ColorSchemes", "ColorTypes", "Contour", "Crayons", "Dates", "LinearAlgebra", "MarchingCubes", "NaNMath", "PrecompileTools", "Printf", "SparseArrays", "StaticArrays", "StatsBase"] +git-tree-sha1 = "24c0e2df19eb3f894d28a64e7486926f38de8a49" +uuid = "b8865327-cd53-5732-bb35-84acbb429228" +version = "3.7.2" + + [deps.UnicodePlots.extensions] + FreeTypeExt = ["FileIO", "FreeType"] + ImageInTerminalExt = "ImageInTerminal" + IntervalSetsExt = "IntervalSets" + TermExt = "Term" + UnitfulExt = "Unitful" + + [deps.UnicodePlots.weakdeps] + FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" + FreeType = "b38be410-82b0-50bf-ab77-7b57e271db43" + ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + Term = "22787eb5-b846-44ae-b979-8e399b8463ab" + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "56c6604ec8b2d82cc4cfe01aa03b00426aac7e1f" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.6.4+1" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.11.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" + +[[deps.qr_mumps_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "METIS_jll", "SCOTCH_jll", "SuiteSparse_jll", "libblastrampoline_jll"] +git-tree-sha1 = "875f1858b94ba19ae0b3b571525a3114ecbb3413" +uuid = "e37b5aa0-c611-5f0f-83fb-aee446c0b77e" +version = "3.1.1+0" From 509d6bd29a3f834762501fbfe038e1436518fc9a Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Sun, 23 Feb 2025 19:03:07 -0500 Subject: [PATCH 34/48] fix comment indentation --- Manifest.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Manifest.toml b/Manifest.toml index 56f38586..8f45a7fa 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,11 @@ julia_version = "1.11.1" manifest_format = "2.0" +<<<<<<< HEAD project_hash = "9fe38f5c99c6c58cf2b2755f50a48fa53f4ecb4e" +======= +project_hash = "261150330d6a84c0b2b794132aee3751ae491129" +>>>>>>> 11a0180 (fix comment indentation) [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -39,6 +43,7 @@ version = "1.1.3" uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" version = "1.1.2" +<<<<<<< HEAD [[deps.Arpack]] deps = ["Arpack_jll", "Libdl", "LinearAlgebra", "Logging"] git-tree-sha1 = "9b9b347613394885fd1c8c7729bfc60528faa436" @@ -51,6 +56,8 @@ git-tree-sha1 = "5ba6c757e8feccf03a1554dfaf3e26b3cfc7fd5e" uuid = "68821587-b530-5797-8361-c406ea357684" version = "3.5.1+1" +======= +>>>>>>> 11a0180 (fix comment indentation) [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" @@ -178,12 +185,15 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" +<<<<<<< HEAD [[deps.DelimitedFiles]] deps = ["Mmap"] git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" version = "1.9.1" +======= +>>>>>>> 11a0180 (fix comment indentation) [[deps.DiffResults]] deps = ["StaticArraysCore"] git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" From fde8a826660a43a8044cfe98c8f3788cb6edd291 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 24 Feb 2025 10:35:35 -0500 Subject: [PATCH 35/48] fix type instability in R2NSolver constructor --- src/R2N.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/R2N.jl b/src/R2N.jl index e4af04c5..c1a1393e 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -28,7 +28,7 @@ mutable struct R2NSolver{ substats::GenericExecutionStats{T, V, V, T} end -function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Solver, m_monotone::Int = 1) where {T, V} +function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver::AbstractOptimizationSolver = R2Solver, m_monotone::Int = 1) where {T, V} x0 = reg_nlp.model.meta.x0 l_bound = reg_nlp.model.meta.lvar u_bound = reg_nlp.model.meta.uvar From 8f354d14f9218b6481aff8b48eb52a4c45ad7fd1 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 24 Feb 2025 11:17:19 -0500 Subject: [PATCH 36/48] revert type instability changes in R2N --- src/R2N.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/R2N.jl b/src/R2N.jl index c1a1393e..e4af04c5 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -28,7 +28,7 @@ mutable struct R2NSolver{ substats::GenericExecutionStats{T, V, V, T} end -function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver::AbstractOptimizationSolver = R2Solver, m_monotone::Int = 1) where {T, V} +function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Solver, m_monotone::Int = 1) where {T, V} x0 = reg_nlp.model.meta.x0 l_bound = reg_nlp.model.meta.lvar u_bound = reg_nlp.model.meta.uvar From cb4400f26a537b5f0adb9bfb19148933744f854b Mon Sep 17 00:00:00 2001 From: Maxence Gollier <134112149+MaxenceGollier@users.noreply.github.com> Date: Mon, 3 Mar 2025 10:06:32 -0500 Subject: [PATCH 37/48] Apply suggestions from code review Co-authored-by: Dominique --- src/R2DH.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index 8e0a4714..18f1e7dd 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -79,21 +79,22 @@ A second-order quadratic regularization method for the problem min f(x) + h(x) -where f: ℝⁿ → ℝ is C¹, and h: ℝⁿ → ℝ is -lower semi-continuous, proper and prox-bounded. +where f: ℝⁿ → ℝ is C¹, and h: ℝⁿ → ℝ is lower semi-continuous, proper and prox-bounded. About each iterate xₖ, a step sₖ is computed as a solution of min φ(s; xₖ) + ½ σₖ ‖s‖² + ψ(s; xₖ) -where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀDₖs is a quadratic approximation of f about xₖ, +where φ(s ; xₖ) = f(xₖ) + ∇f(xₖ)ᵀs + ½ sᵀDₖs is a diagonal quadratic approximation of f about xₖ, ψ(s; xₖ) is either h(xₖ + s) or an approximation of h(xₖ + s), ‖⋅‖ is the ℓ₂ norm and σₖ > 0 is the regularization parameter. -For advanced usage, first define a solver "R2DHSolver" to preallocate the memory used in the algorithm, and then call `solve!`: +For advanced usage, first define a solver `R2DHSolver` to preallocate the memory used in the algorithm, and then call `solve!`: solver = R2DHSolver(reg_nlp; m_monotone = 6) solve!(solver, reg_nlp) +or + stats = RegularizedExecutionStats(reg_nlp) solver = R2DHSolver(reg_nlp) solve!(solver, reg_nlp, stats) From 3d92955e834dc1af32fc85907b3469d1b27ee60a Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Mon, 3 Mar 2025 14:43:35 -0500 Subject: [PATCH 38/48] add opnorm with arpack for R2N --- src/R2N.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/R2N.jl b/src/R2N.jl index e4af04c5..5182eb52 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -131,7 +131,7 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory - `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. - `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model. -- `m_monotone::Int = 1`: monotonicity parameter. By default, R2DH is monotone but the non-monotone variant will be used if `m_monotone > 1` +- `m_monotone::Int = 1`: monotonicity parameter. By default, R2N is monotone but the non-monotone variant will be used if `m_monotone > 1` The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. @@ -289,7 +289,8 @@ function SolverCore.solve!( λmax::T = T(1) solver.subpb.model.B = hess_op(nlp, xk) - λmax = opnorm(solver.subpb.model.B) + λmax, found_λ = opnorm(solver.subpb.model.B) + found_λ || error("operator norm computation failed") ν₁ = θ / (λmax + σk) ν_sub = ν₁ @@ -424,8 +425,9 @@ function SolverCore.solve!( end solver.subpb.model.B = hess_op(nlp, xk) - λmax = opnorm(solver.subpb.model.B) - + λmax, found_λ = opnorm(solver.subpb.model.B) + found_λ || error("operator norm computation failed") + ∇fk⁻ .= ∇fk end From b3a9ba696dcd913126df2c2fce7b9e9b21dcfd29 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 5 Mar 2025 13:32:52 -0500 Subject: [PATCH 39/48] add R2DH function for first order models --- src/R2DH.jl | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index 18f1e7dd..b72b891d 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -25,7 +25,7 @@ mutable struct R2DHSolver{ m_fh_hist::V end -function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int = 6) where{T, V} +function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int = 6, D :: Union{Nothing, AbstractDiagonalQuasiNewtonOperator} = nothing) where{T, V} x0 = reg_nlp.model.meta.x0 l_bound = reg_nlp.model.meta.lvar u_bound = reg_nlp.model.meta.uvar @@ -50,7 +50,7 @@ function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int m_fh_hist = fill(T(-Inf), m_monotone - 1) ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) - D = isa(reg_nlp.model, AbstractDiagonalQNModel) ? hess_op(reg_nlp.model, x0) : SpectralGradient(T(1), reg_nlp.model.meta.nvar) + isnothing(D) && (D = isa(reg_nlp.model, AbstractDiagonalQNModel) ? hess_op(reg_nlp.model, x0) : SpectralGradient(T(1), reg_nlp.model.meta.nvar)) return R2DHSolver( xk, @@ -170,13 +170,48 @@ function R2DH( ) end +function R2DH( + f::F, + ∇f!::G, + h::H, + D::DQN, + options::ROSolverOptions{R}, + x0::AbstractVector{R}; + selected::AbstractVector{<:Integer} = 1:length(x0), + kwargs..., +) where {F <: Function, G <: Function, H, R <: Real, DQN <: AbstractDiagonalQuasiNewtonOperator} + nlp = FirstOrderModel(f, ∇f!, x0) + reg_nlp = RegularizedNLPModel(nlp, h, selected) + stats = R2DH( + reg_nlp, + x = x0, + D = D, + atol = options.ϵa, + rtol = options.ϵr, + neg_tol = options.neg_tol, + verbose = options.verbose, + max_iter = options.maxIter, + max_time = options.maxTime, + σmin = options.σmin, + η1 = options.η1, + η2 = options.η2, + ν = options.ν, + γ = options.γ, + θ = options.θ, + kwargs..., + ) + + return stats.solution, stats.iter, nothing +end + function R2DH( reg_nlp::AbstractRegularizedNLPModel{T, V}; kwargs... ) where{T, V} kwargs_dict = Dict(kwargs...) m_monotone = pop!(kwargs_dict, :m_monotone, 6) - solver = R2DHSolver(reg_nlp, m_monotone = m_monotone) + D = pop!(kwargs_dict, :D, nothing) + solver = R2DHSolver(reg_nlp, m_monotone = m_monotone, D = D) stats = GenericExecutionStats(reg_nlp.model) solve!(solver, reg_nlp, stats; kwargs_dict...) return stats From 339a50562505b5aeb5d9a2fd932cc2fd7fdb578f Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Thu, 6 Mar 2025 15:21:55 -0500 Subject: [PATCH 40/48] remove manifest --- Manifest.toml | 969 -------------------------------------------------- 1 file changed, 969 deletions(-) delete mode 100644 Manifest.toml diff --git a/Manifest.toml b/Manifest.toml deleted file mode 100644 index 8f45a7fa..00000000 --- a/Manifest.toml +++ /dev/null @@ -1,969 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.11.1" -manifest_format = "2.0" -<<<<<<< HEAD -project_hash = "9fe38f5c99c6c58cf2b2755f50a48fa53f4ecb4e" -======= -project_hash = "261150330d6a84c0b2b794132aee3751ae491129" ->>>>>>> 11a0180 (fix comment indentation) - -[[deps.AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "1.5.0" - - [deps.AbstractFFTs.extensions] - AbstractFFTsChainRulesCoreExt = "ChainRulesCore" - AbstractFFTsTestExt = "Test" - - [deps.AbstractFFTs.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.Adapt]] -deps = ["LinearAlgebra", "Requires"] -git-tree-sha1 = "cd8b948862abee8f3d3e9b73a102a9ca924debb0" -uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "4.2.0" -weakdeps = ["SparseArrays", "StaticArrays"] - - [deps.Adapt.extensions] - AdaptSparseArraysExt = "SparseArrays" - AdaptStaticArraysExt = "StaticArrays" - -[[deps.AliasTables]] -deps = ["PtrArrays", "Random"] -git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" -uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" -version = "1.1.3" - -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.2" - -<<<<<<< HEAD -[[deps.Arpack]] -deps = ["Arpack_jll", "Libdl", "LinearAlgebra", "Logging"] -git-tree-sha1 = "9b9b347613394885fd1c8c7729bfc60528faa436" -uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97" -version = "0.5.4" - -[[deps.Arpack_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "OpenBLAS_jll", "Pkg"] -git-tree-sha1 = "5ba6c757e8feccf03a1554dfaf3e26b3cfc7fd5e" -uuid = "68821587-b530-5797-8361-c406ea357684" -version = "3.5.1+1" - -======= ->>>>>>> 11a0180 (fix comment indentation) -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -version = "1.11.0" - -[[deps.Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" - -[[deps.BenchmarkTools]] -deps = ["Compat", "JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"] -git-tree-sha1 = "e38fbc49a620f5d0b660d7f543db1009fe0f8336" -uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -version = "1.6.0" - -[[deps.Bzip2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1b96ea4a01afe0ea4090c5c8039690672dd13f2e" -uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" -version = "1.0.9+0" - -[[deps.CodecBzip2]] -deps = ["Bzip2_jll", "TranscodingStreams"] -git-tree-sha1 = "84990fa864b7f2b4901901ca12736e45ee79068c" -uuid = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" -version = "0.8.5" - -[[deps.CodecZlib]] -deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" -uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.8" - -[[deps.ColorSchemes]] -deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] -git-tree-sha1 = "403f2d8e209681fcbd9468a8514efff3ea08452e" -uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" -version = "3.29.0" - -[[deps.ColorTypes]] -deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" -uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.11.5" - -[[deps.ColorVectorSpace]] -deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "Requires", "Statistics", "TensorCore"] -git-tree-sha1 = "a1f44953f2382ebb937d60dafbe2deea4bd23249" -uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" -version = "0.10.0" -weakdeps = ["SpecialFunctions"] - - [deps.ColorVectorSpace.extensions] - SpecialFunctionsExt = "SpecialFunctions" - -[[deps.Colors]] -deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] -git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" -uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.12.11" - -[[deps.CommonSolve]] -git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c" -uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" -version = "0.2.4" - -[[deps.CommonSubexpressions]] -deps = ["MacroTools"] -git-tree-sha1 = "cda2cfaebb4be89c9084adaca7dd7333369715c5" -uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" -version = "0.3.1" - -[[deps.Compat]] -deps = ["TOML", "UUIDs"] -git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.16.0" -weakdeps = ["Dates", "LinearAlgebra"] - - [deps.Compat.extensions] - CompatLinearAlgebraExt = "LinearAlgebra" - -[[deps.CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.1.1+0" - -[[deps.ConstructionBase]] -git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" -uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.5.8" - - [deps.ConstructionBase.extensions] - ConstructionBaseIntervalSetsExt = "IntervalSets" - ConstructionBaseLinearAlgebraExt = "LinearAlgebra" - ConstructionBaseStaticArraysExt = "StaticArrays" - - [deps.ConstructionBase.weakdeps] - IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" - LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - -[[deps.Contour]] -git-tree-sha1 = "439e35b0b36e2e5881738abc8857bd92ad6ff9a8" -uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" -version = "0.6.3" - -[[deps.Crayons]] -git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.1.1" - -[[deps.DataAPI]] -git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.16.0" - -[[deps.DataStructures]] -deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.20" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" - -<<<<<<< HEAD -[[deps.DelimitedFiles]] -deps = ["Mmap"] -git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" -uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" -version = "1.9.1" - -======= ->>>>>>> 11a0180 (fix comment indentation) -[[deps.DiffResults]] -deps = ["StaticArraysCore"] -git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" -uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" -version = "1.1.0" - -[[deps.DiffRules]] -deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] -git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" -uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "1.15.1" - -[[deps.Distributions]] -deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] -git-tree-sha1 = "03aa5d44647eaec98e1920635cdfed5d5560a8b9" -uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.117" - - [deps.Distributions.extensions] - DistributionsChainRulesCoreExt = "ChainRulesCore" - DistributionsDensityInterfaceExt = "DensityInterface" - DistributionsTestExt = "Test" - - [deps.Distributions.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" - -[[deps.ExprTools]] -git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" -uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.10" - -[[deps.FastClosures]] -git-tree-sha1 = "acebe244d53ee1b461970f8910c235b259e772ef" -uuid = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" -version = "0.3.2" - -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -version = "1.11.0" - -[[deps.FillArrays]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "6a70198746448456524cb442b8af316927ff3e1a" -uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.13.0" -weakdeps = ["PDMats", "SparseArrays", "Statistics"] - - [deps.FillArrays.extensions] - FillArraysPDMatsExt = "PDMats" - FillArraysSparseArraysExt = "SparseArrays" - FillArraysStatisticsExt = "Statistics" - -[[deps.FixedPointNumbers]] -deps = ["Statistics"] -git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" -uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -version = "0.8.5" - -[[deps.ForwardDiff]] -deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] -git-tree-sha1 = "a2df1b776752e3f344e5116c06d75a10436ab853" -uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "0.10.38" -weakdeps = ["StaticArrays"] - - [deps.ForwardDiff.extensions] - ForwardDiffStaticArraysExt = "StaticArrays" - -[[deps.Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" -version = "1.11.0" - -[[deps.Graphics]] -deps = ["Colors", "LinearAlgebra", "NaNMath"] -git-tree-sha1 = "a641238db938fff9b2f60d08ed9030387daf428c" -uuid = "a2bd30eb-e257-5431-a919-1863eab51364" -version = "1.1.3" - -[[deps.HypergeometricFunctions]] -deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] -git-tree-sha1 = "2bd56245074fab4015b9174f24ceba8293209053" -uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" -version = "0.3.27" - -[[deps.ImageCore]] -deps = ["AbstractFFTs", "Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "Reexport"] -git-tree-sha1 = "db645f20b59f060d8cfae696bc9538d13fd86416" -uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" -version = "0.8.22" - -[[deps.InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" - -[[deps.IrrationalConstants]] -git-tree-sha1 = "e2222959fbc6c19554dc15174c81bf7bf3aa691c" -uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" -version = "0.2.4" - -[[deps.IterativeSolvers]] -deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] -git-tree-sha1 = "59545b0a2b27208b0650df0a46b8e3019f85055b" -uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" -version = "0.9.4" - -[[deps.JLLWrappers]] -deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" - -[[deps.JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.4" - -[[deps.JSON3]] -deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] -git-tree-sha1 = "1d322381ef7b087548321d3f878cb4c9bd8f8f9b" -uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -version = "1.14.1" - - [deps.JSON3.extensions] - JSON3ArrowExt = ["ArrowTypes"] - - [deps.JSON3.weakdeps] - ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" - -[[deps.JSOSolvers]] -deps = ["Krylov", "LinearAlgebra", "LinearOperators", "Logging", "NLPModels", "NLPModelsModifiers", "Printf", "SolverCore", "SolverTools"] -git-tree-sha1 = "67f493f5ad881df690c1d9a9bfb8c7c7eda42fba" -uuid = "10dff2fc-5484-5881-a0e0-c90441020f8a" -version = "0.11.2" - -[[deps.Krylov]] -deps = ["LinearAlgebra", "Printf", "SparseArrays"] -git-tree-sha1 = "b29d37ce30fa401a4563b18880ab91f979a29734" -uuid = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" -version = "0.9.10" - -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.4" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.6.0+0" - -[[deps.LibGit2]] -deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -version = "1.11.0" - -[[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.7.2+0" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.0+1" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -version = "1.11.0" - -[[deps.LinearAlgebra]] -deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -version = "1.11.0" - -[[deps.LinearOperators]] -deps = ["FastClosures", "LinearAlgebra", "Printf", "Requires", "SparseArrays", "TimerOutputs"] -git-tree-sha1 = "3468cfb98c69620005b87f6dbeb6aeb8cffe9465" -repo-rev = "main" -repo-url = "https://github.com/JuliaSmoothOptimizers/LinearOperators.jl.git" -uuid = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125" -version = "2.9.0" - - [deps.LinearOperators.extensions] - LinearOperatorsAMDGPUExt = "AMDGPU" - LinearOperatorsCUDAExt = "CUDA" - LinearOperatorsChainRulesCoreExt = "ChainRulesCore" - LinearOperatorsJLArraysExt = "JLArrays" - LinearOperatorsLDLFactorizationsExt = "LDLFactorizations" - LinearOperatorsMetalExt = "Metal" - - [deps.LinearOperators.weakdeps] - AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" - CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" - LDLFactorizations = "40e66cde-538c-5869-a4ad-c39174c6795b" - Metal = "dde4c033-4e86-420c-a63e-0dd931031962" - -[[deps.LogExpFunctions]] -deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] -git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" -uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.3.29" - - [deps.LogExpFunctions.extensions] - LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" - LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" - LogExpFunctionsInverseFunctionsExt = "InverseFunctions" - - [deps.LogExpFunctions.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" - InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" - -[[deps.METIS_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "2eefa8baa858871ae7770c98c3c2a7e46daba5b4" -uuid = "d00139f3-1899-568f-a2f0-47f597d42d70" -version = "5.1.3+0" - -[[deps.MacroTools]] -git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472" -uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.15" - -[[deps.MappedArrays]] -git-tree-sha1 = "2dab0221fe2b0f2cb6754eaa743cc266339f527e" -uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" -version = "0.4.2" - -[[deps.MarchingCubes]] -deps = ["PrecompileTools", "StaticArrays"] -git-tree-sha1 = "0e893025924b6becbae4109f8020ac0e12674b01" -uuid = "299715c1-40a9-479a-aaf9-4a633d36f717" -version = "0.1.11" - -[[deps.Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" - -[[deps.MathOptInterface]] -deps = ["BenchmarkTools", "CodecBzip2", "CodecZlib", "DataStructures", "ForwardDiff", "JSON3", "LinearAlgebra", "MutableArithmetics", "NaNMath", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays", "SpecialFunctions", "Test", "Unicode"] -git-tree-sha1 = "cfc8c22e3b0157430fc80eb6ef46546483356ea9" -uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" -version = "1.36.0" - -[[deps.MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.6+0" - -[[deps.Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.2.0" - -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" - -[[deps.MosaicViews]] -deps = ["MappedArrays", "OffsetArrays", "PaddedViews", "StackViews"] -git-tree-sha1 = "7b86a5d4d70a9f5cdf2dacb3cbe6d251d1a61dbe" -uuid = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" -version = "0.3.4" - -[[deps.MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.12.12" - -[[deps.MutableArithmetics]] -deps = ["LinearAlgebra", "SparseArrays", "Test"] -git-tree-sha1 = "491bdcdc943fcbc4c005900d7463c9f216aabf4c" -uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" -version = "1.6.4" - -[[deps.NLPModels]] -deps = ["FastClosures", "LinearAlgebra", "LinearOperators", "Printf", "SparseArrays"] -git-tree-sha1 = "51b458add76a938917772ee661ffb9d59b4c7e5d" -uuid = "a4795742-8479-5a88-8948-cc11e1c8c1a6" -version = "0.20.0" - -[[deps.NLPModelsModifiers]] -deps = ["FastClosures", "LinearAlgebra", "LinearOperators", "NLPModels", "Printf", "SparseArrays"] -git-tree-sha1 = "a80505adbe42104cbbe9674591a5ccd9e9c2dfda" -uuid = "e01155f1-5c6f-4375-a9d8-616dd036575f" -version = "0.7.2" - -[[deps.NaNMath]] -deps = ["OpenLibm_jll"] -git-tree-sha1 = "cc0a5deefdb12ab3a096f00a6d42133af4560d71" -uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -version = "1.1.2" - -[[deps.NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" - -[[deps.Noise]] -deps = ["ImageCore", "PoissonRandom", "Random"] -git-tree-sha1 = "42224fd87de5c50a593bbf1bce16c67b1d65da88" -uuid = "81d43f40-5267-43b7-ae1c-8b967f377efa" -version = "0.2.2" - -[[deps.OSQP]] -deps = ["Libdl", "LinearAlgebra", "MathOptInterface", "OSQP_jll", "SparseArrays"] -git-tree-sha1 = "50faf456a64ac1ca097b78bcdf288d94708adcdd" -uuid = "ab2f91bb-94b4-55e3-9ba0-7f65df51de79" -version = "0.8.1" - -[[deps.OSQP_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "d0f73698c33e04e557980a06d75c2d82e3f0eb49" -uuid = "9c4f68bf-6205-5545-a508-2878b064d984" -version = "0.600.200+0" - -[[deps.OffsetArrays]] -git-tree-sha1 = "5e1897147d1ff8d98883cda2be2187dcf57d8f0c" -uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "1.15.0" -weakdeps = ["Adapt"] - - [deps.OffsetArrays.extensions] - OffsetArraysAdaptExt = "Adapt" - -[[deps.OpenBLAS32_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] -git-tree-sha1 = "ece4587683695fe4c5f20e990da0ed7e83c351e7" -uuid = "656ef2d0-ae68-5445-9ca0-591084a874a2" -version = "0.3.29+0" - -[[deps.OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.27+1" - -[[deps.OpenLibm_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.1+2" - -[[deps.OpenSpecFun_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" -uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" -version = "0.5.6+0" - -[[deps.OrderedCollections]] -git-tree-sha1 = "cc4054e898b852042d7b503313f7ad03de99c3dd" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.8.0" - -[[deps.PDMats]] -deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "966b85253e959ea89c53a9abebbf2e964fbf593b" -uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.32" - -[[deps.PaddedViews]] -deps = ["OffsetArrays"] -git-tree-sha1 = "0fac6313486baae819364c52b4f483450a9d793f" -uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" -version = "0.5.12" - -[[deps.Parsers]] -deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.1" - -[[deps.Percival]] -deps = ["JSOSolvers", "Krylov", "LinearAlgebra", "LinearOperators", "Logging", "NLPModels", "NLPModelsModifiers", "SolverCore", "SolverTools"] -git-tree-sha1 = "058d2127b2f9e8d5cd75c6123954abec4c214572" -uuid = "01435c0c-c90d-11e9-3788-63660f8fbccc" -version = "0.7.2" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.11.0" - - [deps.Pkg.extensions] - REPLExt = "REPL" - - [deps.Pkg.weakdeps] - REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[deps.PoissonRandom]] -deps = ["Random"] -git-tree-sha1 = "a0f1159c33f846aa77c3f30ebbc69795e5327152" -uuid = "e409e4f3-bfea-5376-8464-e040bb5c01ab" -version = "0.4.4" - -[[deps.PrecompileTools]] -deps = ["Preferences"] -git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" -uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.1" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" - -[[deps.Profile]] -uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" -version = "1.11.0" - -[[deps.ProximalCore]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "1f9f650b4b7a60533098dc5e864458f0e4a5b926" -uuid = "dc4f5ac2-75d1-4f31-931e-60435d74994b" -version = "0.1.2" - -[[deps.ProximalOperators]] -deps = ["IterativeSolvers", "LinearAlgebra", "OSQP", "ProximalCore", "SparseArrays", "SuiteSparse", "TSVD"] -git-tree-sha1 = "13a384f52be09c6795ab1c3ad71c8a207decb0ba" -uuid = "a725b495-10eb-56fe-b38b-717eba820537" -version = "0.15.3" - -[[deps.PtrArrays]] -git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" -uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" -version = "1.3.0" - -[[deps.QRMumps]] -deps = ["Libdl", "LinearAlgebra", "OpenBLAS32_jll", "Printf", "SparseArrays", "qr_mumps_jll"] -git-tree-sha1 = "dcadd41c716d150699e0694cebe854ca73af5a51" -repo-rev = "golub_riley" -repo-url = "https://github.com/MaxenceGollier/QRMumps.jl.git" -uuid = "422b30a1-cc69-4d85-abe7-cc07b540c444" -version = "0.2.7" -weakdeps = ["SparseMatricesCOO"] - - [deps.QRMumps.extensions] - QRMumpsSparseMatricesCOOExt = "SparseMatricesCOO" - -[[deps.QuadGK]] -deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "9da16da70037ba9d701192e27befedefb91ec284" -uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.11.2" - - [deps.QuadGK.extensions] - QuadGKEnzymeExt = "Enzyme" - - [deps.QuadGK.weakdeps] - Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" - -[[deps.Random]] -deps = ["SHA"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" - -[[deps.RecipesBase]] -deps = ["PrecompileTools"] -git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" -uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" -version = "1.3.4" - -[[deps.Reexport]] -git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "1.2.2" - -[[deps.RegularizedProblems]] -deps = ["Distributions", "LinearAlgebra", "NLPModels", "Noise", "Random", "Requires", "SparseArrays"] -git-tree-sha1 = "ae672e0c4f9c7f83a7470175367ef1c01b559d26" -uuid = "ea076b23-609f-44d2-bb12-a4ae45328278" -version = "0.1.1" - -[[deps.Requires]] -deps = ["UUIDs"] -git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" -uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "1.3.0" - -[[deps.Rmath]] -deps = ["Random", "Rmath_jll"] -git-tree-sha1 = "852bd0f55565a9e973fcfee83a84413270224dc4" -uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" -version = "0.8.0" - -[[deps.Rmath_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" -uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" -version = "0.5.1+0" - -[[deps.Roots]] -deps = ["CommonSolve", "Printf", "Setfield"] -git-tree-sha1 = "838b60ee62bebc794864c880a47e331e00c47505" -uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" -version = "1.4.1" - -[[deps.SCOTCH_jll]] -deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll"] -git-tree-sha1 = "ce0770b4aea5f187e0179844503e0d3061d95ab8" -uuid = "a8d0f55d-b80e-548d-aff6-1a04c175f0f9" -version = "7.0.6+1" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" - -[[deps.Setfield]] -deps = ["ConstructionBase", "Future", "MacroTools", "Requires"] -git-tree-sha1 = "38d88503f695eb0301479bc9b0d4320b378bafe5" -uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" -version = "0.8.2" - -[[deps.ShiftedProximalOperators]] -deps = ["LinearAlgebra", "OpenBLAS32_jll", "ProximalOperators", "QRMumps", "Roots", "SparseArrays", "SparseMatricesCOO", "libblastrampoline_jll"] -git-tree-sha1 = "57b818e1e04638cca9b843201b31e809957c7e7d" -repo-rev = "compositeNormL2" -repo-url = "https://github.com/MaxenceGollier/ShiftedProximalOperators.jl.git" -uuid = "d4fd37fa-580c-4e43-9b30-361c21aae263" -version = "0.2.1" - -[[deps.SolverCore]] -deps = ["LinearAlgebra", "NLPModels", "Printf"] -git-tree-sha1 = "03a1e0d2d39b9ebc9510f2452c0adfbe887b9cb2" -uuid = "ff4d7338-4cf1-434d-91df-b86cb86fb843" -version = "0.3.8" - -[[deps.SolverTools]] -deps = ["LinearAlgebra", "LinearOperators", "NLPModels", "Printf"] -git-tree-sha1 = "a09dc3247756965ef4bd0aafc92be4914303a5eb" -uuid = "b5612192-2639-5dc1-abfe-fbedd65fab29" -version = "0.8.8" - -[[deps.SortingAlgorithms]] -deps = ["DataStructures"] -git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.2.1" - -[[deps.SparseArrays]] -deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.11.0" - -[[deps.SparseMatricesCOO]] -deps = ["LinearAlgebra", "SparseArrays", "Test", "UnicodePlots"] -git-tree-sha1 = "b5ca894173efe08725d376063cccb5fa2bcb7a5a" -uuid = "fa32481b-f100-4b48-8dc8-c62f61b13870" -version = "0.2.3" - -[[deps.SpecialFunctions]] -deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "64cca0c26b4f31ba18f13f6c12af7c85f478cfde" -uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.5.0" - - [deps.SpecialFunctions.extensions] - SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" - - [deps.SpecialFunctions.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - -[[deps.StackViews]] -deps = ["OffsetArrays"] -git-tree-sha1 = "46e589465204cd0c08b4bd97385e4fa79a0c770c" -uuid = "cae243ae-269e-4f55-b966-ac2d0dc13c15" -version = "0.1.1" - -[[deps.StaticArrays]] -deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] -git-tree-sha1 = "e3be13f448a43610f978d29b7adf78c76022467a" -uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.9.12" - - [deps.StaticArrays.extensions] - StaticArraysChainRulesCoreExt = "ChainRulesCore" - StaticArraysStatisticsExt = "Statistics" - - [deps.StaticArrays.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - -[[deps.StaticArraysCore]] -git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" -uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" -version = "1.4.3" - -[[deps.Statistics]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.11.1" -weakdeps = ["SparseArrays"] - - [deps.Statistics.extensions] - SparseArraysExt = ["SparseArrays"] - -[[deps.StatsAPI]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" -uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.7.0" - -[[deps.StatsBase]] -deps = ["AliasTables", "DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] -git-tree-sha1 = "29321314c920c26684834965ec2ce0dacc9cf8e5" -uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.34.4" - -[[deps.StatsFuns]] -deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] -git-tree-sha1 = "b423576adc27097764a90e163157bcfc9acf0f46" -uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" -version = "1.3.2" - - [deps.StatsFuns.extensions] - StatsFunsChainRulesCoreExt = "ChainRulesCore" - StatsFunsInverseFunctionsExt = "InverseFunctions" - - [deps.StatsFuns.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" - -[[deps.StructTypes]] -deps = ["Dates", "UUIDs"] -git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" -uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" -version = "1.11.0" - -[[deps.SuiteSparse]] -deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] -uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" - -[[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] -uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.7.0+0" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.TSVD]] -deps = ["Adapt", "LinearAlgebra"] -git-tree-sha1 = "c39caef6bae501e5607a6caf68dd9ac6e8addbcb" -uuid = "9449cd9e-2762-5aa3-a617-5413e99d722e" -version = "0.4.4" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" - -[[deps.TensorCore]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6" -uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50" -version = "0.1.1" - -[[deps.Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -version = "1.11.0" - -[[deps.TimerOutputs]] -deps = ["ExprTools", "Printf"] -git-tree-sha1 = "3832505b94c1868baea47764127e6d36b5c9f29e" -uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.27" - - [deps.TimerOutputs.extensions] - FlameGraphsExt = "FlameGraphs" - - [deps.TimerOutputs.weakdeps] - FlameGraphs = "08572546-2f56-4bcf-ba4e-bab62c3a3f89" - -[[deps.TranscodingStreams]] -git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" -uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.11.3" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -version = "1.11.0" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" - -[[deps.UnicodePlots]] -deps = ["ColorSchemes", "ColorTypes", "Contour", "Crayons", "Dates", "LinearAlgebra", "MarchingCubes", "NaNMath", "PrecompileTools", "Printf", "SparseArrays", "StaticArrays", "StatsBase"] -git-tree-sha1 = "24c0e2df19eb3f894d28a64e7486926f38de8a49" -uuid = "b8865327-cd53-5732-bb35-84acbb429228" -version = "3.7.2" - - [deps.UnicodePlots.extensions] - FreeTypeExt = ["FileIO", "FreeType"] - ImageInTerminalExt = "ImageInTerminal" - IntervalSetsExt = "IntervalSets" - TermExt = "Term" - UnitfulExt = "Unitful" - - [deps.UnicodePlots.weakdeps] - FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" - FreeType = "b38be410-82b0-50bf-ab77-7b57e271db43" - ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" - IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" - Term = "22787eb5-b846-44ae-b979-8e399b8463ab" - Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" - -[[deps.XZ_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "56c6604ec8b2d82cc4cfe01aa03b00426aac7e1f" -uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" -version = "5.6.4+1" - -[[deps.Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+1" - -[[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.11.0+0" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.59.0+0" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+2" - -[[deps.qr_mumps_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "METIS_jll", "SCOTCH_jll", "SuiteSparse_jll", "libblastrampoline_jll"] -git-tree-sha1 = "875f1858b94ba19ae0b3b571525a3114ecbb3413" -uuid = "e37b5aa0-c611-5f0f-83fb-aee446c0b77e" -version = "3.1.1+0" From 1ee48a841243b718435f516a9f6935d6ce4eaa76 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 9 Apr 2025 14:30:29 -0400 Subject: [PATCH 41/48] update theta default value --- src/input_struct.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/input_struct.jl b/src/input_struct.jl index 7242b90a..1b8b1581 100644 --- a/src/input_struct.jl +++ b/src/input_struct.jl @@ -34,7 +34,7 @@ mutable struct ROSolverOptions{R} α::R = 1 / eps(R), ν::R = eps(R)^(1 / 5), γ::R = R(3), - θ::R = eps(R)^(1 / 5), + θ::R = 1/(1+eps(R)^(1 / 5)), β::R = 1 / eps(R), reduce_TR::Bool = true, ) where {R <: Real} From cb8fe6907b945624c8b53042cd9ff30076bd4173 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Thu, 10 Apr 2025 17:07:11 -0400 Subject: [PATCH 42/48] format code --- src/R2DH.jl | 51 ++++++++++++++++++++-------------------- src/R2N.jl | 57 ++++++++++++++++++++++----------------------- src/R2NModel.jl | 39 +++++++++++-------------------- test/test_allocs.jl | 2 +- 4 files changed, 68 insertions(+), 81 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index b72b891d..a1312549 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -6,7 +6,7 @@ mutable struct R2DHSolver{ T <: Real, G <: ShiftedProximableFunction, V <: AbstractVector{T}, - QN <: AbstractDiagonalQuasiNewtonOperator{T} + QN <: AbstractDiagonalQuasiNewtonOperator{T}, } <: AbstractOptimizationSolver xk::V ∇fk::V @@ -25,7 +25,11 @@ mutable struct R2DHSolver{ m_fh_hist::V end -function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int = 6, D :: Union{Nothing, AbstractDiagonalQuasiNewtonOperator} = nothing) where{T, V} +function R2DHSolver( + reg_nlp::AbstractRegularizedNLPModel{T, V}; + m_monotone::Int = 6, + D::Union{Nothing, AbstractDiagonalQuasiNewtonOperator} = nothing, +) where {T, V} x0 = reg_nlp.model.meta.x0 l_bound = reg_nlp.model.meta.lvar u_bound = reg_nlp.model.meta.uvar @@ -49,8 +53,14 @@ function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int end m_fh_hist = fill(T(-Inf), m_monotone - 1) - ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) - isnothing(D) && (D = isa(reg_nlp.model, AbstractDiagonalQNModel) ? hess_op(reg_nlp.model, x0) : SpectralGradient(T(1), reg_nlp.model.meta.nvar)) + ψ = + has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : + shifted(reg_nlp.h, xk) + isnothing(D) && ( + D = + isa(reg_nlp.model, AbstractDiagonalQNModel) ? hess_op(reg_nlp.model, x0) : + SpectralGradient(T(1), reg_nlp.model.meta.nvar) + ) return R2DHSolver( xk, @@ -67,10 +77,9 @@ function R2DHSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; m_monotone::Int u_bound, l_bound_m_x, u_bound_m_x, - m_fh_hist + m_fh_hist, ) end - """ R2DH(reg_nlp; kwargs…) @@ -141,12 +150,7 @@ Notably, you can access, and modify, the following: - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything will stop the algorithm, but you should use `:user` to properly indicate the intention. - `stats.elapsed_time`: elapsed time in seconds. """ -function R2DH( - nlp::AbstractNLPModel{T, V}, - h, - options::ROSolverOptions{T}; - kwargs..., -) where{T, V} +function R2DH(nlp::AbstractNLPModel{T, V}, h, options::ROSolverOptions{T}; kwargs...) where {T, V} kwargs_dict = Dict(kwargs...) selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) @@ -200,14 +204,11 @@ function R2DH( θ = options.θ, kwargs..., ) - + return stats.solution, stats.iter, nothing end -function R2DH( - reg_nlp::AbstractRegularizedNLPModel{T, V}; - kwargs... -) where{T, V} +function R2DH(reg_nlp::AbstractRegularizedNLPModel{T, V}; kwargs...) where {T, V} kwargs_dict = Dict(kwargs...) m_monotone = pop!(kwargs_dict, :m_monotone, 6) D = pop!(kwargs_dict, :D, nothing) @@ -236,8 +237,7 @@ function SolverCore.solve!( ν::T = eps(T)^(1 / 5), γ::T = T(3), θ::T = 1/(1 + eps(T)^(1 / 5)), -) where{T, V} - +) where {T, V} reset!(stats) # Retrieve workspace @@ -269,7 +269,6 @@ function SolverCore.solve!( end m_monotone = length(m_fh_hist) + 1 - # initialize parameters improper = false hk = @views h(xk[selected]) @@ -324,7 +323,7 @@ function SolverCore.solve!( set_objective!(stats, fk + hk) set_solver_specific!(stats, :smooth_obj, fk) set_solver_specific!(stats, :nonsmooth_obj, hk) - m_monotone > 1 && (m_fh_hist[(stats.iter)%(m_monotone - 1) + 1] = fk + hk) + m_monotone > 1 && (m_fh_hist[(stats.iter) % (m_monotone - 1) + 1] = fk + hk) φ(d) = begin result = zero(T) @@ -334,12 +333,12 @@ function SolverCore.solve!( end return result end - + mk(d)::T = φ(d) + ψ(d)::T spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) - mks = mk(s) + mks = mk(s) ξ = hk - mks + max(1, abs(hk)) * 10 * eps() sqrt_ξ_νInv = ξ ≥ 0 ? sqrt(ξ / ν₁) : sqrt(-ξ / ν₁) @@ -431,7 +430,7 @@ function SolverCore.solve!( ν₁ = θ / (DNorm + σk) @. mν∇fk = -ν₁ * ∇fk - m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) + m_monotone > 1 && (m_fh_hist[stats.iter % (m_monotone - 1) + 1] = fk + hk) spectral_test ? prox!(s, ψ, mν∇fk, ν₁) : iprox!(s, ψ, ∇fk, dkσk) mks = mk(s) @@ -479,7 +478,7 @@ function SolverCore.solve!( @info "R2DH: terminating with √(ξ/ν) = $(sqrt_ξ_νInv)" end - set_solution!(stats,xk) + set_solution!(stats, xk) set_residuals!(stats, zero(eltype(xk)), sqrt_ξ_νInv) return stats -end \ No newline at end of file +end diff --git a/src/R2N.jl b/src/R2N.jl index 5182eb52..cc647683 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -7,7 +7,7 @@ mutable struct R2NSolver{ G <: ShiftedProximableFunction, V <: AbstractVector{T}, ST <: AbstractOptimizationSolver, - PB <: AbstractRegularizedNLPModel + PB <: AbstractRegularizedNLPModel, } <: AbstractOptimizationSolver xk::V ∇fk::V @@ -28,7 +28,11 @@ mutable struct R2NSolver{ substats::GenericExecutionStats{T, V, V, T} end -function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Solver, m_monotone::Int = 1) where {T, V} +function R2NSolver( + reg_nlp::AbstractRegularizedNLPModel{T, V}; + subsolver = R2Solver, + m_monotone::Int = 1, +) where {T, V} x0 = reg_nlp.model.meta.x0 l_bound = reg_nlp.model.meta.lvar u_bound = reg_nlp.model.meta.uvar @@ -53,15 +57,12 @@ function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Sol end m_fh_hist = fill(T(-Inf), m_monotone - 1) - ψ = has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : shifted(reg_nlp.h, xk) + ψ = + has_bnds ? shifted(reg_nlp.h, xk, l_bound_m_x, u_bound_m_x, reg_nlp.selected) : + shifted(reg_nlp.h, xk) Bk = hess_op(reg_nlp.model, x0) - sub_nlp = R2NModel( - Bk, - ∇fk, - T(1), - x0 - ) + sub_nlp = R2NModel(Bk, ∇fk, T(1), x0) subpb = RegularizedNLPModel(sub_nlp, ψ) substats = RegularizedExecutionStats(subpb) subsolver = subsolver(subpb) @@ -83,10 +84,9 @@ function R2NSolver(reg_nlp::AbstractRegularizedNLPModel{T, V}; subsolver = R2Sol m_fh_hist, subsolver, subpb, - substats + substats, ) end - """ R2N(reg_nlp; kwargs…) @@ -212,9 +212,9 @@ function SolverCore.solve!( ν::T = eps(T)^(1 / 5), γ::T = T(3), β::T = 1 / eps(T), - θ::T = 1/(1+eps(T)^(1 / 5)), - kwargs... -) where{T, V, G} + θ::T = 1/(1 + eps(T)^(1 / 5)), + kwargs..., +) where {T, V, G} reset!(stats) # Retrieve workspace @@ -263,7 +263,7 @@ function SolverCore.solve!( @info log_header( [:outer, :inner, :fx, :hx, :xi, :ρ, :σ, :normx, :norms, :normB, :arrow], [Int, Int, T, T, T, T, T, T, T, T, Char], - hdr_override = Dict{Symbol, String}( + hdr_override = Dict{Symbol, String}( :fx => "f(x)", :hx => "h(x)", :xi => "√(ξ1/ν)", @@ -294,7 +294,7 @@ function SolverCore.solve!( ν₁ = θ / (λmax + σk) ν_sub = ν₁ - + sqrt_ξ1_νInv = one(T) @. mν∇fk = -ν₁ * ∇fk @@ -305,7 +305,7 @@ function SolverCore.solve!( set_objective!(stats, fk + hk) set_solver_specific!(stats, :smooth_obj, fk) set_solver_specific!(stats, :nonsmooth_obj, hk) - m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) + m_monotone > 1 && (m_fh_hist[stats.iter % (m_monotone - 1) + 1] = fk + hk) φ1 = let ∇fk = ∇fk d -> dot(∇fk, d) @@ -328,7 +328,7 @@ function SolverCore.solve!( (ξ1 < 0 && sqrt_ξ1_νInv > neg_tol) && error("R2N: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") atol += rtol * sqrt_ξ1_νInv # make stopping test absolute and relative - + set_status!( stats, get_status( @@ -348,20 +348,19 @@ function SolverCore.solve!( done = stats.status != :unknown while !done + sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5), sqrt_ξ1_νInv * 1e-3) - sub_atol = stats.iter == 0 ? 1.0e-3 : min(sqrt_ξ1_νInv ^ (1.5) , sqrt_ξ1_νInv * 1e-3) - solver.subpb.model.σ = σk isa(solver.subsolver, R2DHSolver) && (solver.subsolver.D.d[1] = 1/ν₁) ν_sub = isa(solver.subsolver, R2DHSolver) ? 1/σk : ν₁ solve!( - solver.subsolver, - solver.subpb, + solver.subsolver, + solver.subpb, solver.substats; x = s1, atol = sub_atol, ν = ν_sub, - kwargs... + kwargs..., ) s .= solver.substats.solution @@ -424,7 +423,7 @@ function SolverCore.solve!( push!(nlp, s, ∇fk⁻) end solver.subpb.model.B = hess_op(nlp, xk) - + λmax, found_λ = opnorm(solver.subpb.model.B) found_λ || error("operator norm computation failed") @@ -438,9 +437,9 @@ function SolverCore.solve!( if ρk < η1 || ρk == Inf σk = σk * γ end - + ν₁ = θ / (λmax + σk) - m_monotone > 1 && (m_fh_hist[stats.iter%(m_monotone - 1) + 1] = fk + hk) + m_monotone > 1 && (m_fh_hist[stats.iter % (m_monotone - 1) + 1] = fk + hk) set_objective!(stats, fk + hk) set_solver_specific!(stats, :smooth_obj, fk) @@ -448,7 +447,7 @@ function SolverCore.solve!( set_iter!(stats, stats.iter + 1) set_time!(stats, time() - start_time) - @. mν∇fk = - ν₁ * ∇fk + @. mν∇fk = - ν₁ * ∇fk prox!(s1, ψ, mν∇fk, ν₁) mks = mk1(s1) @@ -456,7 +455,7 @@ function SolverCore.solve!( sqrt_ξ1_νInv = ξ1 ≥ 0 ? sqrt(ξ1 / ν₁) : sqrt(-ξ1 / ν₁) solved = (ξ1 < 0 && sqrt_ξ1_νInv ≤ neg_tol) || (ξ1 ≥ 0 && sqrt_ξ1_νInv ≤ atol) - + (ξ1 < 0 && sqrt_ξ1_νInv > neg_tol) && error("R2N: prox-gradient step should produce a decrease but ξ1 = $(ξ1)") set_status!( @@ -501,4 +500,4 @@ function SolverCore.solve!( set_solution!(stats, xk) set_residuals!(stats, zero(eltype(xk)), sqrt_ξ1_νInv) return stats -end \ No newline at end of file +end diff --git a/src/R2NModel.jl b/src/R2NModel.jl index 3b58b70a..a4b2f990 100644 --- a/src/R2NModel.jl +++ b/src/R2NModel.jl @@ -14,44 +14,33 @@ this model represents the smooth R2N subproblem: where `B` is either an approximation of the Hessian of `f` or the Hessian itself and `∇f` represents the gradient of `f` at `x0`. `σ > 0` is a regularization parameter and `v` is a vector of the same size as `x0` used for intermediary computations. """ -mutable struct R2NModel{T <: Real, V <: AbstractVector{T}, G <: AbstractLinearOperator{T}} <: AbstractNLPModel{T, V} - B :: G - ∇f :: V - v :: V - σ :: T +mutable struct R2NModel{T <: Real, V <: AbstractVector{T}, G <: AbstractLinearOperator{T}} <: + AbstractNLPModel{T, V} + B::G + ∇f::V + v::V + σ::T meta::NLPModelMeta{T, V} counters::Counters end - -function R2NModel( - B :: G, - ∇f :: V, - σ :: T, - x0 :: V -) where{T, V, G} + +function R2NModel(B::G, ∇f::V, σ::T, x0::V) where {T, V, G} @assert length(x0) == length(∇f) meta = NLPModelMeta( length(∇f), x0 = x0, # Perhaps we should add lvar and uvar as well here. ) v = similar(x0) - return R2NModel( - B :: G, - ∇f :: V, - v :: V, - σ :: T, - meta, - Counters() - ) + return R2NModel(B::G, ∇f::V, v::V, σ::T, meta, Counters()) end - + function NLPModels.obj(nlp::R2NModel, x::AbstractVector) @lencheck nlp.meta.nvar x increment!(nlp, :neval_obj) mul!(nlp.v, nlp.B, x) - return dot(nlp.v, x)/2 + dot(nlp.∇f, x) + nlp.σ * dot(x, x) / 2 + return dot(nlp.v, x)/2 + dot(nlp.∇f, x) + nlp.σ * dot(x, x) / 2 end - + function NLPModels.grad!(nlp::R2NModel, x::AbstractVector, g::AbstractVector) @lencheck nlp.meta.nvar x @lencheck nlp.meta.nvar g @@ -59,5 +48,5 @@ function NLPModels.grad!(nlp::R2NModel, x::AbstractVector, g::AbstractVector) mul!(g, nlp.B, x) g .+= nlp.∇f g .+= nlp.σ .* x - return g -end \ No newline at end of file + return g +end diff --git a/test/test_allocs.jl b/test/test_allocs.jl index 0489ca56..271c1887 100644 --- a/test/test_allocs.jl +++ b/test/test_allocs.jl @@ -41,7 +41,7 @@ end # Test non allocating solve! @testset "allocs" begin - for (h, h_name) ∈ ((NormL0(λ), "l0"), ) + for (h, h_name) ∈ ((NormL0(λ), "l0"),) for (solver, solver_name) ∈ ((:R2Solver, "R2"), (:R2DHSolver, "R2DH"), (:R2NSolver, "R2N")) @testset "$(solver_name)" begin solver_name == "R2N" && continue #FIXME From bd1816fec6d044039d397946cc4ae5b030537d6a Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 15 Apr 2025 15:29:39 -0400 Subject: [PATCH 43/48] apply suggestions in R2N & R2DH --- src/R2DH.jl | 2 +- src/R2N.jl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index a1312549..be5773ee 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -150,7 +150,7 @@ Notably, you can access, and modify, the following: - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything will stop the algorithm, but you should use `:user` to properly indicate the intention. - `stats.elapsed_time`: elapsed time in seconds. """ -function R2DH(nlp::AbstractNLPModel{T, V}, h, options::ROSolverOptions{T}; kwargs...) where {T, V} +function R2DH(nlp::AbstractDiagonalQNModel{T, V}, h, options::ROSolverOptions{T}; kwargs...) where {T, V} kwargs_dict = Dict(kwargs...) selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) diff --git a/src/R2N.jl b/src/R2N.jl index cc647683..409e8f0b 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -44,7 +44,6 @@ function R2NSolver( xkn = similar(x0) s = similar(x0) s1 = similar(x0) - v = similar(x0) has_bnds = any(l_bound .!= T(-Inf)) || any(u_bound .!= T(Inf)) if has_bnds l_bound_m_x = similar(xk) From 15cfbf0a8d77e042f0d6af58d31a400098424959 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Wed, 16 Apr 2025 18:46:05 -0400 Subject: [PATCH 44/48] apply suggestions for R2DH --- src/R2DH.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index be5773ee..513665c2 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -196,6 +196,7 @@ function R2DH( verbose = options.verbose, max_iter = options.maxIter, max_time = options.maxTime, + σk = options.σk, σmin = options.σmin, η1 = options.η1, η2 = options.η2, @@ -231,6 +232,7 @@ function SolverCore.solve!( max_iter::Int = 10000, max_time::Float64 = 30.0, max_eval::Int = -1, + σk::T = eps(T)^(1 / 5), σmin::T = eps(T), η1::T = √√eps(T), η2::T = T(0.9), @@ -302,8 +304,6 @@ function SolverCore.solve!( local ξ::T local ρk::T = zero(T) - σk = max(1 / ν, σmin) - fk = obj(nlp, xk) grad!(nlp, xk, ∇fk) ∇fk⁻ .= ∇fk From 069b8542600b2a6f3fcb869b68790c538d3cd2bb Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 29 Apr 2025 15:41:37 +0200 Subject: [PATCH 45/48] add sub_kwargs dict for R2N --- src/R2N.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/R2N.jl b/src/R2N.jl index 409e8f0b..7420243a 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -131,6 +131,7 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. - `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model. - `m_monotone::Int = 1`: monotonicity parameter. By default, R2N is monotone but the non-monotone variant will be used if `m_monotone > 1` +- `sub_kwargs::Dict{Symbol}`: a dictionary containing the keyword arguments to be sent to the subsolver. The solver will fail if invalid keyword arguments are provided to the subsolver. The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. @@ -164,6 +165,7 @@ function R2N( selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) reg_nlp = RegularizedNLPModel(nlp, h, selected) + sub_kwargs = pop!(kwargs_dict, :sub_kwargs, Dict{Symbol, Any}()) return R2N( reg_nlp, x = x0, @@ -177,7 +179,8 @@ function R2N( η1 = options.η1, η2 = options.η2, ν = options.ν, - γ = options.γ; + γ = options.γ, + sub_kwargs = sub_kwargs; kwargs_dict..., ) end @@ -212,7 +215,7 @@ function SolverCore.solve!( γ::T = T(3), β::T = 1 / eps(T), θ::T = 1/(1 + eps(T)^(1 / 5)), - kwargs..., + sub_kwargs::Dict{Symbol} = Dict() ) where {T, V, G} reset!(stats) @@ -351,15 +354,15 @@ function SolverCore.solve!( solver.subpb.model.σ = σk isa(solver.subsolver, R2DHSolver) && (solver.subsolver.D.d[1] = 1/ν₁) - ν_sub = isa(solver.subsolver, R2DHSolver) ? 1/σk : ν₁ + sub_ν = isa(solver.subsolver, R2DHSolver) ? 1/σk : ν₁ solve!( solver.subsolver, solver.subpb, solver.substats; x = s1, atol = sub_atol, - ν = ν_sub, - kwargs..., + ν = sub_ν, + sub_kwargs..., ) s .= solver.substats.solution From 10b829831daa984ba109e75b6e0a987964ac7560 Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 29 Apr 2025 16:25:01 +0200 Subject: [PATCH 46/48] apply suggestions for sigma in R2N & R2DH + reformat some doc --- src/R2DH.jl | 14 ++++++++++---- src/R2N.jl | 21 +++++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index 513665c2..e8f88827 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -145,12 +145,17 @@ Notably, you can access, and modify, the following: - `stats`: structure holding the output of the algorithm (`GenericExecutionStats`), which contains, among other things: - `stats.iter`: current iteration counter; - `stats.objective`: current objective function value; - - `stats.solver_specific[:smooth_obj]`: current value of the smooth part of the objective function - - `stats.solver_specific[:nonsmooth_obj]`: current value of the nonsmooth part of the objective function - - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything will stop the algorithm, but you should use `:user` to properly indicate the intention. + - `stats.solver_specific[:smooth_obj]`: current value of the smooth part of the objective function; + - `stats.solver_specific[:nonsmooth_obj]`: current value of the nonsmooth part of the objective function; + - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything will stop the algorithm, but you should use `:user` to properly indicate the intention; - `stats.elapsed_time`: elapsed time in seconds. """ -function R2DH(nlp::AbstractDiagonalQNModel{T, V}, h, options::ROSolverOptions{T}; kwargs...) where {T, V} +function R2DH( + nlp::AbstractDiagonalQNModel{T, V}, + h, + options::ROSolverOptions{T}; + kwargs..., +) where {T, V} kwargs_dict = Dict(kwargs...) selected = pop!(kwargs_dict, :selected, 1:(nlp.meta.nvar)) x0 = pop!(kwargs_dict, :x0, nlp.meta.x0) @@ -164,6 +169,7 @@ function R2DH(nlp::AbstractDiagonalQNModel{T, V}, h, options::ROSolverOptions{T} verbose = options.verbose, max_iter = options.maxIter, max_time = options.maxTime, + σk = options.σk, σmin = options.σmin, η1 = options.η1, η2 = options.η2, diff --git a/src/R2N.jl b/src/R2N.jl index 7420243a..923dea1a 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -119,18 +119,19 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory - `x::V = nlp.meta.x0`: the initial guess; - `atol::T = √eps(T)`: absolute tolerance; - `rtol::T = √eps(T)`: relative tolerance; -- `neg_tol::T = eps(T)^(1 / 4)`: negative tolerance +- `neg_tol::T = eps(T)^(1 / 4)`: negative tolerance; - `max_eval::Int = -1`: maximum number of evaluation of the objective function (negative number means unlimited); - `max_time::Float64 = 30.0`: maximum time limit in seconds; - `max_iter::Int = 10000`: maximum number of iterations; - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration; - `σmin::T = eps(T)`: minimum value of the regularization parameter; +- `σk::T = eps(T)^(1 / 5)`: initial value of the regularization parameter; - `η1::T = √√eps(T)`: successful iteration threshold; - `η2::T = T(0.9)`: very successful iteration threshold; - `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; -- `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. -- `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model. -- `m_monotone::Int = 1`: monotonicity parameter. By default, R2N is monotone but the non-monotone variant will be used if `m_monotone > 1` +- `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful; +- `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model; +- `m_monotone::Int = 1`: monotonicity parameter. By default, R2N is monotone but the non-monotone variant will be used if `m_monotone > 1`; - `sub_kwargs::Dict{Symbol}`: a dictionary containing the keyword arguments to be sent to the subsolver. The solver will fail if invalid keyword arguments are provided to the subsolver. The algorithm stops either when `√(ξₖ/νₖ) < atol + rtol*√(ξ₀/ν₀) ` or `ξₖ < 0` and `√(-ξₖ/νₖ) < neg_tol` where ξₖ := f(xₖ) + h(xₖ) - φ(sₖ; xₖ) - ψ(sₖ; xₖ), and √(ξₖ/νₖ) is a stationarity measure. @@ -150,9 +151,9 @@ Notably, you can access, and modify, the following: - `stats`: structure holding the output of the algorithm (`GenericExecutionStats`), which contains, among other things: - `stats.iter`: current iteration counter; - `stats.objective`: current objective function value; - - `stats.solver_specific[:smooth_obj]`: current value of the smooth part of the objective function - - `stats.solver_specific[:nonsmooth_obj]`: current value of the nonsmooth part of the objective function - - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything other than `:unknown` will stop the algorithm, but you should use `:user` to properly indicate the intention. + - `stats.solver_specific[:smooth_obj]`: current value of the smooth part of the objective function; + - `stats.solver_specific[:nonsmooth_obj]`: current value of the nonsmooth part of the objective function; + - `stats.status`: current status of the algorithm. Should be `:unknown` unless the algorithm has attained a stopping criterion. Changing this to anything other than `:unknown` will stop the algorithm, but you should use `:user` to properly indicate the intention; - `stats.elapsed_time`: elapsed time in seconds. """ function R2N( @@ -176,6 +177,7 @@ function R2N( max_iter = options.maxIter, max_time = options.maxTime, σmin = options.σmin, + σk = options.σk, η1 = options.η1, η2 = options.η2, ν = options.ν, @@ -208,6 +210,7 @@ function SolverCore.solve!( max_iter::Int = 10000, max_time::Float64 = 30.0, max_eval::Int = -1, + σk::T = eps(T)^(1 / 5), σmin::T = eps(T), η1::T = √√eps(T), η2::T = T(0.9), @@ -215,7 +218,7 @@ function SolverCore.solve!( γ::T = T(3), β::T = 1 / eps(T), θ::T = 1/(1 + eps(T)^(1 / 5)), - sub_kwargs::Dict{Symbol} = Dict() + sub_kwargs::Dict{Symbol} = Dict(), ) where {T, V, G} reset!(stats) @@ -281,8 +284,6 @@ function SolverCore.solve!( local ξ1::T local ρk::T = zero(T) - σk = max(1 / ν, σmin) - fk = obj(nlp, xk) grad!(nlp, xk, ∇fk) ∇fk⁻ .= ∇fk From e74f64aae86a45d1cdc7bdd990cb9d658d071b0d Mon Sep 17 00:00:00 2001 From: Maxence Gollier Date: Tue, 29 Apr 2025 17:17:54 +0200 Subject: [PATCH 47/48] remove nu in R2N & R2DH --- src/R2DH.jl | 5 +---- src/R2N.jl | 19 ++++++------------- test/test_allocs.jl | 7 ++++++- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/R2DH.jl b/src/R2DH.jl index e8f88827..417229e6 100644 --- a/src/R2DH.jl +++ b/src/R2DH.jl @@ -121,9 +121,9 @@ or - `max_iter::Int = 10000`: maximum number of iterations; - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration; - `σmin::T = eps(T)`: minimum value of the regularization parameter; +- `σk::T = eps(T)^(1 / 5)`: initial value of the regularization parameter; - `η1::T = √√eps(T)`: very successful iteration threshold; - `η2::T = T(0.9)`: successful iteration threshold; -- `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful. - `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model. - `m_monotone::Int = 6`: monotoneness parameter. By default, R2DH is non-monotone but the monotone variant can be used with `m_monotone = 1` @@ -173,7 +173,6 @@ function R2DH( σmin = options.σmin, η1 = options.η1, η2 = options.η2, - ν = options.ν, γ = options.γ, θ = options.θ, kwargs_dict..., @@ -206,7 +205,6 @@ function R2DH( σmin = options.σmin, η1 = options.η1, η2 = options.η2, - ν = options.ν, γ = options.γ, θ = options.θ, kwargs..., @@ -242,7 +240,6 @@ function SolverCore.solve!( σmin::T = eps(T), η1::T = √√eps(T), η2::T = T(0.9), - ν::T = eps(T)^(1 / 5), γ::T = T(3), θ::T = 1/(1 + eps(T)^(1 / 5)), ) where {T, V} diff --git a/src/R2N.jl b/src/R2N.jl index 923dea1a..bd9ca150 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -128,7 +128,6 @@ For advanced usage, first define a solver "R2NSolver" to preallocate the memory - `σk::T = eps(T)^(1 / 5)`: initial value of the regularization parameter; - `η1::T = √√eps(T)`: successful iteration threshold; - `η2::T = T(0.9)`: very successful iteration threshold; -- `ν::T = eps(T)^(1 / 5)`: inverse of the initial regularization parameter: ν = 1/σ; - `γ::T = T(3)`: regularization parameter multiplier, σ := σ/γ when the iteration is very successful and σ := σγ when the iteration is unsuccessful; - `θ::T = 1/(1 + eps(T)^(1 / 5))`: is the model decrease fraction with respect to the decrease of the Cauchy model; - `m_monotone::Int = 1`: monotonicity parameter. By default, R2N is monotone but the non-monotone variant will be used if `m_monotone > 1`; @@ -180,7 +179,6 @@ function R2N( σk = options.σk, η1 = options.η1, η2 = options.η2, - ν = options.ν, γ = options.γ, sub_kwargs = sub_kwargs; kwargs_dict..., @@ -214,7 +212,6 @@ function SolverCore.solve!( σmin::T = eps(T), η1::T = √√eps(T), η2::T = T(0.9), - ν::T = eps(T)^(1 / 5), γ::T = T(3), β::T = 1 / eps(T), θ::T = 1/(1 + eps(T)^(1 / 5)), @@ -355,16 +352,12 @@ function SolverCore.solve!( solver.subpb.model.σ = σk isa(solver.subsolver, R2DHSolver) && (solver.subsolver.D.d[1] = 1/ν₁) - sub_ν = isa(solver.subsolver, R2DHSolver) ? 1/σk : ν₁ - solve!( - solver.subsolver, - solver.subpb, - solver.substats; - x = s1, - atol = sub_atol, - ν = sub_ν, - sub_kwargs..., - ) + if isa(solver.subsolver, R2Solver) #FIXME + sub_kwargs[:ν] = ν₁ + else + sub_kwargs[:σk] = σk + end + solve!(solver.subsolver, solver.subpb, solver.substats; x = s1, atol = sub_atol, sub_kwargs...) s .= solver.substats.solution diff --git a/test/test_allocs.jl b/test/test_allocs.jl index 271c1887..accf0ccc 100644 --- a/test/test_allocs.jl +++ b/test/test_allocs.jl @@ -48,7 +48,12 @@ end reg_nlp = RegularizedNLPModel(LBFGSModel(bpdn), h) solver = eval(solver)(reg_nlp) stats = RegularizedExecutionStats(reg_nlp) - @test @wrappedallocs(solve!(solver, reg_nlp, stats, ν = 1.0, atol = 1e-6, rtol = 1e-6)) == 0 + solver_name == "R2" && + @test @wrappedallocs(solve!(solver, reg_nlp, stats, ν = 1.0, atol = 1e-6, rtol = 1e-6)) == + 0 + solver_name == "R2DH" && @test @wrappedallocs( + solve!(solver, reg_nlp, stats, σk = 1.0, atol = 1e-6, rtol = 1e-6) + ) == 0 @test stats.status == :first_order end end From d45c5f3f5943ab594ec1ec1505b4bf848b1c26ed Mon Sep 17 00:00:00 2001 From: Maxence Gollier <134112149+MaxenceGollier@users.noreply.github.com> Date: Wed, 30 Apr 2025 18:35:27 +0200 Subject: [PATCH 48/48] Remove nu_sub in R2N --- src/R2N.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/R2N.jl b/src/R2N.jl index bd9ca150..2bd5bfc8 100644 --- a/src/R2N.jl +++ b/src/R2N.jl @@ -293,7 +293,6 @@ function SolverCore.solve!( found_λ || error("operator norm computation failed") ν₁ = θ / (λmax + σk) - ν_sub = ν₁ sqrt_ξ1_νInv = one(T)