diff --git a/lib/OptimizationAuglag/src/OptimizationAuglag.jl b/lib/OptimizationAuglag/src/OptimizationAuglag.jl index 801552ca4..670dfebaf 100644 --- a/lib/OptimizationAuglag/src/OptimizationAuglag.jl +++ b/lib/OptimizationAuglag/src/OptimizationAuglag.jl @@ -16,12 +16,7 @@ using LinearAlgebra: norm ϵ = 1e-8 end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::AugLag) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::AugLag) = true -end +OptimizationBase.supports_opt_cache_interface(::AugLag) = true SciMLBase.allowsbounds(::AugLag) = true SciMLBase.requiresgradient(::AugLag) = true SciMLBase.allowsconstraints(::AugLag) = true diff --git a/lib/OptimizationBBO/src/OptimizationBBO.jl b/lib/OptimizationBBO/src/OptimizationBBO.jl index cb62b3324..c2b964642 100644 --- a/lib/OptimizationBBO/src/OptimizationBBO.jl +++ b/lib/OptimizationBBO/src/OptimizationBBO.jl @@ -10,12 +10,7 @@ abstract type BBO end SciMLBase.requiresbounds(::BBO) = true SciMLBase.allowsbounds(::BBO) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::BBO) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::BBO) = true -end +OptimizationBase.supports_opt_cache_interface(opt::BBO) = true for j in string.(BlackBoxOptim.SingleObjectiveMethodNames) eval(Meta.parse("Base.@kwdef struct BBO_" * j * " <: BBO method=:" * j * " end")) diff --git a/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl b/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl index 90b65b975..8633c0555 100644 --- a/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl +++ b/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl @@ -10,12 +10,7 @@ export CMAEvolutionStrategyOpt struct CMAEvolutionStrategyOpt end SciMLBase.allowsbounds(::CMAEvolutionStrategyOpt) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::CMAEvolutionStrategyOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::CMAEvolutionStrategyOpt) = true -end +OptimizationBase.supports_opt_cache_interface(opt::CMAEvolutionStrategyOpt) = true SciMLBase.requiresgradient(::CMAEvolutionStrategyOpt) = false SciMLBase.requireshessian(::CMAEvolutionStrategyOpt) = false SciMLBase.requiresconsjac(::CMAEvolutionStrategyOpt) = false diff --git a/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl b/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl index 2d9c4c305..9c8ec5a74 100644 --- a/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl +++ b/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl @@ -6,12 +6,7 @@ using SciMLBase SciMLBase.allowsbounds(opt::Evolutionary.AbstractOptimizer) = true SciMLBase.allowsconstraints(opt::Evolutionary.AbstractOptimizer) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Evolutionary.AbstractOptimizer) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Evolutionary.AbstractOptimizer) = true -end +OptimizationBase.supports_opt_cache_interface(opt::Evolutionary.AbstractOptimizer) = true SciMLBase.requiresgradient(opt::Evolutionary.AbstractOptimizer) = false SciMLBase.requiresgradient(opt::Evolutionary.NSGA2) = false SciMLBase.requireshessian(opt::Evolutionary.AbstractOptimizer) = false diff --git a/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl b/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl index 471198447..647d7926f 100644 --- a/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl +++ b/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl @@ -11,12 +11,7 @@ struct GCMAESOpt end SciMLBase.requiresbounds(::GCMAESOpt) = true SciMLBase.allowsbounds(::GCMAESOpt) = true SciMLBase.allowscallback(::GCMAESOpt) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::GCMAESOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::GCMAESOpt) = true -end +OptimizationBase.supports_opt_cache_interface(opt::GCMAESOpt) = true SciMLBase.requiresgradient(::GCMAESOpt) = true SciMLBase.requireshessian(::GCMAESOpt) = false SciMLBase.requiresconsjac(::GCMAESOpt) = false diff --git a/lib/OptimizationIpopt/src/OptimizationIpopt.jl b/lib/OptimizationIpopt/src/OptimizationIpopt.jl index 269a127ff..9c6898c91 100644 --- a/lib/OptimizationIpopt/src/OptimizationIpopt.jl +++ b/lib/OptimizationIpopt/src/OptimizationIpopt.jl @@ -166,15 +166,8 @@ https://coin-or.github.io/Ipopt/OPTIONS.html additional_options::Dict{String, Any} = Dict{String, Any}() end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - function SciMLBase.supports_opt_cache_interface(alg::IpoptOptimizer) - true - end -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - function OptimizationBase.supports_opt_cache_interface(alg::IpoptOptimizer) - true - end +function OptimizationBase.supports_opt_cache_interface(alg::IpoptOptimizer) + true end function SciMLBase.requiresgradient(opt::IpoptOptimizer) diff --git a/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl b/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl index c39ac837b..f4e2cbaf1 100644 --- a/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl +++ b/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl @@ -31,12 +31,7 @@ References ϵ = 1e-8 end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::LBFGSB) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::LBFGSB) = true -end +OptimizationBase.supports_opt_cache_interface(::LBFGSB) = true SciMLBase.allowsbounds(::LBFGSB) = true SciMLBase.requiresgradient(::LBFGSB) = true SciMLBase.allowsconstraints(::LBFGSB) = true diff --git a/lib/OptimizationMOI/src/OptimizationMOI.jl b/lib/OptimizationMOI/src/OptimizationMOI.jl index ee3c2c6af..ca033b06f 100644 --- a/lib/OptimizationMOI/src/OptimizationMOI.jl +++ b/lib/OptimizationMOI/src/OptimizationMOI.jl @@ -283,17 +283,9 @@ end include("nlp.jl") include("moi.jl") -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - function SciMLBase.supports_opt_cache_interface(alg::Union{MOI.AbstractOptimizer, - MOI.OptimizerWithAttributes}) - true - end -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - function OptimizationBase.supports_opt_cache_interface(alg::Union{MOI.AbstractOptimizer, - MOI.OptimizerWithAttributes}) - true - end +function OptimizationBase.supports_opt_cache_interface(alg::Union{MOI.AbstractOptimizer, + MOI.OptimizerWithAttributes}) + true end function SciMLBase.__init(prob::OptimizationProblem, diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index ecae49c4e..a4f7d0100 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -12,12 +12,7 @@ internal state. """ abstract type AbstractManoptOptimizer end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::AbstractManoptOptimizer) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::AbstractManoptOptimizer) = true -end +OptimizationBase.supports_opt_cache_interface(opt::AbstractManoptOptimizer) = true function __map_optimizer_args!(cache::OptimizationBase.OptimizationCache, opt::AbstractManoptOptimizer; diff --git a/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl b/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl index 65ae4c935..045a10701 100644 --- a/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl +++ b/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl @@ -7,12 +7,7 @@ using SciMLBase SciMLBase.requiresbounds(opt::Metaheuristics.AbstractAlgorithm) = true SciMLBase.allowsbounds(opt::Metaheuristics.AbstractAlgorithm) = true SciMLBase.allowscallback(opt::Metaheuristics.AbstractAlgorithm) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Metaheuristics.AbstractAlgorithm) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Metaheuristics.AbstractAlgorithm) = true -end +OptimizationBase.supports_opt_cache_interface(opt::Metaheuristics.AbstractAlgorithm) = true function initial_population!(opt, cache, bounds, f) opt_init = deepcopy(opt) diff --git a/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl b/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl index 8bc79152f..6d8aba4f0 100644 --- a/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl +++ b/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl @@ -7,12 +7,7 @@ using SciMLBase SciMLBase.requiresbounds(opt::MultistartOptimization.TikTak) = true SciMLBase.allowsbounds(opt::MultistartOptimization.TikTak) = true SciMLBase.allowscallback(opt::MultistartOptimization.TikTak) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::MultistartOptimization.TikTak) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::MultistartOptimization.TikTak) = true -end +OptimizationBase.supports_opt_cache_interface(opt::MultistartOptimization.TikTak) = true function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, opt::MultistartOptimization.TikTak, diff --git a/lib/OptimizationNLopt/src/OptimizationNLopt.jl b/lib/OptimizationNLopt/src/OptimizationNLopt.jl index ee9640024..31787dcb3 100644 --- a/lib/OptimizationNLopt/src/OptimizationNLopt.jl +++ b/lib/OptimizationNLopt/src/OptimizationNLopt.jl @@ -8,12 +8,8 @@ using OptimizationBase: deduce_retcode (f::NLopt.Algorithm)() = f SciMLBase.allowsbounds(opt::Union{NLopt.Algorithm, NLopt.Opt}) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Union{NLopt.Algorithm, NLopt.Opt}) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Union{NLopt.Algorithm, NLopt.Opt}) = true -end + +OptimizationBase.supports_opt_cache_interface(opt::Union{NLopt.Algorithm, NLopt.Opt}) = true function SciMLBase.requiresgradient(opt::Union{NLopt.Algorithm, NLopt.Opt}) # https://github.com/JuliaOpt/NLopt.jl/blob/master/src/NLopt.jl#L18C7-L18C16 diff --git a/lib/OptimizationODE/src/OptimizationODE.jl b/lib/OptimizationODE/src/OptimizationODE.jl index eb8638b9b..97bbe63c1 100644 --- a/lib/OptimizationODE/src/OptimizationODE.jl +++ b/lib/OptimizationODE/src/OptimizationODE.jl @@ -30,7 +30,7 @@ DAEMassMatrix() = DAEOptimizer(Rodas5P(autodiff = false)) SciMLBase.requiresbounds(::ODEOptimizer) = false SciMLBase.allowsbounds(::ODEOptimizer) = false SciMLBase.allowscallback(::ODEOptimizer) = true -SciMLBase.supports_opt_cache_interface(::ODEOptimizer) = true +OptimizationBase.supports_opt_cache_interface(::ODEOptimizer) = true SciMLBase.requiresgradient(::ODEOptimizer) = true SciMLBase.requireshessian(::ODEOptimizer) = false SciMLBase.requiresconsjac(::ODEOptimizer) = false @@ -41,7 +41,7 @@ SciMLBase.requiresbounds(::DAEOptimizer) = false SciMLBase.allowsbounds(::DAEOptimizer) = false SciMLBase.allowsconstraints(::DAEOptimizer) = true SciMLBase.allowscallback(::DAEOptimizer) = true -SciMLBase.supports_opt_cache_interface(::DAEOptimizer) = true +OptimizationBase.supports_opt_cache_interface(::DAEOptimizer) = true SciMLBase.requiresgradient(::DAEOptimizer) = true SciMLBase.requireshessian(::DAEOptimizer) = false SciMLBase.requiresconsjac(::DAEOptimizer) = true @@ -109,7 +109,7 @@ function solve_ode(cache, dt, maxit, u0, p) else solve_kwargs = Dict{Symbol, Any}() end - + if !isnothing(maxit) solve_kwargs[:maxiters] = maxit end @@ -170,7 +170,7 @@ function solve_dae_mass_matrix(cache, dt, maxit, u0, p) else solve_kwargs = Dict{Symbol, Any}() end - + solve_kwargs[:progress] = cache.progress if maxit !== nothing; solve_kwargs[:maxiters] = maxit; end if dt !== nothing; solve_kwargs[:dt] = dt; end @@ -179,7 +179,7 @@ function solve_dae_mass_matrix(cache, dt, maxit, u0, p) # if sol.retcode ≠ ReturnCode.Success # # you may still accept Default or warn # end - u_ext = sol.u + u_ext = sol.u u_final = u_ext[1:n] return SciMLBase.build_solution(cache, cache.opt, u_final, cache.f(u_final, p); retcode = sol.retcode) @@ -221,7 +221,7 @@ function solve_dae_implicit(cache, dt, maxit, u0, p) else solve_kwargs = Dict{Symbol, Any}() end - + solve_kwargs[:progress] = cache.progress if maxit !== nothing; solve_kwargs[:maxiters] = maxit; end @@ -237,4 +237,4 @@ function solve_dae_implicit(cache, dt, maxit, u0, p) end -end +end diff --git a/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl b/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl index 8af15fdbb..79001e761 100644 --- a/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl +++ b/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl @@ -11,16 +11,9 @@ SciMLBase.allowsbounds(opt::Optim.AbstractOptimizer) = true SciMLBase.allowsbounds(opt::Optim.SimulatedAnnealing) = false SciMLBase.requiresbounds(opt::Optim.Fminbox) = true SciMLBase.requiresbounds(opt::Optim.SAMIN) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Optim.AbstractOptimizer) = true - SciMLBase.supports_opt_cache_interface(opt::Union{Optim.Fminbox, Optim.SAMIN}) = true - SciMLBase.supports_opt_cache_interface(opt::Optim.ConstrainedOptimizer) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Optim.AbstractOptimizer) = true - OptimizationBase.supports_opt_cache_interface(opt::Union{Optim.Fminbox, Optim.SAMIN}) = true - OptimizationBase.supports_opt_cache_interface(opt::Optim.ConstrainedOptimizer) = true -end +OptimizationBase.supports_opt_cache_interface(opt::Optim.AbstractOptimizer) = true +OptimizationBase.supports_opt_cache_interface(opt::Union{Optim.Fminbox, Optim.SAMIN}) = true +OptimizationBase.supports_opt_cache_interface(opt::Optim.ConstrainedOptimizer) = true function SciMLBase.requiresgradient(opt::Optim.AbstractOptimizer) !(opt isa Optim.ZerothOrderOptimizer) end diff --git a/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl b/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl index a86b23ac8..bf2af0479 100644 --- a/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl +++ b/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl @@ -4,12 +4,7 @@ using Reexport, Printf, ProgressLogging @reexport using Optimisers, OptimizationBase using SciMLBase -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::AbstractRule) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::AbstractRule) = true -end +OptimizationBase.supports_opt_cache_interface(opt::AbstractRule) = true SciMLBase.requiresgradient(opt::AbstractRule) = true SciMLBase.allowsfg(opt::AbstractRule) = true diff --git a/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl b/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl index 2ba274ea2..256855065 100644 --- a/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl +++ b/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl @@ -11,12 +11,7 @@ struct BOBYQA <: PRIMASolvers end struct LINCOA <: PRIMASolvers end struct COBYLA <: PRIMASolvers end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::PRIMASolvers) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::PRIMASolvers) = true -end +OptimizationBase.supports_opt_cache_interface(::PRIMASolvers) = true SciMLBase.allowsconstraints(::Union{LINCOA, COBYLA}) = true SciMLBase.allowsbounds(opt::Union{BOBYQA, LINCOA, COBYLA}) = true SciMLBase.requiresconstraints(opt::COBYLA) = true @@ -35,7 +30,7 @@ function OptimizationBase.OptimizationCache(prob::SciMLBase.OptimizationProblem, reinit_cache = OptimizationBase.ReInitCache(prob.u0, prob.p) num_cons = prob.ucons === nothing ? 0 : length(prob.ucons) if prob.f.adtype isa SciMLBase.NoAD && opt isa COBYLA - throw("We evaluate the jacobian and hessian of the constraints once to automatically detect + throw("We evaluate the jacobian and hessian of the constraints once to automatically detect linear and nonlinear constraints, please provide a valid AD backend for using COBYLA.") else if opt isa COBYLA diff --git a/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl b/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl index 312732b5f..4d78412a6 100644 --- a/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl +++ b/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl @@ -19,12 +19,7 @@ end # Defining the SciMLBase interface for PyCMAOpt SciMLBase.allowsbounds(::PyCMAOpt) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::PyCMAOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::PyCMAOpt) = true -end +OptimizationBase.supports_opt_cache_interface(opt::PyCMAOpt) = true SciMLBase.allowscallback(::PyCMAOpt) = true SciMLBase.requiresgradient(::PyCMAOpt) = false SciMLBase.requireshessian(::PyCMAOpt) = false @@ -43,7 +38,7 @@ function __map_optimizer_args(prob::OptimizationBase.OptimizationCache, opt::PyC end # Converting OptimizationBase.jl args to PyCMA opts - # OptimizationBase.jl kwargs will overwrite PyCMA kwargs supplied to solve() + # OptimizationBase.jl kwargs will overwrite PyCMA kwargs supplied to solve() mapped_args = Dict{String, Any}() diff --git a/lib/OptimizationSciPy/src/OptimizationSciPy.jl b/lib/OptimizationSciPy/src/OptimizationSciPy.jl index 712958b8d..8ba06ff79 100644 --- a/lib/OptimizationSciPy/src/OptimizationSciPy.jl +++ b/lib/OptimizationSciPy/src/OptimizationSciPy.jl @@ -216,32 +216,17 @@ for opt_type in [:ScipyMinimize, :ScipyDifferentialEvolution, :ScipyBasinhopping :ScipyLinprog, :ScipyMilp] @eval begin SciMLBase.allowsbounds(::$opt_type) = true - @static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::$opt_type) = true - end - @static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::$opt_type) = true - end + OptimizationBase.supports_opt_cache_interface(::$opt_type) = true end end for opt_type in [:ScipyMinimizeScalar, :ScipyRootScalar, :ScipyLeastSquares] @eval begin - @static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::$opt_type) = true - end - @static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::$opt_type) = true - end + OptimizationBase.supports_opt_cache_interface(::$opt_type) = true end end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::ScipyRoot) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::ScipyRoot) = true -end +OptimizationBase.supports_opt_cache_interface(::ScipyRoot) = true function SciMLBase.requiresgradient(opt::ScipyMinimize) gradient_free = ["Nelder-Mead", "Powell", "COBYLA", "COBYQA"] diff --git a/lib/OptimizationSophia/src/OptimizationSophia.jl b/lib/OptimizationSophia/src/OptimizationSophia.jl index f02dfff1e..51c5c8206 100644 --- a/lib/OptimizationSophia/src/OptimizationSophia.jl +++ b/lib/OptimizationSophia/src/OptimizationSophia.jl @@ -10,15 +10,15 @@ using Random A second-order optimizer that incorporates diagonal Hessian information for faster convergence. -Based on the paper "Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training" +Based on the paper "Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training" (https://arxiv.org/abs/2305.14342). Sophia uses an efficient estimate of the diagonal of the Hessian -matrix to adaptively adjust the learning rate for each parameter, achieving faster convergence than +matrix to adaptively adjust the learning rate for each parameter, achieving faster convergence than first-order methods like Adam and SGD while avoiding the computational cost of full second-order methods. ## Arguments - `η::Float64 = 1e-3`: Learning rate (step size) - - `βs::Tuple{Float64, Float64} = (0.9, 0.999)`: Exponential decay rates for the first moment (β₁) + - `βs::Tuple{Float64, Float64} = (0.9, 0.999)`: Exponential decay rates for the first moment (β₁) and diagonal Hessian (β₂) estimates - `ϵ::Float64 = 1e-8`: Small constant for numerical stability - `λ::Float64 = 1e-1`: Weight decay coefficient for L2 regularization @@ -46,9 +46,9 @@ Sophia is particularly effective for: - Large-scale optimization problems - Neural network training - Problems where second-order information can significantly improve convergence - + The algorithm maintains computational efficiency by only estimating the diagonal of the Hessian -matrix using a Hutchinson trace estimator with random vectors, making it more scalable than +matrix using a Hutchinson trace estimator with random vectors, making it more scalable than full second-order methods while still leveraging curvature information. """ struct Sophia @@ -60,12 +60,7 @@ struct Sophia ρ::Float64 end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Sophia) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Sophia) = true -end +OptimizationBase.supports_opt_cache_interface(opt::Sophia) = true SciMLBase.requiresgradient(opt::Sophia) = true SciMLBase.allowsfg(opt::Sophia) = true SciMLBase.requireshessian(opt::Sophia) = true diff --git a/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl b/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl index e7fd92d10..f9469a506 100644 --- a/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl +++ b/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl @@ -10,12 +10,7 @@ struct SpeedMappingOpt end SciMLBase.allowsbounds(::SpeedMappingOpt) = true SciMLBase.allowscallback(::SpeedMappingOpt) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::SpeedMappingOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::SpeedMappingOpt) = true -end +OptimizationBase.supports_opt_cache_interface(opt::SpeedMappingOpt) = true SciMLBase.requiresgradient(opt::SpeedMappingOpt) = true function __map_optimizer_args(cache::OptimizationBase.OptimizationCache, opt::SpeedMappingOpt;