New Subpackage for LBFGS

Author: ParamThakkar123

lib/OptimizationLBFGS/Project.toml

@@ -0,0 +1,24 @@
+name = "OptimizationLBFGS"
+uuid = "22f7324a-a79d-40f2-bebe-3af60c77bd15"
+authors = ["paramthakkar123 <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LBFGSB = "5be7bae1-8223-5378-bac3-9e7378a2f6e6"
+MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
+Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
+OptimizationBase = "bca83a33-5cc9-4baa-983d-23429ab6bcbb"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+
+[compat]
+DocStringExtensions = "0.9.5"
+ForwardDiff = "1.0.1"
+LBFGSB = "0.4.1"
+MLUtils = "0.4.8"
+Optimization = "4.4.0"
+OptimizationBase = "2.10.0"
+Test = "1.11.0"
+Zygote = "0.7.10"

src/lbfgsb.jl renamed to lib/OptimizationLBFGS/src/OptimizationLBFGS.jl

@@ -1,4 +1,12 @@
-using Optimization.SciMLBase, LBFGSB
+module OptimizationLBFGS
+
+using Optimization
+using DocStringExtensions
+using LBFGSB
+using OptimizationBase.SciMLBase: OptimizationStats, OptimizationFunction
+using OptimizationBase: ReturnCode
+using OptimizationBase.LinearAlgebra: norm
+using Optimization: deduce_retcode
 
 """
 $(TYPEDEF)
@@ -33,7 +41,7 @@ function task_message_to_string(task::Vector{UInt8})
     return String(task)
 end
 
-function __map_optimizer_args(cache::Optimization.OptimizationCache, opt::LBFGS;
+function __map_optimizer_args(cache::OptimizationCache, opt::LBFGS;
         callback = nothing,
         maxiters::Union{Number, Nothing} = nothing,
         maxtime::Union{Number, Nothing} = nothing,
@@ -125,7 +133,7 @@ function SciMLBase.__solve(cache::OptimizationCache{
             cons_tmp[eq_inds] .= cons_tmp[eq_inds] - cache.lcons[eq_inds]
             cons_tmp[ineq_inds] .= cons_tmp[ineq_inds] .- cache.ucons[ineq_inds]
             opt_state = Optimization.OptimizationState(
-                u = θ, objective = x[1], p = cache.p, iter = iter_count[])
+                u = θ, objective = x[1])
             if cache.callback(opt_state, x...)
                 error("Optimization halted by callback.")
             end
@@ -205,7 +213,7 @@ function SciMLBase.__solve(cache::OptimizationCache{
             end
         end
 
-        stats = Optimization.OptimizationStats(; iterations = maxiters,
+        stats = OptimizationStats(; iterations = maxiters,
             time = 0.0, fevals = maxiters, gevals = maxiters)
         return SciMLBase.build_solution(
             cache, cache.opt, res[2], cache.f(res[2], cache.p)[1],
@@ -216,7 +224,7 @@ function SciMLBase.__solve(cache::OptimizationCache{
             x = cache.f(θ, cache.p)
             iter_count[] += 1
             opt_state = Optimization.OptimizationState(
-                u = θ, objective = x[1], p = cache.p, iter = iter_count[])
+                u = θ, objective = x[1])
             if cache.callback(opt_state, x...)
                 error("Optimization halted by callback.")
             end
@@ -257,3 +265,5 @@ function SciMLBase.__solve(cache::OptimizationCache{
             retcode = opt_ret, original = optimizer)
     end
 end
+
+end

lib/OptimizationLBFGS/test/runtests.jl

@@ -0,0 +1,55 @@
+using OptimizationBase
+using OptimizationBase: ReturnCode
+using OptimizationBase.SciMLBase: OptimizationFunction, OptimizationProblem
+using ForwardDiff, Zygote
+using OptimizationLBFGS
+using MLUtils
+using LBFGSB
+using Test
+
+x0 = zeros(2)
+rosenbrock(x, p = nothing) = (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2
+l1 = rosenbrock(x0)
+
+optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoForwardDiff())
+prob = OptimizationProblem(optf, x0)
+@time res = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 100)
+@test res.retcode == ReturnCode.Success
+
+prob = OptimizationProblem(optf, x0, lb = [-1.0, -1.0], ub = [1.0, 1.0])
+@time res = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 100)
+@test res.retcode == ReturnCode.Success
+
+function con2_c(res, x, p)
+    res .= [x[1]^2 + x[2]^2, (x[2] * sin(x[1]) + x[1]) - 5]
+end
+
+optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoZygote(), cons = con2_c)
+prob = OptimizationProblem(optf, x0, lcons = [1.0, -Inf],
+    ucons = [1.0, 0.0], lb = [-1.0, -1.0],
+    ub = [1.0, 1.0])
+@time res = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 100)
+@test res.retcode == SciMLBase.ReturnCode.Success
+
+x0 = (-pi):0.001:pi
+y0 = sin.(x0)
+data = MLUtils.DataLoader((x0, y0), batchsize = 126)
+function loss(coeffs, data)
+    ypred = [evalpoly(data[1][i], coeffs) for i in eachindex(data[1])]
+    return sum(abs2, ypred .- data[2])
+end
+
+function cons1(res, coeffs, p = nothing)
+    res[1] = coeffs[1] * coeffs[5] - 1
+    return nothing
+end
+
+optf = OptimizationFunction(loss, AutoSparseForwardDiff(), cons = cons1)
+callback = (st, l) -> (@show l; return false)
+
+initpars = rand(5)
+l0 = optf(initpars, (x0, y0))
+prob = OptimizationProblem(optf, initpars, (x0, y0), lcons = [-Inf], ucons = [0.5],
+    lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])
+opt1 = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 1000, callback = callback)
+@test opt1.objective < l0

src/Optimization.jl

@@ -22,7 +22,6 @@ export ObjSense, MaxSense, MinSense
 
 include("utils.jl")
 include("state.jl")
-include("lbfgsb.jl")
 include("sophia.jl")
 include("auglag.jl")
 

test/native.jl

@@ -1,31 +1,6 @@
 using Optimization
 using ForwardDiff, Zygote, ReverseDiff, FiniteDiff
 using Test
-
-x0 = zeros(2)
-rosenbrock(x, p = nothing) = (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2
-l1 = rosenbrock(x0)
-
-optf = OptimizationFunction(rosenbrock, AutoForwardDiff())
-prob = OptimizationProblem(optf, x0)
-@time res = solve(prob, Optimization.LBFGS(), maxiters = 100)
-@test res.retcode == Optimization.SciMLBase.ReturnCode.Success
-
-prob = OptimizationProblem(optf, x0, lb = [-1.0, -1.0], ub = [1.0, 1.0])
-@time res = solve(prob, Optimization.LBFGS(), maxiters = 100)
-@test res.retcode == Optimization.SciMLBase.ReturnCode.Success
-
-function con2_c(res, x, p)
-    res .= [x[1]^2 + x[2]^2, (x[2] * sin(x[1]) + x[1]) - 5]
-end
-
-optf = OptimizationFunction(rosenbrock, AutoZygote(), cons = con2_c)
-prob = OptimizationProblem(optf, x0, lcons = [1.0, -Inf],
-    ucons = [1.0, 0.0], lb = [-1.0, -1.0],
-    ub = [1.0, 1.0])
-@time res = solve(prob, Optimization.LBFGS(), maxiters = 100)
-@test res.retcode == SciMLBase.ReturnCode.Success
-
 using MLUtils, OptimizationOptimisers
 
 x0 = (-pi):0.001:pi
@@ -46,10 +21,6 @@ callback = (st, l) -> (@show l; return false)
 
 initpars = rand(5)
 l0 = optf(initpars, (x0, y0))
-prob = OptimizationProblem(optf, initpars, (x0, y0), lcons = [-Inf], ucons = [0.5],
-    lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])
-opt1 = solve(prob, Optimization.LBFGS(), maxiters = 1000, callback = callback)
-@test opt1.objective < l0
 
 prob = OptimizationProblem(optf, initpars, data, lcons = [-Inf], ucons = [1],
     lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])