Merge branch 'master' into kellertuer/properManopt

Author: oscardssmith

Project.toml

@@ -1,6 +1,6 @@
 name = "Optimization"
 uuid = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
-version = "4.7.0"
+version = "4.8.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -20,6 +20,9 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed"
 
+[sources]
+OptimizationBase = {path = "lib/OptimizationBase"}
+
 [compat]
 ADTypes = "1.2"
 Aqua = "0.8"
@@ -42,7 +45,7 @@ Logging = "1.10"
 LoggingExtras = "0.4, 1"
 Lux = "1.12.4"
 MLUtils = "0.4"
-ModelingToolkit = "10"
+ModelingToolkit = "10.23"
 Mooncake = "0.4.138"
 Optim = ">= 1.4.1"
 OptimizationBase = "2"
@@ -57,10 +60,9 @@ Random = "1.10"
 Reexport = "1.2"
 ReverseDiff = "1"
 SafeTestsets = "0.1"
-SciMLBase = "2.82"
+SciMLBase = "2.104"
 SciMLSensitivity = "7"
 SparseArrays = "1.10"
-SparseDiffTools = "2"
 Symbolics = "6"
 TerminalLoggers = "0.1"
 Test = "1.10"
@@ -96,7 +98,6 @@ ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
-SparseDiffTools = "47a9eef4-7e08-11e9-0b38-333d64bd3804"
 Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
@@ -106,5 +107,5 @@ Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6"
 [targets]
 test = ["Aqua", "BenchmarkTools", "Boltz", "ComponentArrays", "DiffEqFlux", "Enzyme", "FiniteDiff", "Flux", "ForwardDiff",
     "Ipopt", "IterTools", "Lux", "MLUtils", "ModelingToolkit", "Optim", "OptimizationMOI", "OptimizationOptimJL", "OptimizationOptimisers", 
-    "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReverseDiff", "SafeTestsets", "SciMLSensitivity", "SparseArrays", "SparseDiffTools",
+    "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReverseDiff", "SafeTestsets", "SciMLSensitivity", "SparseArrays",
     "Symbolics",  "Test", "Tracker", "Zygote", "Mooncake"]

docs/Project.toml

@@ -1,4 +1,5 @@
 [deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 AmplNLWriter = "7c4d4715-977e-5154-bfe0-e096adeac482"
 ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
@@ -60,7 +61,7 @@ Lux = "1"
 MLUtils = "0.4.4"
 Manifolds = "0.10"
 Manopt = "0.5"
-ModelingToolkit = "10"
+ModelingToolkit = "10.23"
 NLPModels = "0.21"
 NLPModelsTest = "0.10"
 NLopt = "0.6, 1"

docs/make.jl

@@ -1,5 +1,6 @@
 using Documenter, Optimization
 using FiniteDiff, ForwardDiff, ModelingToolkit, ReverseDiff, Tracker, Zygote
+using ADTypes
 
 cp("./docs/Manifest.toml", "./docs/src/assets/Manifest.toml", force = true)
 cp("./docs/Project.toml", "./docs/src/assets/Project.toml", force = true)
@@ -9,9 +10,9 @@ include("pages.jl")
 makedocs(sitename = "Optimization.jl",
     authors = "Chris Rackauckas, Vaibhav Kumar Dixit et al.",
     modules = [Optimization, Optimization.SciMLBase, Optimization.OptimizationBase,
-        FiniteDiff, ForwardDiff, ModelingToolkit, ReverseDiff, Tracker, Zygote],
+        FiniteDiff, ForwardDiff, ModelingToolkit, ReverseDiff, Tracker, Zygote, ADTypes],
     clean = true, doctest = false, linkcheck = true,
-    warnonly = [:missing_docs],
+    warnonly = [:missing_docs, :cross_references],
     format = Documenter.HTML(assets = ["assets/favicon.ico"],
         canonical = "https://docs.sciml.ai/Optimization/stable/"),
     pages = pages)

docs/src/API/ad.md

@@ -23,5 +23,5 @@ OptimizationBase.AutoZygote
 OptimizationBase.AutoTracker
 OptimizationBase.AutoModelingToolkit
 OptimizationBase.AutoEnzyme
-OptimizationBase.AutoMooncake
+ADTypes.AutoMooncake
 ```

docs/src/optimization_packages/optimization.md

@@ -9,7 +9,7 @@ There are some solvers that are available in the Optimization.jl package directl
     This can also handle arbitrary non-linear constraints through a Augmented Lagrangian method with bounds constraints described in 17.4 of Numerical Optimization by Nocedal and Wright. Thus serving as a general-purpose nonlinear optimization solver available directly in Optimization.jl.
 
 ```@docs
-Sophia
+Optimization.Sophia
 ```
 
 ## Examples

lib/OptimizationBBO/Project.toml

@@ -1,7 +1,7 @@
 name = "OptimizationBBO"
 uuid = "3e6eede4-6085-4f62-9a71-46d9bc1eb92b"
 authors = ["Vaibhav Dixit <[email protected]> and contributors"]
-version = "0.4.2"
+version = "0.4.3"
 
 [deps]
 BlackBoxOptim = "a134a8b2-14d6-55f6-9291-3336d3ab0209"

lib/OptimizationBBO/src/OptimizationBBO.jl

@@ -2,6 +2,7 @@ module OptimizationBBO
 
 using Reexport
 import Optimization
+import Optimization: OptimizationBase
 import BlackBoxOptim, Optimization.SciMLBase
 import Optimization.SciMLBase: MultiObjectiveOptimizationFunction
 

lib/OptimizationBase/Project.toml

@@ -1,7 +1,7 @@
 name = "OptimizationBase"
 uuid = "bca83a33-5cc9-4baa-983d-23429ab6bcbb"
 authors = ["Vaibhav Dixit <[email protected]> and contributors"]
-version = "2.11.0"
+version = "2.12.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -51,11 +51,11 @@ ForwardDiff = "0.10.26, 1"
 LinearAlgebra = "1.9, 1.10"
 MLDataDevices = "1"
 MLUtils = "0.4"
-ModelingToolkit = "9, 10"
+ModelingToolkit = "10.23"
 PDMats = "0.11"
 Reexport = "1.2"
 ReverseDiff = "1.14"
-SciMLBase = "2"
+SciMLBase = "2.104"
 SparseConnectivityTracer = "0.6, 1"
 SparseMatrixColorings = "0.4"
 SymbolicAnalysis = "0.3"

lib/OptimizationBase/ext/OptimizationEnzymeExt.jl

@@ -28,16 +28,15 @@ function inner_grad(mode::Mode, θ, bθ, f, p) where {Mode}
     return nothing
 end
 
-function hv_f2_alloc(mode::Mode, x, f, p) where {Mode}
-    dx = Enzyme.make_zero(x)
+function hv_f2_alloc(mode::Mode, xdup, f, p) where {Mode}
     Enzyme.autodiff(mode,
         Const(firstapply),
         Active,
         Const(f),
-        Enzyme.Duplicated(x, dx),
+        xdup,
         Const(p)
     )
-    return dx
+    return xdup
 end
 
 function inner_cons(x, fcons::Function, p::Union{SciMLBase.NullParameters, Nothing},
@@ -200,10 +199,12 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
 
     if hv == true && f.hv === nothing
         function hv!(H, θ, v, p = p)
+            x = Duplicated(θ, v)
+            dx = Enzyme.make_zero(x)
             H .= Enzyme.autodiff(
-                fmode, hv_f2_alloc, Const(rmode), Duplicated(θ, v),
+                fmode, hv_f2_alloc, Const(rmode), Duplicated(x,dx),
                 Const(f.f), Const(p)
-            )[1]
+            )[1].dval
         end
     elseif hv == true
         hv! = (H, θ, v, p = p) -> f.hv(H, θ, v, p)
@@ -553,10 +554,12 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
 
     if hv == true && f.hv === nothing
         function hv!(θ, v, p = p)
+            x = Duplicated(θ, v)
+            dx = Enzyme.make_zero(x)
             return Enzyme.autodiff(
-                fmode, hv_f2_alloc, DuplicatedNoNeed, Const(rmode), Duplicated(θ, v),
+                fmode, hv_f2_alloc, DuplicatedNoNeed, Const(rmode), Duplicated(x, dx), 
                 Const(_f), Const(f.f), Const(p)
-            )[1]
+            )[1].dval
         end
     elseif hv == true
         hv! = (θ, v, p = p) -> f.hv(θ, v, p)

lib/OptimizationBase/ext/OptimizationMTKExt.jl

@@ -21,24 +21,25 @@ function OptimizationBase.instantiate_function(
             num_cons))))
     #sys = ModelingToolkit.structural_simplify(sys)
     # don't need to pass `x` or `p` since they're defaults now
-    f = OptimizationProblem(sys, nothing; grad = g, hess = h,
+    mtkprob = OptimizationProblem(sys, nothing; grad = g, hess = h,
         sparse = true, cons_j = cons_j, cons_h = cons_h,
-        cons_sparse = true).f
+        cons_sparse = true)
+    f = mtkprob.f
 
-    grad = (G, θ, args...) -> f.grad(G, θ, p, args...)
+    grad = (G, θ, args...) -> f.grad(G, θ, mtkprob.p, args...)
 
-    hess = (H, θ, args...) -> f.hess(H, θ, p, args...)
+    hess = (H, θ, args...) -> f.hess(H, θ, mtkprob.p, args...)
 
     hv = function (H, θ, v, args...)
-        res = (eltype(θ)).(f.hess_prototype)
+        res = similar(f.hess_prototype, eltype(θ))
         hess(res, θ, args...)
         H .= res * v
     end
 
     if !isnothing(f.cons)
-        cons = (res, θ) -> f.cons(res, θ, p)
-        cons_j = (J, θ) -> f.cons_j(J, θ, p)
-        cons_h = (res, θ) -> f.cons_h(res, θ, p)
+        cons = (res, θ) -> f.cons(res, θ, mtkprob.p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, mtkprob.p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, mtkprob.p)
     else
         cons = nothing
         cons_j = nothing
@@ -72,24 +73,24 @@ function OptimizationBase.instantiate_function(
             num_cons))))
     #sys = ModelingToolkit.structural_simplify(sys)
     # don't need to pass `x` or `p` since they're defaults now
-    f = OptimizationProblem(sys, nothing; grad = g, hess = h,
+    mtkprob = OptimizationProblem(sys, nothing; grad = g, hess = h,
         sparse = true, cons_j = cons_j, cons_h = cons_h,
-        cons_sparse = true).f
+        cons_sparse = true)
+    f = mtkprob.f
 
-    grad = (G, θ, args...) -> f.grad(G, θ, cache.p, args...)
+    grad = (G, θ, args...) -> f.grad(G, θ, mtkprob.p, args...)
 
-    hess = (H, θ, args...) -> f.hess(H, θ, cache.p, args...)
+    hess = (H, θ, args...) -> f.hess(H, θ, mtkprob.p, args...)
 
     hv = function (H, θ, v, args...)
-        res = (eltype(θ)).(f.hess_prototype)
+        res = similar(f.hess_prototype, eltype(θ))
         hess(res, θ, args...)
         H .= res * v
     end
-
     if !isnothing(f.cons)
-        cons = (res, θ) -> f.cons(res, θ, cache.p)
-        cons_j = (J, θ) -> f.cons_j(J, θ, cache.p)
-        cons_h = (res, θ) -> f.cons_h(res, θ, cache.p)
+        cons = (res, θ) -> f.cons(res, θ, mtkprob.p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, mtkprob.p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, mtkprob.p)
     else
         cons = nothing
         cons_j = nothing
@@ -121,13 +122,14 @@ function OptimizationBase.instantiate_function(
             num_cons))))
     #sys = ModelingToolkit.structural_simplify(sys)
     # don't need to pass `x` or `p` since they're defaults now
-    f = OptimizationProblem(sys, nothing; grad = g, hess = h,
+    mtkprob = OptimizationProblem(sys, nothing; grad = g, hess = h,
         sparse = false, cons_j = cons_j, cons_h = cons_h,
-        cons_sparse = false).f
+        cons_sparse = false)
+    f = mtkprob.f
 
-    grad = (G, θ, args...) -> f.grad(G, θ, p, args...)
+    grad = (G, θ, args...) -> f.grad(G, θ, mtkprob.p, args...)
 
-    hess = (H, θ, args...) -> f.hess(H, θ, p, args...)
+    hess = (H, θ, args...) -> f.hess(H, θ, mtkprob.p, args...)
 
     hv = function (H, θ, v, args...)
         res = ArrayInterface.zeromatrix(θ)
@@ -136,9 +138,9 @@ function OptimizationBase.instantiate_function(
     end
 
     if !isnothing(f.cons)
-        cons = (res, θ) -> f.cons(res, θ, p)
-        cons_j = (J, θ) -> f.cons_j(J, θ, p)
-        cons_h = (res, θ) -> f.cons_h(res, θ, p)
+        cons = (res, θ) -> f.cons(res, θ, mtkprob.p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, mtkprob.p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, mtkprob.p)
     else
         cons = nothing
         cons_j = nothing
@@ -172,13 +174,14 @@ function OptimizationBase.instantiate_function(
             num_cons))))
     #sys = ModelingToolkit.structural_simplify(sys)
     # don't need to pass `x` or `p` since they're defaults now
-    f = OptimizationProblem(sys, nothing; grad = g, hess = h,
+    mtkprob = OptimizationProblem(sys, nothing; grad = g, hess = h,
         sparse = false, cons_j = cons_j, cons_h = cons_h,
-        cons_sparse = false).f
+        cons_sparse = false)
+    f = mtkprob.f
 
-    grad = (G, θ, args...) -> f.grad(G, θ, cache.p, args...)
+    grad = (G, θ, args...) -> f.grad(G, θ, mtkprob.p, args...)
 
-    hess = (H, θ, args...) -> f.hess(H, θ, cache.p, args...)
+    hess = (H, θ, args...) -> f.hess(H, θ, mtkprob.p, args...)
 
     hv = function (H, θ, v, args...)
         res = ArrayInterface.zeromatrix(θ)
@@ -187,9 +190,9 @@ function OptimizationBase.instantiate_function(
     end
 
     if !isnothing(f.cons)
-        cons = (res, θ) -> f.cons(res, θ, cache.p)
-        cons_j = (J, θ) -> f.cons_j(J, θ, cache.p)
-        cons_h = (res, θ) -> f.cons_h(res, θ, cache.p)
+        cons = (res, θ) -> f.cons(res, θ, mtkprob.p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, mtkprob.p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, mtkprob.p)
     else
         cons = nothing
         cons_j = nothing