Merge branch 'master' into nloptcons

Author: Vaibhavdixit02

.github/workflows/CI.yml

@@ -22,7 +22,6 @@ jobs:
           - OptimizationBBO
           - OptimizationCMAEvolutionStrategy
           - OptimizationEvolutionary
-          - OptimizationFlux
           - OptimizationGCMAES
           - OptimizationManopt
           - OptimizationMetaheuristics
@@ -36,6 +35,7 @@ jobs:
           - OptimizationQuadDIRECT
           - OptimizationSpeedMapping
           - OptimizationPolyalgorithms
+          - OptimizationNLPModels
         version:
           - '1'
     steps:
@@ -61,7 +61,7 @@ jobs:
           GROUP: ${{ matrix.group }}
       - uses: julia-actions/julia-processcoverage@v1
         with:
-          directories: src,lib/OptimizationBBO/src,lib/OptimizationCMAEvolutionStrategy/src,lib/OptimizationEvolutionary/src,lib/OptimizationFlux/src,lib/OptimizationGCMAES/src,lib/OptimizationMOI/src,lib/OptimizationMetaheuristics/src,lib/OptimizationMultistartOptimization/src,lib/OptimizationNLopt/src,lib/OptimizationNOMAD/src,lib/OptimizationOptimJL/src,lib/OptimizationOptimisers/src,lib/OptimizationPolyalgorithms/src,lib/OptimizationQuadDIRECT/src,lib/OptimizationSpeedMapping/src
+          directories: src,lib/OptimizationBBO/src,lib/OptimizationCMAEvolutionStrategy/src,lib/OptimizationEvolutionary/src,lib/OptimizationGCMAES/src,lib/OptimizationManopt/src,lib/OptimizationMOI/src,lib/OptimizationMetaheuristics/src,lib/OptimizationMultistartOptimization/src,lib/OptimizationNLopt/src,lib/OptimizationNOMAD/src,lib/OptimizationOptimJL/src,lib/OptimizationOptimisers/src,lib/OptimizationPolyalgorithms/src,lib/OptimizationQuadDIRECT/src,lib/OptimizationSpeedMapping/src
       - uses: codecov/codecov-action@v4
         with:
           file: lcov.info

Project.toml

@@ -1,6 +1,6 @@
 name = "Optimization"
 uuid = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
-version = "3.27.0"
+version = "3.28.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -11,8 +11,8 @@ LBFGSB = "5be7bae1-8223-5378-bac3-9e7378a2f6e6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
+MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
 OptimizationBase = "bca83a33-5cc9-4baa-983d-23429ab6bcbb"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 ProgressLogging = "33c8b6b6-d38a-422a-b730-caa89a2f386c"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
@@ -29,8 +29,8 @@ LBFGSB = "0.4.1"
 LinearAlgebra = "1.10"
 Logging = "1.10"
 LoggingExtras = "0.4, 1"
-OptimizationBase = "1.3.3"
-Pkg = "1"
+MLUtils = "0.4.4"
+OptimizationBase = "2.0.3"
 Printf = "1.10"
 ProgressLogging = "0.1"
 Reexport = "1.2"

docs/Project.toml

@@ -12,6 +12,8 @@ Juniper = "2ddba703-00a4-53a7-87a5-e8b9971dde84"
 Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e"
 Manopt = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5"
 ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
+NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6"
+NLPModelsTest = "7998695d-6960-4d3a-85c4-e1bceb8cd856"
 NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd"
 Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
 OptimizationBBO = "3e6eede4-6085-4f62-9a71-46d9bc1eb92b"
@@ -24,6 +26,7 @@ OptimizationMOI = "fd9f6733-72f4-499f-8506-86b2bdd0dea1"
 OptimizationManopt = "e57b7fff-7ee7-4550-b4f0-90e9476e9fb6"
 OptimizationMetaheuristics = "3aafef2f-86ae-4776-b337-85a36adf0b55"
 OptimizationMultistartOptimization = "e4316d97-8bbb-4fd3-a7d8-3851d2a72823"
+OptimizationNLPModels = "064b21be-54cf-11ef-1646-cdfee32b588f"
 OptimizationNLopt = "4e6fcdb7-1186-4e1f-a706-475e75c168bb"
 OptimizationNOMAD = "2cab0595-8222-4775-b714-9828e6a9e01b"
 OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e"
@@ -53,6 +56,8 @@ Juniper = "0.9"
 Manifolds = "0.9"
 Manopt = "0.4"
 ModelingToolkit = "9"
+NLPModels = "0.21"
+NLPModelsTest = "0.10"
 NLopt = "0.6, 1"
 Optimization = "3"
 OptimizationBBO = "0.1, 0.2, 0.3"
@@ -65,6 +70,7 @@ OptimizationMOI = "0.1, 0.2, 0.3, 0.4"
 OptimizationManopt = "0.0.2, 0.0.3"
 OptimizationMetaheuristics = "0.1, 0.2"
 OptimizationMultistartOptimization = "0.1, 0.2"
+OptimizationNLPModels = "0.0.1"
 OptimizationNLopt = "0.1, 0.2"
 OptimizationNOMAD = "0.1, 0.2"
 OptimizationOptimJL = "0.1, 0.2, 0.3"
@@ -76,5 +82,7 @@ OrdinaryDiffEq = "6"
 ReverseDiff = ">= 1.9.0"
 SciMLBase = "2.30.0"
 SciMLSensitivity = "7"
+SymbolicAnalysis = "0.3"
+Symbolics = "6"
 Tracker = ">= 0.2"
 Zygote = ">= 0.5"

docs/pages.jl

@@ -36,6 +36,7 @@ pages = ["index.md",
         "PRIMA.jl" => "optimization_packages/prima.md",
         "Polyalgorithms.jl" => "optimization_packages/polyopt.md",
         "QuadDIRECT.jl" => "optimization_packages/quaddirect.md",
-        "SpeedMapping.jl" => "optimization_packages/speedmapping.md"
+        "SpeedMapping.jl" => "optimization_packages/speedmapping.md",
+        "NLPModels.jl" => "optimization_packages/nlpmodels.md"
     ]
 ]

docs/src/index.md

@@ -170,9 +170,7 @@ to add the specific wrapper packages.
     - Unconstrained: ✅
 </details>
 🟡 = supported in downstream library but not yet implemented in `Optimization.jl`; PR to add this functionality are welcome
-
 ## Citation
-
 ```
 @software{vaibhav_kumar_dixit_2023_7738525,
 	author = {Vaibhav Kumar Dixit and Christopher Rackauckas},
@@ -184,48 +182,37 @@ to add the specific wrapper packages.
   	url = {https://doi.org/10.5281/zenodo.7738525},
 	year = 2023}
 ```
-
 ## Reproducibility
-
 ```@raw html
 <details><summary>The documentation of this SciML package was built using these direct dependencies,</summary>
 ```
-
 ```@example
 using Pkg # hide
 Pkg.status() # hide
 ```
-
 ```@raw html
 </details>
 ```
-
 ```@raw html
 <details><summary>and using this machine and Julia version.</summary>
 ```
-
 ```@example
 using InteractiveUtils # hide
 versioninfo() # hide
 ```
-
 ```@raw html
 </details>
 ```
-
 ```@raw html
 <details><summary>A more complete overview of all dependencies and their versions is also provided.</summary>
 ```
-
 ```@example
 using Pkg # hide
 Pkg.status(; mode = PKGMODE_MANIFEST) # hide
 ```
-
 ```@raw html
 </details>
 ```
-
 ```@eval
 using TOML
 using Markdown

docs/src/optimization_packages/nlpmodels.md

@@ -0,0 +1,54 @@
+# NLPModels.jl
+
+[NLPModels](https://jso.dev/NLPModels.jl/latest/), similarly to Optimization.jl itself,
+provides a standardized modeling interface for representing Non-Linear Programs that
+facilitates using different solvers on the same problem. The Optimization.jl extension of
+NLPModels aims to provide a thin translation layer to make `NLPModel`s, the main export of
+the package, compatible with the optimizers in the Optimization.jl ecosystem.
+
+## Installation: NLPModels.jl
+
+To translate an `NLPModel`, install the OptimizationNLPModels package:
+
+```julia
+import Pkg;
+Pkg.add("OptimizationNLPModels")
+```
+
+The package NLPModels.jl itself contains no optimizers or models. Several packages
+provide optimization problem ([CUTEst.jl](https://jso.dev/CUTEst.jl/stable/),
+[NLPModelsTest.jl](https://jso.dev/NLPModelsTest.jl/dev/)) which can then be solved with
+any optimizer supported by Optimization.jl
+
+## Usage
+
+For example, solving a problem defined in `NLPModelsTest` with
+[`Ipopt.jl`](https://github.com/jump-dev/Ipopt.jl). First, install the packages like so:
+
+```julia
+import Pkg;
+Pkg.add("NLPModelsTest", "Ipopt")
+```
+
+We instantiate [problem
+10](https://jso.dev/NLPModelsTest.jl/dev/reference/#NLPModelsTest.HS10) in the
+Hock--Schittkowski optimization suite available from `NLPModelsTest` as `HS10`, then
+translate it to an `OptimizationProblem`.
+
+```@example NLPModels
+using OptimizationNLPModels, Optimization, NLPModelsTest, Ipopt
+using Optimization: OptimizationProblem
+nlpmodel = NLPModelsTest.HS10()
+prob = OptimizationProblem(nlpmodel, AutoForwardDiff())
+```
+
+which can now be solved like any other `OptimizationProblem`:
+
+```@example NLPModels
+sol = solve(prob, Ipopt.Optimizer())
+```
+
+## API
+
+Problems represented as `NLPModel`s can be used to create [`OptimizationProblem`](@ref)s and
+[`OptimizationFunction`](@ref).

docs/src/tutorials/remakecomposition.md

@@ -0,0 +1,55 @@
+# Creating polyalgorithms by chaining solvers using `remake`
+
+The general framework of using multiple solvers to use exploration-convergence alternations is commonly
+known as polyalgorithms. In the past Optimization.jl has provided a `PolyOpt` solver in [`OptimizationPolyalgorithms.jl`](@ref) which combined Adam from Optimisers.jl with BFGS from Optim.jl.
+With the large number of choices available through the interface unique combinations of solvers can be effective for specific problems.
+
+In this tutorial we will demonstrate how to use the `remake` function to chain together solvers to create your own polyalgorithms.
+
+The SciML interface provides a `remake` function which allows you to recreate the `OptimizationProblem` from a previously defined `OptimizationProblem` with different initial guess for the optimization variables.
+
+Let's look at a 10 dimensional schwefel function in the hypercube $x_i \in [-500, 500]$.
+
+```@example polyalg
+using Optimization, Random
+using OptimizationBBO, ReverseDiff
+
+Random.seed!(122333)
+
+function f_schwefel(x, p = [418.9829])
+    result = p[1] * length(x)
+    for i in 1:length(x)
+        result -= x[i] * sin(sqrt(abs(x[i])))
+    end
+    return result
+end
+
+optf = OptimizationFunction(f_schwefel, Optimization.AutoReverseDiff(compile = true))
+
+x0 = ones(10) .* 200.0
+prob = OptimizationProblem(
+    optf, x0, [418.9829], lb = fill(-500.0, 10), ub = fill(500.0, 10))
+
+@show f_schwefel(x0)
+```
+
+Our polyalgorithm strategy will to use BlackBoxOptim's global optimizers for efficient exploration of the
+parameter space followed by a quasi-Newton LBFGS method to (hopefully) converge to the global
+optimum.
+
+```@example polyalg
+res1 = solve(prob, BBO_adaptive_de_rand_1_bin(), maxiters = 4000)
+
+@show res1.objective
+```
+
+This is a good start can we converge to the global optimum?
+
+```@example polyalg
+prob = remake(prob, u0 = res1.minimizer)
+res2 = solve(prob, Optimization.LBFGS(), maxiters = 100)
+
+@show res2.objective
+```
+
+Yay! We have found the global optimum (this is known to be at $x_i = 420.9687$).

lib/OptimizationBBO/src/OptimizationBBO.jl

@@ -3,6 +3,7 @@ module OptimizationBBO
 using Reexport
 import Optimization
 import BlackBoxOptim, Optimization.SciMLBase
+import Optimization.SciMLBase: MultiObjectiveOptimizationFunction
 
 abstract type BBO end
 
@@ -15,6 +16,11 @@ for j in string.(BlackBoxOptim.SingleObjectiveMethodNames)
     eval(Meta.parse("export BBO_" * j))
 end
 
+Base.@kwdef struct BBO_borg_moea <: BBO
+    method = :borg_moea
+end
+export BBO_borg_moea
+
 function decompose_trace(opt::BlackBoxOptim.OptRunController, progress)
     if progress
         maxiters = opt.max_steps
@@ -105,12 +111,6 @@ function SciMLBase.__solve(cache::Optimization.OptimizationCache{
 }
     local x, cur, state
 
-    if cache.data != Optimization.DEFAULT_DATA
-        maxiters = length(cache.data)
-    end
-
-    cur, state = iterate(cache.data)
-
     function _cb(trace)
         if cache.callback === Optimization.DEFAULT_CALLBACK
             cb_call = false
@@ -132,33 +132,24 @@ function SciMLBase.__solve(cache::Optimization.OptimizationCache{
             BlackBoxOptim.shutdown_optimizer!(trace) #doesn't work
         end
 
-        if cache.data !== Optimization.DEFAULT_DATA
-            cur, state = iterate(cache.data, state)
-        end
         cb_call
     end
 
     maxiters = Optimization._check_and_convert_maxiters(cache.solver_args.maxiters)
     maxtime = Optimization._check_and_convert_maxtime(cache.solver_args.maxtime)
 
     _loss = function (θ)
-        if cache.callback === Optimization.DEFAULT_CALLBACK &&
-           cache.data === Optimization.DEFAULT_DATA
-            return first(cache.f(θ, cache.p))
-        elseif cache.callback === Optimization.DEFAULT_CALLBACK
-            return first(cache.f(θ, cache.p, cur...))
-        elseif cache.data !== Optimization.DEFAULT_DATA
-            x = cache.f(θ, cache.p)
-            return first(x)
+        if isa(cache.f, MultiObjectiveOptimizationFunction)
+            x = (cache.f(θ, cache.p),)
+            return x[1]
         else
-            x = cache.f(θ, cache.p, cur...)
+            x = cache.f(θ, cache.p)
             return first(x)
         end
     end
 
     opt_args = __map_optimizer_args(cache, cache.opt;
-        callback = cache.callback === Optimization.DEFAULT_CALLBACK &&
-                   cache.data === Optimization.DEFAULT_DATA ?
+        callback = cache.callback === Optimization.DEFAULT_CALLBACK ?
                    nothing : _cb,
         cache.solver_args...,
         maxiters = maxiters,