Merge branch 'master' of https://github.com/ParasPuneetSingh/Optimization.jl

Author: ParasPuneetSingh

.github/workflows/CI.yml

@@ -33,7 +33,9 @@ jobs:
           - OptimizationOptimJL
           - OptimizationOptimisers
           - OptimizationPRIMA
+          - OptimizationPyCMA
           - OptimizationQuadDIRECT
+          - OptimizationSciPy
           - OptimizationSpeedMapping
           - OptimizationPolyalgorithms
           - OptimizationNLPModels

Project.toml

@@ -1,6 +1,6 @@
 name = "Optimization"
 uuid = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
-version = "4.3.0"
+version = "4.4.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

docs/Project.toml

@@ -44,6 +44,9 @@ SymbolicAnalysis = "4297ee4d-0239-47d8-ba5d-195ecdf594fe"
 Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+BlackBoxOptim = "a134a8b2-14d6-55f6-9291-3336d3ab0209"
+Metaheuristics = "bcdb8e00-2c21-11e9-3065-2b553b22f898"
+Evolutionary = "86b6b26d-c046-49b6-aa0b-5f0f74682bd6"
 
 [compat]
 AmplNLWriter = "1"
@@ -90,3 +93,7 @@ SymbolicAnalysis = "0.3"
 Symbolics = "6"
 Tracker = ">= 0.2"
 Zygote = ">= 0.5"
+BlackBoxOptim = "0.6"
+Metaheuristics = "3"
+Evolutionary = "0.11"
+

docs/pages.jl

@@ -23,7 +23,7 @@ pages = ["index.md",
         "API/modelingtoolkit.md",
         "API/FAQ.md"
     ],
-    "Optimizer Packages" => [
+    "Optimizer Packages" => [   
         "BlackBoxOptim.jl" => "optimization_packages/blackboxoptim.md",
         "CMAEvolutionStrategy.jl" => "optimization_packages/cmaevolutionstrategy.md",
         "Evolutionary.jl" => "optimization_packages/evolutionary.md",
@@ -40,7 +40,9 @@ pages = ["index.md",
         "Optimization.jl" => "optimization_packages/optimization.md",
         "Polyalgorithms.jl" => "optimization_packages/polyopt.md",
         "PRIMA.jl" => "optimization_packages/prima.md",
+        "PyCMA.jl" => "optimization_packages/pycma.md",
         "QuadDIRECT.jl" => "optimization_packages/quaddirect.md",
-        "SpeedMapping.jl" => "optimization_packages/speedmapping.md"
+        "SpeedMapping.jl" => "optimization_packages/speedmapping.md",
+        "SciPy.jl" => "optimization_packages/scipy.md"
     ]
 ]

docs/src/API/modelingtoolkit.md

@@ -16,4 +16,4 @@ Secondly, one can generate `OptimizationProblem`s for use in
 Optimization.jl from purely a symbolic front-end. This is the form
 users will encounter when using ModelingToolkit.jl directly, and it is
 also the form supplied by domain-specific languages. For more information,
-see the [OptimizationSystem documentation](https://docs.sciml.ai/ModelingToolkit/stable/systems/OptimizationSystem/).
+see the [OptimizationSystem documentation](https://docs.sciml.ai/ModelingToolkit/stable/API/problems/#SciMLBase.OptimizationProblem).

docs/src/optimization_packages/blackboxoptim.md

@@ -67,3 +67,21 @@ prob = Optimization.OptimizationProblem(f, x0, p, lb = [-1.0, -1.0], ub = [1.0,
 sol = solve(prob, BBO_adaptive_de_rand_1_bin_radiuslimited(), maxiters = 100000,
     maxtime = 1000.0)
 ```
+
+## Multi-objective optimization
+The optimizer for Multi-Objective Optimization is `BBO_borg_moea()`. Your objective function should return a vector of the objective values and you should indicate the fitness scheme to be (typically) Pareto fitness and specify the number of objectives. Otherwise, the use is similar, here is an example:
+
+```@example MOO-BBO
+using OptimizationBBO, Optimization, BlackBoxOptim
+using SciMLBase: MultiObjectiveOptimizationFunction
+u0 = [0.25, 0.25]
+opt = OptimizationBBO.BBO_borg_moea()
+function multi_obj_func(x, p)
+    f1 = (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2  # Rosenbrock function
+    f2 = -20.0 * exp(-0.2 * sqrt(0.5 * (x[1]^2 + x[2]^2))) - exp(0.5 * (cos(2π * x[1]) + cos(2π * x[2]))) + exp(1) + 20.0  # Ackley function
+    return (f1, f2)
+end
+mof = MultiObjectiveOptimizationFunction(multi_obj_func)
+prob = Optimization.OptimizationProblem(mof, u0; lb = [0.0, 0.0], ub = [2.0, 2.0])
+sol = solve(prob, opt, NumDimensions=2, FitnessScheme=ParetoFitnessScheme{2}(is_minimizing=true))
+```

docs/src/optimization_packages/evolutionary.md

@@ -41,3 +41,20 @@ f = OptimizationFunction(rosenbrock)
 prob = Optimization.OptimizationProblem(f, x0, p, lb = [-1.0, -1.0], ub = [1.0, 1.0])
 sol = solve(prob, Evolutionary.CMAES(μ = 40, λ = 100))
 ```
+
+## Multi-objective optimization
+The Rosenbrock and Ackley functions can be optimized using the `Evolutionary.NSGA2()` as follows:
+
+```@example MOO-Evolutionary
+using Optimization, OptimizationEvolutionary, Evolutionary
+function func(x, p=nothing)::Vector{Float64}
+  f1 = (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2  # Rosenbrock function
+  f2 = -20.0 * exp(-0.2 * sqrt(0.5 * (x[1]^2 + x[2]^2))) - exp(0.5 * (cos(2π * x[1]) + cos(2π * x[2]))) + exp(1) + 20.0  # Ackley function
+  return [f1, f2]
+end
+initial_guess = [1.0, 1.0]
+obj_func = MultiObjectiveOptimizationFunction(func)
+algorithm = OptimizationEvolutionary.NSGA2()
+problem = OptimizationProblem(obj_func, initial_guess)
+result = solve(problem, algorithm)
+```

docs/src/optimization_packages/metaheuristics.md

@@ -70,3 +70,46 @@ sol = solve(prob, ECA(), use_initial = true, maxiters = 100000, maxtime = 1000.0
 ### With Constraint Equations
 
 While `Metaheuristics.jl` supports such constraints, `Optimization.jl` currently does not relay these constraints.
+
+
+## Multi-objective optimization
+The zdt1 functions can be optimized using the `Metaheuristics.jl` as follows:
+
+```@example MOO-Metaheuristics
+using Optimization, OptimizationEvolutionary,OptimizationMetaheuristics, Metaheuristics
+function zdt1(x)
+    f1 = x[1]
+    g = 1 + 9 * mean(x[2:end])
+    h = 1 - sqrt(f1 / g)
+    f2 = g * h
+    # In this example, we have no constraints
+    gx = [0.0]  # Inequality constraints (not used)
+    hx = [0.0]  # Equality constraints (not used)
+    return [f1, f2], gx, hx
+end
+multi_obj_fun = MultiObjectiveOptimizationFunction((x, p) -> zdt1(x))
+
+# Define the problem bounds
+lower_bounds = [0.0, 0.0, 0.0]
+upper_bounds = [1.0, 1.0, 1.0]
+
+# Define the initial guess
+initial_guess = [0.5, 0.5, 0.5]
+
+# Create the optimization problem
+prob = OptimizationProblem(multi_obj_fun, initial_guess; lb = lower_bounds, ub = upper_bounds)
+
+nobjectives = 2
+npartitions = 100
+
+# reference points (Das and Dennis's method)
+weights = Metaheuristics.gen_ref_dirs(nobjectives, npartitions)
+
+# Choose the algorithm and solve the problem
+sol1 = solve(prob, Metaheuristics.NSGA2(); maxiters = 100, use_initial = true)
+sol2 = solve(prob, Metaheuristics.NSGA3(); maxiters = 100, use_initial = true)
+sol3 = solve(prob, Metaheuristics.SPEA2(); maxiters = 100, use_initial = true)
+sol4 = solve(prob, Metaheuristics.CCMO(NSGA2(N=100, p_m=0.001)))
+sol5 = solve(prob, Metaheuristics.MOEAD_DE(weights, options=Options(debug=false, iterations = 250)); maxiters = 100, use_initial = true)
+sol6 = solve(prob, Metaheuristics.SMS_EMOA(); maxiters = 100, use_initial = true)
+```

docs/src/optimization_packages/pycma.md

@@ -0,0 +1,63 @@
+# PyCMA.jl
+
+[`PyCMA`](https://github.com/CMA-ES/pycma) is a Python implementation of CMA-ES and a few related numerical optimization tools. `OptimizationPyCMA.jl` gives access to the CMA-ES optimizer through the unified `Optimization.jl` interface just like any native Julia optimizer.
+
+`OptimizationPyCMA.jl` relies on [`PythonCall`](https://github.com/cjdoris/PythonCall.jl). A minimal Python distribution containing PyCMA will be installed automatically on first use, so no manual Python set-up is required.
+
+## Installation: OptimizationPyCMA.jl
+
+```julia
+import Pkg
+Pkg.add("OptimizationPyCMA")
+```
+
+## Methods
+
+`PyCMAOpt` supports the usual keyword arguments `maxiters`, `maxtime`, `abstol`, `reltol`, `callback` in addition to any PyCMA-specific options (passed verbatim via keyword arguments to `solve`).
+
+## Example
+
+```@example PyCMA
+using OptimizationPyCMA
+
+rosenbrock(x, p) = (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2
+x0 = zeros(2)
+_p = [1.0, 100.0]
+l1 = rosenbrock(x0, _p)
+f = OptimizationFunction(rosenbrock)
+prob = OptimizationProblem(f, x0, _p, lb = [-1.0, -1.0], ub = [0.8, 0.8])
+sol = solve(prob, PyCMAOpt())
+```
+
+## Passing solver-specific options
+
+Any keyword that `Optimization.jl` does not interpret is forwarded directly to PyCMA. 
+
+In the event an `Optimization.jl` keyword overlaps with a `PyCMA` keyword, the `Optimization.jl` keyword takes precedence. 
+
+An exhaustive list of keyword arguments can be found by running the following python script:
+
+```python
+import cma
+options = cma.CMAOptions()
+print(options)
+```
+
+An example passing the `PyCMA` keywords "verbose" and "seed":
+```julia
+sol = solve(prob, PyCMA(), verbose = -9, seed = 42)
+```
+
+## Troubleshooting
+
+The original Python result object is attached to the solution in the `original` field:
+
+```julia
+sol = solve(prob, PyCMAOpt())
+println(sol.original) 
+```
+
+## Contributing
+
+Bug reports and feature requests are welcome in the [Optimization.jl](https://github.com/SciML/Optimization.jl) issue tracker.  Pull requests that improve either the Julia wrapper or the documentation are highly appreciated. 
+

docs/src/optimization_packages/scipy.md

@@ -0,0 +1,133 @@
+# SciPy.jl
+
+[`SciPy`](https://scipy.org/) is a mature Python library that offers a rich family of optimization, root–finding and linear‐programming algorithms.  `OptimizationSciPy.jl` gives access to these routines through the unified `Optimization.jl` interface just like any native Julia optimizer.
+
+!!! note
+    `OptimizationSciPy.jl` relies on [`PythonCall`](https://github.com/cjdoris/PythonCall.jl).  A minimal Python distribution containing SciPy will be installed automatically on first use, so no manual Python set-up is required.
+
+## Installation: OptimizationSciPy.jl
+
+```julia
+import Pkg
+Pkg.add("OptimizationSciPy")
+```
+
+## Methods
+
+Below is a catalogue of the solver families exposed by `OptimizationSciPy.jl` together with their convenience constructors.  All of them accept the usual keyword arguments `maxiters`, `maxtime`, `abstol`, `reltol`, `callback`, `progress` in addition to any SciPy-specific options (passed verbatim via keyword arguments to `solve`).
+
+### Local Optimizer
+
+#### Derivative-Free
+
+  * `ScipyNelderMead()` – Simplex Nelder–Mead algorithm
+  * `ScipyPowell()` – Powell search along conjugate directions
+  * `ScipyCOBYLA()` – Linear approximation of constraints (supports nonlinear constraints)
+
+#### Gradient-Based
+
+  * `ScipyCG()` – Non-linear conjugate gradient
+  * `ScipyBFGS()` – Quasi-Newton BFGS
+  * `ScipyLBFGSB()` – Limited-memory BFGS with simple bounds
+  * `ScipyNewtonCG()` – Newton-conjugate gradient (requires Hessian-vector products)
+  * `ScipyTNC()` – Truncated Newton with bounds
+  * `ScipySLSQP()` – Sequential least-squares programming (supports constraints)
+  * `ScipyTrustConstr()` – Trust-region method for non-linear constraints
+
+#### Hessian–Based / Trust-Region
+
+  * `ScipyDogleg()`, `ScipyTrustNCG()`, `ScipyTrustKrylov()`, `ScipyTrustExact()` – Trust-region algorithms that optionally use or build Hessian information
+
+### Global Optimizer
+
+  * `ScipyDifferentialEvolution()` – Differential evolution (requires bounds)
+  * `ScipyBasinhopping()` – Basin-hopping with local search
+  * `ScipyDualAnnealing()` – Dual annealing simulated annealing
+  * `ScipyShgo()` – Simplicial homology global optimisation (supports constraints)
+  * `ScipyDirect()` – Deterministic `DIRECT` algorithm (requires bounds)
+  * `ScipyBrute()` – Brute-force grid search (requires bounds)
+
+### Linear & Mixed-Integer Programming
+
+  * `ScipyLinprog("highs")` – LP solvers from the HiGHS project and legacy interior-point/simplex methods
+  * `ScipyMilp()` – Mixed-integer linear programming via HiGHS branch-and-bound
+
+### Root Finding & Non-Linear Least Squares *(experimental)*
+
+Support for `ScipyRoot`, `ScipyRootScalar` and `ScipyLeastSquares` is available behind the scenes and will be documented once the APIs stabilise.
+
+## Examples
+
+### Unconstrained minimisation
+
+```@example SciPy1
+using Optimization, OptimizationSciPy
+
+rosenbrock(x, p) = (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2
+x0 = zeros(2)
+p  = [1.0, 100.0]
+
+f   = OptimizationFunction(rosenbrock, Optimization.AutoZygote())
+prob = OptimizationProblem(f, x0, p)
+
+sol = solve(prob, ScipyBFGS())
+@show sol.objective   # ≈ 0 at optimum
+```
+
+### Constrained optimisation with COBYLA
+
+```@example SciPy2
+using Optimization, OptimizationSciPy
+
+# Objective
+obj(x, p) = (x[1] + x[2] - 1)^2
+
+# Single non-linear constraint: x₁² + x₂² ≈ 1 (with small tolerance)
+cons(res, x, p) = (res .= [x[1]^2 + x[2]^2 - 1.0])
+
+x0   = [0.5, 0.5]
+prob = OptimizationProblem(
+    OptimizationFunction(obj; cons = cons),
+    x0, nothing, lcons = [-1e-6], ucons = [1e-6])  # Small tolerance instead of exact equality
+
+sol = solve(prob, ScipyCOBYLA())
+@show sol.u, sol.objective
+```
+
+### Differential evolution (global) with custom options
+
+```@example SciPy3
+using Optimization, OptimizationSciPy, Random, Statistics
+Random.seed!(123)
+
+ackley(x, p) = -20exp(-0.2*sqrt(mean(x .^ 2))) - exp(mean(cos.(2π .* x))) + 20 + ℯ
+x0 = zeros(2)                    # initial guess is ignored by DE
+prob = OptimizationProblem(ackley, x0; lb = [-5.0, -5.0], ub = [5.0, 5.0])
+
+sol = solve(prob, ScipyDifferentialEvolution(); popsize = 20, mutation = (0.5, 1))
+@show sol.objective
+```
+
+## Passing solver-specific options
+
+Any keyword that `Optimization.jl` does not interpret is forwarded directly to SciPy.  Refer to the [SciPy optimisation API](https://docs.scipy.org/doc/scipy/reference/optimize.html) for the exhaustive list of options.
+
+```julia
+sol = solve(prob, ScipyTrustConstr(); verbose = 3, maxiter = 10_000)
+```
+
+## Troubleshooting
+
+The original Python result object is attached to the solution in the `original` field:
+
+```julia
+sol = solve(prob, ScipyBFGS())
+println(sol.original)  # SciPy OptimizeResult
+```
+
+If SciPy raises an error it is re-thrown as a Julia `ErrorException` carrying the Python message, so look there first.
+
+## Contributing
+
+Bug reports and feature requests are welcome in the [Optimization.jl](https://github.com/SciML/Optimization.jl) issue tracker.  Pull requests that improve either the Julia wrapper or the documentation are highly appreciated. 
+