Use autodiff API based on ADTypes instead of symbols

Author: devmotion

Project.toml

@@ -4,6 +4,7 @@ version = "1.14.0"
 
 
 [deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 EnumX = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
 FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
@@ -25,6 +26,7 @@ MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
 OptimMOIExt = "MathOptInterface"
 
 [compat]
+ADTypes = "1.11.0"
 Compat = "3.2.0, 3.3.0, 3.4.0, 3.5.0, 3.6.0, 4"
 EnumX = "1.0.4"
 FillArrays = "0.6.2, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12, 0.13, 1"
@@ -44,7 +46,6 @@ Test = "<0.0.1, 1.6"
 julia = "1.10"
 
 [extras]
-ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
 MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
@@ -59,4 +60,4 @@ StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "Distributions", "MathOptInterface", "Measurements", "OptimTestProblems", "Random", "RecursiveArrayTools", "StableRNGs", "LineSearches", "NLSolversBase", "PositiveFactorizations", "ReverseDiff", "ADTypes"]
+test = ["Test", "Distributions", "MathOptInterface", "Measurements", "OptimTestProblems", "Random", "RecursiveArrayTools", "StableRNGs", "LineSearches", "NLSolversBase", "PositiveFactorizations", "ReverseDiff"]

docs/src/examples/ipnewton_basics.jl

@@ -78,7 +78,7 @@ using Test                 #src
 @test Optim.converged(res)      #src
 @test Optim.minimum(res) ≈ 0.25 #src
 
-# Like the rest of Optim, you can also use `autodiff=:forward` and just pass in
+# Like the rest of Optim, you can also use `autodiff=ADTypes.AutoForwardDiff()` and just pass in
 # `fun`.
 
 # If we only want to set lower bounds, use `ux = fill(Inf, 2)`

docs/src/examples/maxlikenlm.jl

@@ -22,6 +22,7 @@
 using Optim, NLSolversBase
 using LinearAlgebra: diag
 using ForwardDiff
+using ADTypes: AutoForwardDiff
 
 #md # !!! tip
 #md #     Add Optim with the following command at the Julia command prompt:
@@ -152,7 +153,7 @@ end
 func = TwiceDifferentiable(
     vars -> Log_Likelihood(x, y, vars[1:nvar], vars[nvar+1]),
     ones(nvar + 1);
-    autodiff = :forward,
+    autodiff = AutoForwardDiff(),
 );
 
 # The above statment accepts 4 inputs: the x matrix, the dependent
@@ -163,7 +164,7 @@ func = TwiceDifferentiable(
 # the error variance.
 #
 # The `ones(nvar+1)` are the starting values for the parameters and
-# the `autodiff=:forward` command performs forward mode automatic
+# the `autodiff=ADTypes.AutoForwardDiff()` command performs forward mode automatic
 # differentiation.
 #
 # The actual optimization of the likelihood function is accomplished

docs/src/user/gradientsandhessians.md

@@ -16,10 +16,10 @@ Automatic differentiation techniques are a middle ground between finite differen
 
 Reverse-mode automatic differentiation can be seen as an automatic implementation of the adjoint method mentioned above, and requires a runtime comparable to only one evaluation of ``f``. It is however considerably more complex to implement, requiring to record the execution of the program to then run it backwards, and incurs a larger overhead.
 
-Forward-mode automatic differentiation is supported through the [ForwardDiff.jl](https://github.com/JuliaDiff/ForwardDiff.jl) package by providing the `autodiff=:forward` keyword to `optimize`.
-More generic automatic differentiation is supported thanks to [DifferentiationInterface.jl](https://github.com/JuliaDiff/DifferentiationInterface.jl), by setting `autodiff` to any compatible backend object from [ADTypes.jl](https://github.com/SciML/ADTypes.jl).
-For instance, the user can choose `autodiff=AutoReverseDiff()`, `autodiff=AutoEnzyme()`, `autodiff=AutoMooncake()` or `autodiff=AutoZygote()` for a reverse-mode gradient computation, which is generally faster than forward mode on large inputs.
-Each of these choices requires loading the corresponding package beforehand.
+Generic automatic differentiation is supported thanks to [DifferentiationInterface.jl](https://github.com/JuliaDiff/DifferentiationInterface.jl), by setting `autodiff` to any compatible backend object from [ADTypes.jl](https://github.com/SciML/ADTypes.jl).
+For instance, forward-mode automatic differentiation through the [ForwardDiff.jl](https://github.com/JuliaDiff/ForwardDiff.jl) package by providing the `autodiff=ADTypes.AutoForwardDiff()` keyword to `optimize`.
+Additionally, the user can choose `autodiff=AutoReverseDiff()`, `autodiff=AutoEnzyme()`, `autodiff=AutoMooncake()` or `autodiff=AutoZygote()` for a reverse-mode gradient computation, which is generally faster than forward mode on large inputs.
+Each of these choices requires loading the `ADTypes` package and the corresponding automatic differentiation package (e.g., `ForwardDiff` or `ReverseDiff`) beforehand.
 
 ## Example
 
@@ -66,14 +66,14 @@ julia> Optim.minimizer(optimize(f, initial_x, BFGS()))
 ```
 Still looks good. Returning to automatic differentiation, let us try both solvers using this
 method.  We enable [forward mode](https://github.com/JuliaDiff/ForwardDiff.jl) automatic
-differentiation by using the `autodiff = :forward` keyword.
+differentiation by using the `autodiff = AutoForwardDiff()` keyword.
 ```jlcon
-julia> Optim.minimizer(optimize(f, initial_x, BFGS(); autodiff = :forward))
+julia> Optim.minimizer(optimize(f, initial_x, BFGS(); autodiff = AutoForwardDiff()))
 2-element Array{Float64,1}:
  1.0
  1.0
 
-julia> Optim.minimizer(optimize(f, initial_x, Newton(); autodiff = :forward))
+julia> Optim.minimizer(optimize(f, initial_x, Newton(); autodiff = AutoForwardDiff()))
 2-element Array{Float64,1}:
  1.0
  1.0

docs/src/user/minimization.md

@@ -26,9 +26,9 @@ If we pass `f` alone, Optim will construct an approximate gradient for us using
 ```jl
 optimize(f, x0, LBFGS())
 ```
-For better performance and greater precision, you can pass your own gradient function. If your objective is written in all Julia code with no special calls to external (that is non-Julia) libraries, you can also use automatic differentiation, by using the `autodiff` keyword and setting it to `:forward`:
+For better performance and greater precision, you can pass your own gradient function. If your objective is written in all Julia code with no special calls to external (that is non-Julia) libraries, you can also use automatic differentiation, by using the `autodiff` keyword and setting it to `AutoForwardDiff()`:
 ```julia
-optimize(f, x0, LBFGS(); autodiff = :forward)
+optimize(f, x0, LBFGS(); autodiff = AutoForwardDiff())
 ```
 
 For the Rosenbrock example, the analytical gradient can be shown to be:

ext/OptimMOIExt.jl

@@ -335,7 +335,7 @@ function MOI.optimize!(model::Optimizer{T}) where {T}
             inplace = true,
         )
     else
-        d = Optim.promote_objtype(method, initial_x, :finite, true, f, g!, h!)
+        d = Optim.promote_objtype(method, initial_x, Optim.DEFAULT_AD_TYPE, true, f, g!, h!)
         options = Optim.Options(; Optim.default_options(method)..., options...)
         if nl_constrained || has_bounds
             if nl_constrained

src/Optim.jl

@@ -47,6 +47,8 @@ import NLSolversBase:
 # var for NelderMead
 import StatsBase: var
 
+import ADTypes
+
 import LinearAlgebra
 import LinearAlgebra:
     Diagonal,

src/multivariate/optimize/interface.jl

@@ -4,6 +4,9 @@ fallback_method(f) = NelderMead()
 fallback_method(f, g!) = LBFGS()
 fallback_method(f, g!, h!) = Newton()
 
+# By default, use central finite difference method
+const DEFAULT_AD_TYPE = ADTypes.AutoFiniteDiff(; fdtype = Val(:central))
+
 function fallback_method(f::InplaceObjective)
     if !(f.fdf isa Nothing)
         if !(f.hv isa Nothing)
@@ -137,7 +140,7 @@ function optimize(
     f,
     initial_x::AbstractArray;
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
     method = fallback_method(f)
     d = promote_objtype(method, initial_x, autodiff, inplace, f)
@@ -149,7 +152,7 @@ function optimize(
     f,
     g,
     initial_x::AbstractArray;
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
     inplace = true,
 )
 
@@ -166,7 +169,7 @@ function optimize(
     h,
     initial_x::AbstractArray;
     inplace = true,
-    autodiff = :finite
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE, 
 )
     method = fallback_method(f, g, h)
     d = promote_objtype(method, initial_x, autodiff, inplace, f, g, h)
@@ -189,7 +192,7 @@ function optimize(
     initial_x::AbstractArray,
     options::Options;
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
     method = fallback_method(f)
     d = promote_objtype(method, initial_x, autodiff, inplace, f)
@@ -201,7 +204,7 @@ function optimize(
     initial_x::AbstractArray,
     options::Options;
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
 
     method = fallback_method(f, g)
@@ -215,7 +218,7 @@ function optimize(
     initial_x::AbstractArray{T},
     options::Options;
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 ) where {T}
     method = fallback_method(f, g, h)
     d = promote_objtype(method, initial_x, autodiff, inplace, f, g, h)
@@ -230,7 +233,7 @@ function optimize(
     method::AbstractOptimizer,
     options::Options = Options(; default_options(method)...);
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
     d = promote_objtype(method, initial_x, autodiff, inplace, f)
     optimize(d, initial_x, method, options)
@@ -242,7 +245,7 @@ function optimize(
     method::AbstractOptimizer,
     options::Options = Options(; default_options(method)...);
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
 
     d = promote_objtype(method, initial_x, autodiff, inplace, f)
@@ -255,7 +258,7 @@ function optimize(
     method::AbstractOptimizer,
     options::Options = Options(; default_options(method)...);
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
     d = promote_objtype(method, initial_x, autodiff, inplace, f, g)
 
@@ -269,7 +272,7 @@ function optimize(
     method::AbstractOptimizer,
     options::Options = Options(; default_options(method)...);
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 
 )
     d = promote_objtype(method, initial_x, autodiff, inplace, f, g, h)
@@ -283,7 +286,7 @@ function optimize(
     method::SecondOrderOptimizer,
     options::Options = Options(; default_options(method)...);
     inplace = true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 ) where {D<:Union{NonDifferentiable,OnceDifferentiable}}
     d = promote_objtype(method, initial_x, autodiff, inplace, d)
     optimize(d, initial_x, method, options)

src/multivariate/solvers/constrained/fminbox.jl

@@ -278,7 +278,7 @@ function optimize(
     F::Fminbox = Fminbox(),
     options::Options = Options();
     inplace::Bool=true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
     if f isa NonDifferentiable
         f = f.f
@@ -304,7 +304,7 @@ function optimize(
     optimize(od, l, u, initial_x, F, options)
 end
 
-function optimize(f, l::Number, u::Number, initial_x::AbstractArray; autodiff = :finite)
+function optimize(f, l::Number, u::Number, initial_x::AbstractArray; autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE)
     T = eltype(initial_x)
     optimize(
         OnceDifferentiable(f, initial_x, zero(T); autodiff),
@@ -324,7 +324,7 @@ optimize(
     mo::AbstractConstrainedOptimizer,
     opt::Options = Options();
     inplace::Bool=true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 ) = optimize(
     f,
     Fill(T(l), size(initial_x)...),
@@ -343,7 +343,7 @@ function optimize(
     mo::AbstractConstrainedOptimizer = Fminbox(),
     opt::Options = Options();
     inplace::Bool=true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
   T = eltype(initial_x)
 optimize(f, T.(l), Fill(T(u), size(initial_x)...), initial_x, mo, opt; inplace, autodiff)
@@ -356,7 +356,7 @@ function optimize(
     mo::AbstractConstrainedOptimizer=Fminbox(),
     opt::Options = Options();
     inplace::Bool=true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
     T = eltype(initial_x)
     optimize(f, Fill(T(l), size(initial_x)...), T.(u), initial_x, mo, opt; inplace, autodiff)
@@ -369,7 +369,7 @@ function optimize(
     initial_x::AbstractArray,
     opt::Options;
     inplace::Bool=true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 ) 
 
 T = eltype(initial_x)
@@ -393,7 +393,7 @@ function optimize(
     initial_x::AbstractArray,
     opt::Options;
     inplace::Bool=true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
 T = eltype(initial_x)
 optimize(f, g, T.(l), Fill(T(u), size(initial_x)...), initial_x, opt; inplace, autodiff)
@@ -407,7 +407,7 @@ function optimize(
     initial_x::AbstractArray,
     opt::Options;
     inplace::Bool=true,
-    autodiff = :finite,
+    autodiff::ADTypes.AbstractADType = DEFAULT_AD_TYPE,
 )
     T= eltype(initial_x)
     optimize(f, g, Fill(T(l), size(initial_x)...), T.(u), initial_x, opt, inplace, autodiff)

test/general/objective_types.jl

@@ -4,8 +4,8 @@
         @test_throws ErrorException OnceDifferentiable(x -> x, rand(10); autodiff = :wah)
 
         for T in (OnceDifferentiable, TwiceDifferentiable)
-            odad1 = T(x -> 5.0, rand(1); autodiff = :finite)
-            odad2 = T(x -> 5.0, rand(1); autodiff = :forward)
+            odad1 = T(x -> 5.0, rand(1); autodiff = AutoFiniteDiff(; fdtype = Val(:central)))
+            odad2 = T(x -> 5.0, rand(1); autodiff = AutoForwardDiff())
             odad3 = T(x -> 5.0, rand(1); autodiff = AutoReverseDiff())
             Optim.gradient!(odad1, rand(1))
             Optim.gradient!(odad2, rand(1))
@@ -17,8 +17,8 @@
 
         for a in (1.0, 5.0)
             xa = rand(1)
-            odad1 = OnceDifferentiable(x -> a * x[1], xa; autodiff = :finite)
-            odad2 = OnceDifferentiable(x -> a * x[1], xa; autodiff = :forward)
+            odad1 = OnceDifferentiable(x -> a * x[1], xa; autodiff = AutoFiniteDiff(; fdtype = Val(:central)))
+            odad2 = OnceDifferentiable(x -> a * x[1], xa; autodiff = AutoForwardDiff())
             odad3 = OnceDifferentiable(x -> a * x[1], xa; autodiff = AutoReverseDiff())
             Optim.gradient!(odad1, xa)
             Optim.gradient!(odad2, xa)
@@ -29,8 +29,8 @@
         end
         for a in (1.0, 5.0)
             xa = rand(1)
-            odad1 = OnceDifferentiable(x -> a * x[1]^2, xa; autodiff = :finite)
-            odad2 = OnceDifferentiable(x -> a * x[1]^2, xa; autodiff = :forward)
+            odad1 = OnceDifferentiable(x -> a * x[1]^2, xa; autodiff = AutoFiniteDiff(; fdtype = Val(:central)))
+            odad2 = OnceDifferentiable(x -> a * x[1]^2, xa; autodiff = AutoForwardDiff())
             odad3 = OnceDifferentiable(x -> a * x[1]^2, xa; autodiff = AutoReverseDiff())
             Optim.gradient!(odad1, xa)
             Optim.gradient!(odad2, xa)
@@ -40,7 +40,7 @@
             @test Optim.gradient(odad3) == 2.0 * a * xa
         end
         for dtype in (OnceDifferentiable, TwiceDifferentiable)
-            for autodiff in (:finite, :forward, AutoReverseDiff())
+            for autodiff in (AutoFiniteDiff(; fdtype = Val(:central)), AutoForwardDiff(), AutoReverseDiff())
                 differentiable = dtype(x -> sum(x), rand(2); autodiff = autodiff)
                 Optim.value(differentiable)
                 Optim.value!(differentiable, rand(2))