Replace ADAM with Adam etc (#78)

* replace ADAM with Adam etc

* also update names in tests

Author: mcabbott

Project.toml

@@ -1,7 +1,7 @@
 name = "Optimisers"
 uuid = "3bd65402-5787-11e9-1adc-39752487f4e2"
 authors = ["Mike J Innes <[email protected]>"]
-version = "0.2.4"
+version = "0.2.5"
 
 [deps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"

README.md

@@ -36,7 +36,7 @@ It is initialised by `setup`, and then at each step, `update` returns both the n
 state, and the model with its trainable parameters adjusted:
 
 ```julia
-state = Optimisers.setup(Optimisers.ADAM(), model)  # just once
+state = Optimisers.setup(Optimisers.Adam(), model)  # just once
 
 state, model = Optimisers.update(state, model, grad)  # at every step
 ```

src/Optimisers.jl

@@ -9,8 +9,8 @@ include("destructure.jl")
 export destructure
 
 include("rules.jl")
-export Descent, ADAM, Momentum, Nesterov, RMSProp,
-       ADAGrad, AdaMax, ADADelta, AMSGrad, NADAM, ADAMW, RADAM, OADAM, AdaBelief,
+export Descent, Adam, Momentum, Nesterov, RMSProp,
+       AdaGrad, AdaMax, AdaDelta, AMSGrad, NAdam, AdamW, RAdam, OAdam, AdaBelief,
        WeightDecay, ClipGrad, ClipNorm, OptimiserChain
 
 """

src/rules.jl

@@ -1,3 +1,11 @@
+@deprecate ADAM Adam
+@deprecate NADAM NAdam
+@deprecate ADAMW AdamW
+@deprecate RADAM RAdam
+@deprecate OADAM OAdam
+@deprecate ADAGrad AdaGrad
+@deprecate ADADelta AdaDelta
+
 """
     Descent(η = 1f-1)
 
@@ -110,9 +118,9 @@ function apply!(o::RMSProp, state, x, dx)
 end
 
 """
-    ADAM(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
+    Adam(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
 
-[ADAM](https://arxiv.org/abs/1412.6980) optimiser.
+[Adam](https://arxiv.org/abs/1412.6980) optimiser.
 
 # Parameters
 - Learning rate (`η`): Amount by which gradients are discounted before updating
@@ -122,16 +130,18 @@ end
 - Machine epsilon (`ϵ`): Constant to prevent division by zero
                          (no need to change default)
 """
-struct ADAM{T}
+struct Adam{T}
   eta::T
   beta::Tuple{T, T}
   epsilon::T
 end
-ADAM(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η))) = ADAM{typeof(η)}(η, β, ϵ)
+Adam(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η))) = Adam{typeof(η)}(η, β, ϵ)
+
+const Adam = Adam
 
-init(o::ADAM, x::AbstractArray) = (zero(x), zero(x), o.beta)
+init(o::Adam, x::AbstractArray) = (zero(x), zero(x), o.beta)
 
-function apply!(o::ADAM, state, x, dx)
+function apply!(o::Adam, state, x, dx)
   η, β, ϵ = o.eta, o.beta, o.epsilon
   mt, vt, βt = state
 
@@ -143,9 +153,9 @@ function apply!(o::ADAM, state, x, dx)
 end
 
 """
-    RADAM(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
+    RAdam(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
 
-[Rectified ADAM](https://arxiv.org/abs/1908.03265) optimizer.
+[Rectified Adam](https://arxiv.org/abs/1908.03265) optimizer.
 
 # Parameters
 - Learning rate (`η`): Amount by which gradients are discounted before updating
@@ -155,16 +165,16 @@ end
 - Machine epsilon (`ϵ`): Constant to prevent division by zero
                          (no need to change default)
 """
-struct RADAM{T}
+struct RAdam{T}
   eta::T
   beta::Tuple{T, T}
   epsilon::T
 end
-RADAM(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η))) = RADAM{typeof(η)}(η, β, ϵ)
+RAdam(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η))) = RAdam{typeof(η)}(η, β, ϵ)
 
-init(o::RADAM, x::AbstractArray) = (zero(x), zero(x), o.beta, 1)
+init(o::RAdam, x::AbstractArray) = (zero(x), zero(x), o.beta, 1)
 
-function apply!(o::RADAM, state, x, dx)
+function apply!(o::RAdam, state, x, dx)
   η, β, ϵ = o.eta, o.beta, o.epsilon
   ρ∞ = 2/(1-β[2])-1
 
@@ -186,7 +196,7 @@ end
 """
     AdaMax(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
 
-[AdaMax](https://arxiv.org/abs/1412.6980) is a variant of ADAM based on the ∞-norm.
+[AdaMax](https://arxiv.org/abs/1412.6980) is a variant of Adam based on the ∞-norm.
 
 # Parameters
 - Learning rate (`η`): Amount by which gradients are discounted before updating
@@ -217,10 +227,10 @@ function apply!(o::AdaMax, state, x, dx)
 end
 
 """
-    OADAM(η = 1f-3, β = (5f-1, 9f-1), ϵ = eps(typeof(η)))
+    OAdam(η = 1f-3, β = (5f-1, 9f-1), ϵ = eps(typeof(η)))
 
-[OADAM](https://arxiv.org/abs/1711.00141) (Optimistic ADAM)
-is a variant of ADAM adding an "optimistic" term suitable for adversarial training.
+[OAdam](https://arxiv.org/abs/1711.00141) (Optimistic Adam)
+is a variant of Adam adding an "optimistic" term suitable for adversarial training.
 
 # Parameters
 - Learning rate (`η`): Amount by which gradients are discounted before updating
@@ -230,16 +240,16 @@ is a variant of ADAM adding an "optimistic" term suitable for adversarial traini
 - Machine epsilon (`ϵ`): Constant to prevent division by zero
                          (no need to change default)
 """
-struct OADAM{T}
+struct OAdam{T}
   eta::T
   beta::Tuple{T, T}
   epsilon::T
 end
-OADAM(η = 1f-3, β = (5f-1, 9f-1), ϵ = eps(typeof(η))) = OADAM{typeof(η)}(η, β, ϵ)
+OAdam(η = 1f-3, β = (5f-1, 9f-1), ϵ = eps(typeof(η))) = OAdam{typeof(η)}(η, β, ϵ)
 
-init(o::OADAM, x::AbstractArray) = (zero(x), zero(x), o.beta, zero(x))
+init(o::OAdam, x::AbstractArray) = (zero(x), zero(x), o.beta, zero(x))
 
-function apply!(o::OADAM, state, x, dx)
+function apply!(o::OAdam, state, x, dx)
   η, β, ϵ = o.eta, o.beta, o.epsilon
   mt, vt, βt, term = state
 
@@ -253,9 +263,9 @@ function apply!(o::OADAM, state, x, dx)
 end
 
 """
-    ADAGrad(η = 1f-1, ϵ = eps(typeof(η)))
+    AdaGrad(η = 1f-1, ϵ = eps(typeof(η)))
 
-[ADAGrad](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) optimizer. It has
+[AdaGrad](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) optimizer. It has
 parameter specific learning rates based on how frequently it is updated.
 Parameters don't need tuning.
 
@@ -265,15 +275,15 @@ Parameters don't need tuning.
 - Machine epsilon (`ϵ`): Constant to prevent division by zero
                          (no need to change default)
 """
-struct ADAGrad{T}
+struct AdaGrad{T}
   eta::T
   epsilon::T
 end
-ADAGrad(η = 1f-1, ϵ = eps(typeof(η))) = ADAGrad{typeof(η)}(η, ϵ)
+AdaGrad(η = 1f-1, ϵ = eps(typeof(η))) = AdaGrad{typeof(η)}(η, ϵ)
 
-init(o::ADAGrad, x::AbstractArray) = onevalue(o.epsilon, x)
+init(o::AdaGrad, x::AbstractArray) = onevalue(o.epsilon, x)
 
-function apply!(o::ADAGrad, state, x, dx)
+function apply!(o::AdaGrad, state, x, dx)
   η, ϵ = o.eta, o.epsilon
   acc = state
 
@@ -284,9 +294,9 @@ function apply!(o::ADAGrad, state, x, dx)
 end
 
 """
-    ADADelta(ρ = 9f-1, ϵ = eps(typeof(ρ)))
+    AdaDelta(ρ = 9f-1, ϵ = eps(typeof(ρ)))
 
-[ADADelta](https://arxiv.org/abs/1212.5701) is a version of ADAGrad adapting its learning
+[AdaDelta](https://arxiv.org/abs/1212.5701) is a version of AdaGrad adapting its learning
 rate based on a window of past gradient updates.
 Parameters don't need tuning.
 
@@ -295,15 +305,15 @@ Parameters don't need tuning.
 - Machine epsilon (`ϵ`): Constant to prevent division by zero
                          (no need to change default)
 """
-struct ADADelta{T}
+struct AdaDelta{T}
   rho::T
   epsilon::T
 end
-ADADelta(ρ = 9f-1, ϵ = eps(typeof(ρ))) = ADADelta{typeof(ρ)}(ρ, ϵ)
+AdaDelta(ρ = 9f-1, ϵ = eps(typeof(ρ))) = AdaDelta{typeof(ρ)}(ρ, ϵ)
 
-init(o::ADADelta, x::AbstractArray) = (zero(x), zero(x))
+init(o::AdaDelta, x::AbstractArray) = (zero(x), zero(x))
 
-function apply!(o::ADADelta, state, x, dx)
+function apply!(o::AdaDelta, state, x, dx)
   ρ, ϵ = o.rho, o.epsilon
   acc, Δacc = state
 
@@ -318,7 +328,7 @@ end
 """
     AMSGrad(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
 
-The [AMSGrad](https://openreview.net/forum?id=ryQu7f-RZ) version of the ADAM
+The [AMSGrad](https://openreview.net/forum?id=ryQu7f-RZ) version of the Adam
 optimiser. Parameters don't need tuning.
 
 # Parameters
@@ -352,9 +362,9 @@ function apply!(o::AMSGrad, state, x, dx)
 end
 
 """
-    NADAM(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
+    NAdam(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η)))
 
-[NADAM](https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ) is a Nesterov variant of ADAM.
+[NAdam](https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ) is a Nesterov variant of Adam.
 Parameters don't need tuning.
 
 # Parameters
@@ -365,16 +375,16 @@ Parameters don't need tuning.
 - Machine epsilon (`ϵ`): Constant to prevent division by zero
                          (no need to change default)
 """
-struct NADAM{T}
+struct NAdam{T}
   eta::T
   beta::Tuple{T, T}
   epsilon::T
 end
-NADAM(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η))) = NADAM{typeof(η)}(η, β, ϵ)
+NAdam(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = eps(typeof(η))) = NAdam{typeof(η)}(η, β, ϵ)
 
-init(o::NADAM, x::AbstractArray) = (zero(x), zero(x), o.beta)
+init(o::NAdam, x::AbstractArray) = (zero(x), zero(x), o.beta)
 
-function apply!(o::NADAM, state, x, dx)
+function apply!(o::NAdam, state, x, dx)
   η, β, ϵ = o.eta, o.beta, o.epsilon
 
   mt, vt, βt = state
@@ -388,9 +398,9 @@ function apply!(o::NADAM, state, x, dx)
 end
 
 """
-    ADAMW(η = 1f-3, β = (9f-1, 9.99f-1), γ = 0, ϵ = eps(typeof(η)))
+    AdamW(η = 1f-3, β = (9f-1, 9.99f-1), γ = 0, ϵ = eps(typeof(η)))
 
-[ADAMW](https://arxiv.org/abs/1711.05101) is a variant of ADAM fixing (as in repairing) its
+[AdamW](https://arxiv.org/abs/1711.05101) is a variant of Adam fixing (as in repairing) its
 weight decay regularization.
 
 # Parameters
@@ -402,14 +412,14 @@ weight decay regularization.
 - Machine epsilon (`ϵ`): Constant to prevent division by zero
                          (no need to change default)
 """
-ADAMW(η = 1f-3, β = (9f-1, 9.99f-1), γ = 0, ϵ = eps(typeof(η))) =
-  OptimiserChain(ADAM{typeof(η)}(η, β, ϵ), WeightDecay{typeof(η)}(γ))
+AdamW(η = 1f-3, β = (9f-1, 9.99f-1), γ = 0, ϵ = eps(typeof(η))) =
+  OptimiserChain(Adam{typeof(η)}(η, β, ϵ), WeightDecay{typeof(η)}(γ))
 
 """
     AdaBelief(η = 1f-3, β = (9f-1, 9.99f-1), ϵ = 1e-16)
 
 The [AdaBelief](https://arxiv.org/abs/2010.07468) optimiser is a variant of the well-known
-ADAM optimiser.
+Adam optimiser.
 
 # Parameters
 - Learning rate (`η`): Amount by which gradients are discounted before updating

test/rules.jl

@@ -6,14 +6,14 @@ Random.seed!(1)
 
 RULES = [
   # All the rules at default settings:
-  Descent(), ADAM(), Momentum(), Nesterov(), RMSProp(),
-  ADAGrad(), AdaMax(), ADADelta(), AMSGrad(), NADAM(),
-  ADAMW(), RADAM(), OADAM(), AdaBelief(),
+  Descent(), Adam(), Momentum(), Nesterov(), RMSProp(),
+  AdaGrad(), AdaMax(), AdaDelta(), AMSGrad(), NAdam(),
+  AdamW(), RAdam(), OAdam(), AdaBelief(),
   # A few chained combinations:
-  OptimiserChain(WeightDecay(), ADAM(0.001)),
-  OptimiserChain(ClipNorm(), ADAM(0.001)),
+  OptimiserChain(WeightDecay(), Adam(0.001)),
+  OptimiserChain(ClipNorm(), Adam(0.001)),
   OptimiserChain(ClipGrad(0.5), Momentum()),
-  OptimiserChain(WeightDecay(), OADAM(), ClipGrad(1)),
+  OptimiserChain(WeightDecay(), OAdam(), ClipGrad(1)),
 ]
 
 name(o) = typeof(o).name.name  # just for printing testset headings
@@ -177,10 +177,10 @@ end
 @testset "with complex numbers: Flux#1776" begin
   empty!(LOG)
   @testset "$(name(opt))" for opt in [
-              # The Flux PR had 1e-2 for all. But ADADelta(ρ) needs ρ≈0.9 not small. And it helps to make ε not too small too:
-              ADAM(1e-2), RMSProp(1e-2), RADAM(1e-2), OADAM(1e-2), ADAGrad(1e-2), ADADelta(0.9, 1e-5), NADAM(1e-2), AdaBelief(1e-2),
+              # The Flux PR had 1e-2 for all. But AdaDelta(ρ) needs ρ≈0.9 not small. And it helps to make ε not too small too:
+              Adam(1e-2), RMSProp(1e-2), RAdam(1e-2), OAdam(1e-2), AdaGrad(1e-2), AdaDelta(0.9, 1e-5), NAdam(1e-2), AdaBelief(1e-2),
               # These weren't in Flux PR:
-              Descent(1e-2), Momentum(1e-2), Nesterov(1e-2), ADAMW(1e-2), 
+              Descent(1e-2), Momentum(1e-2), Nesterov(1e-2), AdamW(1e-2), 
               ]
     # Our "model" is just a complex number
     model = (w = zeros(ComplexF64, 1),)

test/runtests.jl

@@ -160,9 +160,9 @@ Optimisers.trainable(x::TwoThirds) = (a = x.a,)
       ok = (1.0:3.0, sin, "abc", :abc)
       m = (α = ok, β = rand(3), γ = ok)
       m1 = (rand(3), m, rand(3))
-      @test Optimisers.setup(ADAMW(), m1) isa Tuple
+      @test Optimisers.setup(AdamW(), m1) isa Tuple
       m2 = (rand(3), m, rand(3), m, rand(3))  # illegal
-      @test_throws ArgumentError Optimisers.setup(ADAMW(), m2)
+      @test_throws ArgumentError Optimisers.setup(AdamW(), m2)
     end
 
   end