Merge pull request #105 from mkg33/master

Reorganized documentation

Author: ChrisRackauckas

docs/make.jl

@@ -12,22 +12,29 @@ makedocs(
 
     pages=[
         "GalacticOptim.jl: Unified Global Optimization Package" => "index.md",
+
         "Tutorials" => [
-            "Introduction to GalacticOptim.jl" => "tutorials/intro.md"
+            "Introduction to GalacticOptim.jl" => "tutorials/intro.md",
+            "Rosenbrock function" => "tutorials/rosenbrock.md",
+            "Minibatch" => "tutorials/minibatch.md"
+        ],
+
+        "API" => [
+            "OptimizationProblem" => "API/optimization_problem.md",
+            "OptimizationFunction" => "API/optimization_function.md",
+            "solve" => "API/solve.md"
         ],
-        "Basics" => [
-            "OptimizationProblem" => "basics/problem.md",
-            "Solver Options" => "basics/solve.md"
+
+        "Global Optimizers" => [
+            "global_optimizers/global.md",
+            "global_optimizers/global_constrained.md"
         ],
+
         "Local Optimizers" => [
             "local_optimizers/local_gradient.md",
             "local_optimizers/local_derivative_free.md",
             "local_optimizers/local_hessian.md",
-            "local_optimizers/local_hessian_free.md",
-        ],
-        "Global Optimizers" => [
-            "global_optimizers/global.md",
-            "global_optimizers/global_constrained.md"
+            "local_optimizers/local_hessian_free.md"
         ]
     ]
 )

docs/src/basics/problem.md renamed to docs/src/API/optimization_function.md

@@ -1,29 +1,4 @@
-# Defining OptimizationProblems
-
-All optimizations start by defining an `OptimizationProblem` as follows:
-
-```julia
-OptimizationProblem(f, x, p = DiffEqBase.NullParameters(),;
-                    lb = nothing,
-                    ub = nothing,
-                    lcons = nothing,
-                    ucons = nothing,
-                    kwargs...)
-```
-
-Formally, the `OptimizationProblem` finds the minimum of `f(x,p)` with an
-initial condition `x`. The parameters `p` are optional. `lb` and `ub`
-are arrays matching the size of `x` which stand for the lower and upper
-bounds of `x` respectively.
-
-`f` is an `OptimizationFunction`, as defined below. If `f` is a
-standard Julia function, it is automatically converted into an
-`OptimizationFunction` with `NoAD()`, i.e. no automatic generation
-of the derivative functions.
-
-Any extra keyword arguments are captured to be sent to the optimizers.
-
-## OptimizationFunction
+# [OptimizationFunction](@id optfunction)
 
 The `OptimizationFunction` type is a function type that holds all of
 the extra differentiation data required to do fast and accurate

docs/src/API/optimization_problem.md

@@ -0,0 +1,24 @@
+# Defining OptimizationProblems
+
+All optimizations start by defining an `OptimizationProblem` as follows:
+
+```julia
+OptimizationProblem(f, x, p = DiffEqBase.NullParameters(),;
+                    lb = nothing,
+                    ub = nothing,
+                    lcons = nothing,
+                    ucons = nothing,
+                    kwargs...)
+```
+
+Formally, the `OptimizationProblem` finds the minimum of `f(x,p)` with an
+initial condition `x`. The parameters `p` are optional. `lb` and `ub`
+are arrays matching the size of `x`, which stand for the lower and upper
+bounds of `x`, respectively.
+
+`f` is an `OptimizationFunction`, as defined [here](@ref optfunction).
+If `f` is a standard Julia function, it is automatically converted into an
+`OptimizationFunction` with `NoAD()`, i.e., no automatic generation
+of the derivative functions.
+
+Any extra keyword arguments are captured to be sent to the optimizers.

docs/src/basics/solve.md renamed to docs/src/API/solve.md

@@ -0,0 +1,65 @@
+# Minibatch examples
+
+```julia
+using DiffEqFlux, GalacticOptim, OrdinaryDiffEq
+
+function newtons_cooling(du, u, p, t)
+    temp = u[1]
+    k, temp_m = p
+    du[1] = dT = -k*(temp-temp_m)
+  end
+
+function true_sol(du, u, p, t)
+    true_p = [log(2)/8.0, 100.0]
+    newtons_cooling(du, u, true_p, t)
+end
+
+function dudt_(u,p,t)
+    ann(u, p).* u
+end
+
+cb = function (p,l,pred;doplot=false) #callback function to observe training
+    display(l)
+    # plot current prediction against data
+    if doplot
+      pl = scatter(t,ode_data[1,:],label="data")
+      scatter!(pl,t,pred[1,:],label="prediction")
+      display(plot(pl))
+    end
+    return false
+end
+
+u0 = Float32[200.0]
+datasize = 30
+tspan = (0.0f0, 1.5f0)
+
+t = range(tspan[1], tspan[2], length=datasize)
+true_prob = ODEProblem(true_sol, u0, tspan)
+ode_data = Array(solve(true_prob, Tsit5(), saveat=t))
+
+ann = FastChain(FastDense(1,8,tanh), FastDense(8,1,tanh))
+pp = initial_params(ann)
+prob = ODEProblem{false}(dudt_, u0, tspan, pp)
+
+function predict_adjoint(fullp, time_batch)
+    Array(solve(prob, Tsit5(), p = fullp, saveat = time_batch))
+end
+
+function loss_adjoint(fullp, batch, time_batch)
+    pred = predict_adjoint(fullp,time_batch)
+    sum(abs2, batch .- pred), pred
+end
+
+
+k = 10
+train_loader = Flux.Data.DataLoader(ode_data, t, batchsize = k)
+
+numEpochs = 300
+l1 = loss_adjoint(pp, train_loader.data[1], train_loader.data[2])[1]
+
+optfun = OptimizationFunction((θ, p, batch, time_batch) -> loss_adjoint(θ, batch, time_batch), GalacticOptim.AutoZygote())
+optprob = OptimizationProblem(optfun, pp)
+using IterTools: ncycle
+res1 = GalacticOptim.solve(optprob, ADAM(0.05), ncycle(train_loader, numEpochs), cb = cb, maxiters = numEpochs)
+@test 10res1.minimum < l1
+```

docs/src/tutorials/minibatch.md

@@ -0,0 +1,120 @@
+# Rosenbrock function examples
+
+```julia
+using GalacticOptim, Optim, Test, Random
+
+rosenbrock(x, p) =  (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2
+x0 = zeros(2)
+_p  = [1.0, 100.0]
+
+f = OptimizationFunction(rosenbrock, GalacticOptim.AutoForwardDiff())
+l1 = rosenbrock(x0, _p)
+prob = OptimizationProblem(f, x0, _p)
+sol = solve(prob, SimulatedAnnealing())
+@test 10*sol.minimum < l1
+
+Random.seed!(1234)
+prob = OptimizationProblem(f, x0, _p, lb=[-1.0, -1.0], ub=[0.8, 0.8])
+sol = solve(prob, SAMIN())
+@test 10*sol.minimum < l1
+
+using CMAEvolutionStrategy
+sol = solve(prob, CMAEvolutionStrategyOpt())
+@test 10*sol.minimum < l1
+
+rosenbrock(x, p=nothing) =  (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2
+
+l1 = rosenbrock(x0)
+prob = OptimizationProblem(rosenbrock, x0)
+sol = solve(prob, NelderMead())
+@test 10*sol.minimum < l1
+
+cons= (x,p) -> [x[1]^2 + x[2]^2]
+optprob = OptimizationFunction(rosenbrock, GalacticOptim.AutoForwardDiff();cons= cons)
+
+prob = OptimizationProblem(optprob, x0)
+
+sol = solve(prob, ADAM(0.1), maxiters = 1000)
+@test 10*sol.minimum < l1
+
+sol = solve(prob, BFGS())
+@test 10*sol.minimum < l1
+
+sol = solve(prob, Newton())
+@test 10*sol.minimum < l1
+
+sol = solve(prob, Optim.KrylovTrustRegion())
+@test 10*sol.minimum < l1
+
+prob = OptimizationProblem(optprob, x0, lcons = [-Inf], ucons = [Inf])
+sol = solve(prob, IPNewton())
+@test 10*sol.minimum < l1
+
+prob = OptimizationProblem(optprob, x0, lcons = [-5.0], ucons = [10.0])
+sol = solve(prob, IPNewton())
+@test 10*sol.minimum < l1
+
+prob = OptimizationProblem(optprob, x0, lcons = [-Inf], ucons = [Inf], lb = [-500.0,-500.0], ub=[50.0,50.0])
+sol = solve(prob, IPNewton())
+@test sol.minimum < l1
+
+function con2_c(x,p)
+    [x[1]^2 + x[2]^2, x[2]*sin(x[1])-x[1]]
+end
+
+optprob = OptimizationFunction(rosenbrock, GalacticOptim.AutoForwardDiff();cons= con2_c)
+prob = OptimizationProblem(optprob, x0, lcons = [-Inf,-Inf], ucons = [Inf,Inf])
+sol = solve(prob, IPNewton())
+@test 10*sol.minimum < l1
+
+cons_circ = (x,p) -> [x[1]^2 + x[2]^2]
+optprob = OptimizationFunction(rosenbrock, GalacticOptim.AutoForwardDiff();cons= cons_circ)
+prob = OptimizationProblem(optprob, x0, lcons = [-Inf], ucons = [0.25^2])
+sol = solve(prob, IPNewton())
+@test sqrt(cons(sol.minimizer,nothing)[1]) ≈ 0.25 rtol = 1e-6
+
+optprob = OptimizationFunction(rosenbrock, GalacticOptim.AutoZygote())
+prob = OptimizationProblem(optprob, x0)
+sol = solve(prob, ADAM(), maxiters = 1000, progress = false)
+@test 10*sol.minimum < l1
+
+prob = OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
+sol = solve(prob, Fminbox())
+@test 10*sol.minimum < l1
+
+prob = OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
+@test_broken @test_nowarn sol = solve(prob, SAMIN())
+@test 10*sol.minimum < l1
+
+using NLopt
+prob = OptimizationProblem(optprob, x0)
+sol = solve(prob, Opt(:LN_BOBYQA, 2))
+@test 10*sol.minimum < l1
+
+sol = solve(prob, Opt(:LD_LBFGS, 2))
+@test 10*sol.minimum < l1
+
+prob = OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
+sol = solve(prob, Opt(:LD_LBFGS, 2))
+@test 10*sol.minimum < l1
+
+sol = solve(prob, Opt(:G_MLSL_LDS, 2), nstart=2, local_method = Opt(:LD_LBFGS, 2), maxiters=10000)
+@test 10*sol.minimum < l1
+
+# using MultistartOptimization
+# sol = solve(prob, MultistartOptimization.TikTak(100), local_method = NLopt.LD_LBFGS)
+# @test 10*sol.minimum < l1
+
+# using QuadDIRECT
+# sol = solve(prob, QuadDirect(); splits = ([-0.5, 0.0, 0.5],[-0.5, 0.0, 0.5]))
+# @test 10*sol.minimum < l1
+
+using Evolutionary
+sol = solve(prob, CMAES(μ =40 , λ = 100),abstol=1e-15)
+@test 10*sol.minimum < l1
+
+using BlackBoxOptim
+prob = GalacticOptim.OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
+sol = solve(prob, BBO())
+@test 10*sol.minimum < l1
+```