bench: added allocation benchmarks on CI

.github/workflows/benchmark.yml

@@ -0,0 +1,13 @@
+name: Benchmark this PR
+on:
+  pull_request_target:
+    branches: [ main ]
+permissions:
+  pull-requests: write    # needed to post comments
+jobs:
+  bench:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: MilesCranmer/AirspeedVelocity.jl@action-v1
+        with:
+          julia-version: '1'

benchmark/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+ControlSystemsBase = "aaaaaaaa-a6ca-5380-bf3e-84a91bcd477e"
 DAQP = "c47d62df-3981-49c8-9651-128b1cd08617"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"

benchmark/benchmarks.jl

@@ -1,25 +1,141 @@
 using BenchmarkTools
-using Random
+using ModelPredictiveControl, ControlSystemsBase, LinearAlgebra
+
+Ts = 400.0
+sys = [ tf(1.90,[1800.0,1])   tf(1.90,[1800.0,1])   tf(1.90,[1800.0,1]);
+        tf(-0.74,[800.0,1])   tf(0.74,[800.0,1])    tf(-0.74,[800.0,1])   ] 
 
 const SUITE = BenchmarkGroup()
 
-SUITE["utf8"] = BenchmarkGroup(["string", "unicode"])
-teststr = String(join(rand(MersenneTwister(1), 'a':'d', 10^4)))
-SUITE["utf8"]["replace"] = @benchmarkable replace($teststr, "a" => "b")
-SUITE["utf8"]["join"] = @benchmarkable join($teststr, $teststr)
-SUITE["utf8"]["plots"] = BenchmarkGroup()
-
-SUITE["trigonometry"] = BenchmarkGroup(["math", "triangles"])
-SUITE["trigonometry"]["circular"] = BenchmarkGroup()
-for f in (sin, cos, tan)
-    for x in (0.0, pi)
-        SUITE["trigonometry"]["circular"][string(f), x] = @benchmarkable ($f)($x)
-    end
+## ==================================================================================
+## ================== SimModel benchmarks ===========================================
+## ==================================================================================
+linmodel = setop!(LinModel(sys, Ts, i_d=[3]), uop=[10, 50], yop=[50, 30], dop=[5])
+function f!(ẋ, x, u, d, p)
+    mul!(ẋ, p.A, x)
+    mul!(ẋ, p.Bu, u, 1, 1)
+    mul!(ẋ, p.Bd, d, 1, 1)
+    return nothing
+end
+function h!(y, x, d, p)
+    mul!(y, p.C, x)
+    mul!(y, p.Dd, d, 1, 1)
+    return nothing
 end
+nonlinmodel = NonLinModel(f!, h!, Ts, 2, 4, 2, 1, p=linmodel, solver=nothing)
+nonlinmodel = setop!(nonlinmodel, uop=[10, 50], yop=[50, 30], dop=[5])
+u, d, y = [10, 50], [5], [50, 30]
+
+SUITE["SimModel"]["allocation"] = BenchmarkGroup(["allocation"])
+SUITE["SimModel"]["allocation"]["LinModel_updatestate!"] = @benchmarkable(
+    updatestate!($linmodel, $u, $d),
+    samples=1
+)
+SUITE["SimModel"]["allocation"]["LinModel_evaloutput"] = @benchmarkable(
+    evaloutput($linmodel, $d),
+    samples=1
+)
+SUITE["SimModel"]["allocation"]["NonLinModel_updatestate!"] = @benchmarkable(
+    updatestate!($nonlinmodel, $u, $d),
+    samples=1
+)
+SUITE["SimModel"]["allocation"]["NonLinModel_evaloutput"] = @benchmarkable(
+    evaloutput($nonlinmodel, $d),
+    samples=1
+)
+
+SUITE["SimModel"]["allocation"]["NonLinModel_linearize!"] = @benchmarkable(
+    linearize!($linmodel, $nonlinmodel),
+    samples=1
+)
+
+## ==================================================================================
+## ================== StateEstimator benchmarks =====================================
+## ==================================================================================
+skf = SteadyKalmanFilter(linmodel)
+SUITE["StateEstimator"]["allocation"] = BenchmarkGroup(["allocation"])
+SUITE["StateEstimator"]["allocation"]["SteadyKalmanFilter_preparestate!"] = @benchmarkable(
+    preparestate!($skf, $y, $d),
+    samples=1
+)
+SUITE["StateEstimator"]["allocation"]["SteadyKalmanFilter_updatestate!"] = @benchmarkable(
+    updatestate!($skf, $u, $y, $d),
+    setup=preparestate!($skf, $y, $d),
+    samples=1
+)
+SUITE["StateEstimator"]["allocation"]["SteadyKalmanFilter_evaloutput"] = @benchmarkable(
+    evaloutput($skf, $d),
+    setup=preparestate!($skf, $y, $d),
+    samples=1
+)
+
+kf = KalmanFilter(linmodel, nint_u=[1, 1], direct=false)
+SUITE["StateEstimator"]["allocation"]["KalmanFilter_preparestate!"] = @benchmarkable(
+    preparestate!($kf, $y, $d),
+    samples=1
+)
+SUITE["StateEstimator"]["allocation"]["KalmanFilter_updatestate!"] = @benchmarkable(
+    updatestate!($kf, $u, $y, $d),
+    setup=preparestate!($kf, $y, $d),
+    samples=1
+)
+
+lo = Luenberger(linmodel, nint_u=[1, 1])
+#SUITE["StateEstimator"]["allocation"]["Luenberger_preparestate!"] = @benchmarkable(
+#    preparestate!($lo, $y, $d),
+#    samples=1
+#)
+SUITE["StateEstimator"]["allocation"]["Luenberger_updatestate!"] = @benchmarkable(
+    updatestate!($lo, $u, $y, $d),
+    setup=preparestate!($lo, $y, $d),
+    samples=1
+)
+
+im = InternalModel(nonlinmodel)
+SUITE["StateEstimator"]["allocation"]["InternalModel_preparestate!"] = @benchmarkable(
+    preparestate!($im, $y, $d),
+    samples=1
+)
+SUITE["StateEstimator"]["allocation"]["InternalModel_updatestate!"] = @benchmarkable(
+    updatestate!($im, $u, $y, $d),
+    setup=preparestate!($im, $y, $d),
+    samples=1
+)
+
+ukf = UnscentedKalmanFilter(nonlinmodel)
+SUITE["StateEstimator"]["allocation"]["UnscentedKalmanFilter_preparestate!"] = @benchmarkable(
+    preparestate!($ukf, $y, $d),
+    samples=1
+)
+SUITE["StateEstimator"]["allocation"]["UnscentedKalmanFilter_updatestate!"] = @benchmarkable(
+    updatestate!($ukf, $u, $y,  $d),
+    setup=preparestate!($ukf, $y, $d),
+    samples=1
+)
+SUITE["StateEstimator"]["allocation"]["UnscentedKalmanFilter_evaloutput"] = @benchmarkable(
+    evaloutput($ukf, $d),
+    setup=preparestate!($ukf, $y, $d),
+    samples=1
+)
+
+ekf = ExtendedKalmanFilter(linmodel, nint_u=[1, 1], direct=false)
+SUITE["StateEstimator"]["allocation"]["ExtendedKalmanFilter_preparestate!"] = @benchmarkable(
+    preparestate!($ekf, $y, $d),
+    samples=1
+)
+SUITE["StateEstimator"]["allocation"]["ExtendedKalmanFilter_updatestate!"] = @benchmarkable(
+    updatestate!($ekf, $u, $y, $d),
+    setup=preparestate!($ekf, $y, $d),
+    samples=1
+)
 
-SUITE["trigonometry"]["hyperbolic"] = BenchmarkGroup()
-for f in (sin, cos, tan)
-    for x in (0.0, pi)
-        SUITE["trigonometry"]["hyperbolic"][string(f), x] = @benchmarkable ($f)($x)
-    end
-end
+## ==================================================================================
+## ================== PredictiveController benchmarks ===============================
+## ==================================================================================
+empc = ExplicitMPC(linmodel, Mwt=[1, 1], Nwt=[0.1, 0.1], Lwt=[0.1, 0.1])
+SUITE["PredictiveController"]["allocation"] = BenchmarkGroup(["allocation"])
+SUITE["PredictiveController"]["allocation"]["ExplicitMPC_moveinput!"] = @benchmarkable(
+    moveinput!($empc, $y, $d),
+    setup=preparestate!($empc, $y, $d),
+    samples=1
+)

test/1_test_sim_model.jl

@@ -108,11 +108,9 @@ end
     u, d = [10, 50], Float64[]
     @test updatestate!(linmodel1, u) ≈ zeros(2)
     @test updatestate!(linmodel1, u, d) ≈ zeros(2)
-    @test_skip @allocations(updatestate!(linmodel1, u)) == 0
     @test linmodel1.x0 ≈ zeros(2)
     @test evaloutput(linmodel1) ≈ linmodel1() ≈ [50,30]
     @test evaloutput(linmodel1, Float64[]) ≈ linmodel1(Float64[]) ≈ [50,30]
-    @test_skip @allocations(evaloutput(linmodel1)) == 0
     x = initstate!(linmodel1, [10, 60])
     @test evaloutput(linmodel1) ≈ [50 + 19.0, 30 + 7.4]
     @test preparestate!(linmodel1) ≈ x           # new method
@@ -283,11 +281,9 @@ end
     u, d = zeros(2), Float64[]
     @test updatestate!(nonlinmodel, u) ≈ zeros(2) 
     @test updatestate!(nonlinmodel, u, d) ≈ zeros(2)
-    @test_skip @allocations(updatestate!(nonlinmodel, u)) == 0
     @test nonlinmodel.x0 ≈ zeros(2)
     @test evaloutput(nonlinmodel) ≈ nonlinmodel() ≈ zeros(2)
     @test evaloutput(nonlinmodel, d) ≈ nonlinmodel(Float64[]) ≈ zeros(2)
-    @test_skip @allocations(evaloutput(nonlinmodel)) == 0
 
     x = initstate!(nonlinmodel, [0, 10]) # do nothing for NonLinModel
     @test evaloutput(nonlinmodel) ≈ [0, 0]
@@ -375,19 +371,6 @@ end
     h2!(y, x, _, _) = (y .= x)
     nonlinmodel4 = NonLinModel(f2!,h2!,Ts,1,1,1,0,solver=nothing,jacobian=AutoFiniteDiff())
     @test_nowarn linearize(nonlinmodel4, x=[1], u=[2])
-
-    function f3!(xnext, x, u, d, _)
-        xnext .= x.*u .+ x.*d
-    end
-    function h3!(y, x, d, _)
-        y .= x.*d
-    end
-    nonlinmodel4 = NonLinModel(f3!, h3!, Ts, 1, 1, 1, 1, solver=nothing)
-    linmodel4 = linearize(nonlinmodel4; x, u, d)
-    # return nothing (see this issue : https://github.com/JuliaLang/julia/issues/51112):
-    linearize2!(linmodel, model) = (linearize!(linmodel, model); nothing)
-    linearize2!(linmodel4, nonlinmodel4)
-    @test_skip @allocations(linearize2!(linmodel4, nonlinmodel4)) == 0
 end
 
 @testitem "NonLinModel real time simulations" setup=[SetupMPCtests] begin

test/2_test_state_estim.jl

@@ -71,12 +71,9 @@ end
     preparestate!(kalmanfilter1, y)
     @test updatestate!(kalmanfilter1, u, y, d) ≈ zeros(4)
     @test kalmanfilter1.x̂0 ≈ zeros(4)
-    @test_skip @allocations(preparestate!(kalmanfilter1, y)) == 0
-    @test_skip @allocations(updatestate!(kalmanfilter1, u, y)) == 0
     preparestate!(kalmanfilter1, y)
     @test evaloutput(kalmanfilter1) ≈ kalmanfilter1() ≈ [50, 30]
     @test evaloutput(kalmanfilter1, d) ≈ kalmanfilter1(d) ≈ [50, 30]
-    @test_skip @allocations(evaloutput(kalmanfilter1, d)) == 0
     @test initstate!(kalmanfilter1, [10, 50], [50, 30+1]) ≈ [zeros(3); [1]]
     linmodel2 = LinModel(append(tf(1, [1, 0]), tf(2, [10, 1])), 1.0)
     kalmanfilter2 = SteadyKalmanFilter(linmodel2, nint_u=[1, 1], direct=false)
@@ -205,12 +202,9 @@ end
     preparestate!(kalmanfilter1, y)
     @test updatestate!(kalmanfilter1, u, y, d) ≈ zeros(4)
     @test kalmanfilter1.x̂0 ≈ zeros(4)
-    @test_skip @allocations(preparestate!(kalmanfilter1, y)) == 0
-    @test_skip @allocations(updatestate!(kalmanfilter1, u, y)) == 0
     preparestate!(kalmanfilter1, y)
     @test evaloutput(kalmanfilter1) ≈ kalmanfilter1() ≈ [50, 30]
     @test evaloutput(kalmanfilter1, d) ≈ kalmanfilter1(d) ≈ [50, 30]
-    @test_skip @allocations(evaloutput(kalmanfilter1, d)) == 0
     @test initstate!(kalmanfilter1, [10, 50], [50, 30+1]) ≈ [zeros(3); [1]]
     setstate!(kalmanfilter1, [1,2,3,4], diagm(.1:.1:.4))
     @test kalmanfilter1.x̂0 ≈ [1,2,3,4]
@@ -326,12 +320,9 @@ end
     preparestate!(lo1, y)
     @test updatestate!(lo1, u, y, d) ≈ zeros(4)
     @test lo1.x̂0 ≈ zeros(4)
-    @test_skip @allocations(preparestate!(lo1, y)) == 0
-    @test_skip @allocations(updatestate!(lo1, u, y)) == 0
     preparestate!(lo1, y)
     @test evaloutput(lo1) ≈ lo1() ≈ [50, 30]
     @test evaloutput(lo1, d) ≈ lo1(d) ≈ [50, 30]
-    @test_skip @allocations(evaloutput(lo1, d)) == 0
     @test initstate!(lo1, [10, 50], [50, 30+1]) ≈ [zeros(3); [1]]
     setstate!(lo1, [1,2,3,4])
     @test lo1.x̂0 ≈ [1,2,3,4]
@@ -457,11 +448,8 @@ end
     @test updatestate!(internalmodel1, u, y, d) ≈ zeros(2)
     @test internalmodel1.x̂d ≈ internalmodel1.x̂0 ≈ zeros(2)
     @test internalmodel1.x̂s ≈ ones(2)
-    @test_skip @allocations(preparestate!(internalmodel1, y)) == 0
-    @test_skip @allocations(updatestate!(internalmodel1, u, y)) == 0
     preparestate!(internalmodel1, y)
     @test evaloutput(internalmodel1, d) ≈ [51,31]
-    @test_skip @allocations(evaloutput(internalmodel1, d)) == 0
     @test initstate!(internalmodel1, [10, 50], [50, 30]) ≈ zeros(2)
     @test internalmodel1.x̂s ≈ zeros(2)
     setstate!(internalmodel1, [1,2])
@@ -593,12 +581,9 @@ end
     preparestate!(ukf1, y)
     @test updatestate!(ukf1, u, y, d) ≈ zeros(4) atol=1e-9
     @test ukf1.x̂0 ≈ zeros(4) atol=1e-9
-    @test_skip @allocations(preparestate!(ukf1, y)) == 0
-    @test_skip @allocations(updatestate!(ukf1, u, y)) == 0
     preparestate!(ukf1, y)
     @test evaloutput(ukf1) ≈ ukf1() ≈ [50, 30]
     @test evaloutput(ukf1, d) ≈ ukf1(d) ≈ [50, 30]
-    @test_skip @allocations(evaloutput(ukf1, d)) == 0
     @test initstate!(ukf1, [10, 50], [50, 30+1]) ≈ zeros(4) atol=1e-9
     setstate!(ukf1, [1,2,3,4], diagm(.1:.1:.4))
     @test ukf1.x̂0 ≈ [1,2,3,4]
@@ -752,12 +737,9 @@ end
     preparestate!(ekf1, y)
     @test updatestate!(ekf1, u, y, d) ≈ zeros(4) atol=1e-9
     @test ekf1.x̂0 ≈ zeros(4) atol=1e-9
-    @test_skip @allocations(preparestate!(ekf1, y)) == 0
-    @test_skip @allocations(updatestate!(ekf1, u, y)) == 0
     preparestate!(ekf1, y)
     @test evaloutput(ekf1) ≈ ekf1() ≈ [50, 30]
     @test evaloutput(ekf1, d) ≈ ekf1(d) ≈ [50, 30]
-    @test_skip @allocations(evaloutput(ekf1, d)) == 0
     @test initstate!(ekf1, [10, 50], [50, 30+1]) ≈ zeros(4);
     setstate!(ekf1, [1,2,3,4], diagm(.1:.1:.4))
     @test ekf1.x̂0 ≈ [1,2,3,4]

test/3_test_predictive_control.jl

@@ -496,7 +496,6 @@ end
     @test u ≈ [1] atol=1e-2
     u = mpc1(r)
     @test u ≈ [1] atol=1e-2
-    @test_skip @allocations(moveinput!(mpc1, r)) == 0
     info = getinfo(mpc1)
     @test info[:u] ≈ u
     @test info[:Ŷ][end] ≈ r[1] atol=1e-2