bench: increase samples to reduce uncertainty

Author: franckgaga

benchmark/1_bench_sim_model.jl

@@ -1,5 +1,7 @@
-## ----------------- Unit tests  ----------------------------------------------------------
-const UNIT_MODEL = SUITE["unit tests"]["SimModel"]
+## ----------------------------------------------------------------------------------------
+## ----------------- UNIT TESTS  ----------------------------------------------------------
+## ----------------------------------------------------------------------------------------
+const UNIT_MODEL = SUITE["UNIT TESTS"]["SimModel"]
 
 UNIT_MODEL["LinModel"]["updatestate!"] = 
     @benchmarkable(
@@ -22,6 +24,8 @@ UNIT_MODEL["NonLinModel"]["linearize!"] =
         linearize!($linmodel, $nonlinmodel); 
     )
 
-## ----------------- Case studies ---------------------------------------------------------
-const CASE_MODEL = SUITE["case studies"]["SimModel"]
+## ----------------------------------------------------------------------------------------
+## ----------------- CASE STUDIES ---------------------------------------------------------
+## ----------------------------------------------------------------------------------------
+const CASE_MODEL = SUITE["CASE STUDIES"]["SimModel"]
 # TODO: Add case study benchmarks for SimModel

benchmark/2_bench_state_estim.jl

@@ -1,5 +1,7 @@
-## ----------------- Unit tests -----------------------------------------------------------
-const UNIT_ESTIM = SUITE["unit tests"]["StateEstimator"]
+## ----------------------------------------------------------------------------------------
+## ----------------- UNIT TESTS -----------------------------------------------------------
+## ----------------------------------------------------------------------------------------
+const UNIT_ESTIM = SUITE["UNIT TESTS"]["StateEstimator"]
 
 skf = SteadyKalmanFilter(linmodel)
 UNIT_ESTIM["SteadyKalmanFilter"]["preparestate!"] = 
@@ -193,8 +195,10 @@ UNIT_ESTIM["MovingHorizonEstimator"]["updatestate!"]["NonLinModel"]["Prediction
         samples=samples, evals=evals, seconds=seconds,
     ) 
 
-## ----------------- Case studies ---------------------------------------------------
-const CASE_ESTIM = SUITE["case studies"]["StateEstimator"]
+## ----------------------------------------------------------------------------------------
+## ----------------- CASE STUDIES ---------------------------------------------------------
+## ----------------------------------------------------------------------------------------
+const CASE_ESTIM = SUITE["CASE STUDIES"]["StateEstimator"]
 
 ## ----------------- Case study: CSTR -----------------------------------------------------
 G = [ tf(1.90, [18, 1]) tf(1.90, [18, 1]);
@@ -258,7 +262,7 @@ mhe_cstr_ipopt_pred = MovingHorizonEstimator(model; He, nint_u, σQint_u, optim,
 mhe_cstr_ipopt_pred = setconstraint!(mhe_cstr_ipopt_pred, v̂min=[-1, -0.5], v̂max=[+1, +0.5])
 JuMP.unset_time_limit_sec(mhe_cstr_ipopt_pred.optim)
 
-samples, evals = 500, 1
+samples, evals = 5000, 1
 CASE_ESTIM["CSTR"]["MovingHorizonEstimator"]["OSQP"]["Current form"] =
     @benchmarkable(test_mhe($mhe_cstr_osqp_curr, $plant); 
         samples=samples, evals=evals

benchmark/3_bench_predictive_control.jl

@@ -1,5 +1,7 @@
-# ---------------------- Unit tests -------------------------------------------------------
-const UNIT_MPC = SUITE["unit tests"]["PredictiveController"]
+# -----------------------------------------------------------------------------------------
+# ---------------------- UNIT TESTS -------------------------------------------------------
+# -----------------------------------------------------------------------------------------
+const UNIT_MPC = SUITE["UNIT TESTS"]["PredictiveController"]
 
 linmpc_ss = LinMPC(
     linmodel, transcription=SingleShooting(), 
@@ -10,7 +12,7 @@ linmpc_ms = LinMPC(
     Mwt=[1, 1], Nwt=[0.1, 0.1], Lwt=[0.1, 0.1], Hp=10
 )
 
-samples, evals, seconds = 500, 1, 60
+samples, evals, seconds = 5000, 1, 60
 UNIT_MPC["LinMPC"]["moveinput!"]["SingleShooting"] = 
     @benchmarkable(
         moveinput!($linmpc_ss, $y, $d),
@@ -49,7 +51,7 @@ nmpc_nonlin_ms = NonLinMPC(
     Mwt=[1, 1], Nwt=[0.1, 0.1], Lwt=[0.1, 0.1], Hp=10    
 )
 
-samples, evals, seconds = 500, 1, 60
+samples, evals, seconds = 5000, 1, 60
 UNIT_MPC["NonLinMPC"]["moveinput!"]["LinModel"]["SingleShooting"] =
     @benchmarkable(
         moveinput!($nmpc_lin_ss, $y, $d),
@@ -75,9 +77,10 @@ UNIT_MPC["NonLinMPC"]["moveinput!"]["NonLinModel"]["MultipleShooting"] =
         samples=samples, evals=evals, seconds=seconds
     )
 
-
-## ---------------------- Case studies ----------------------------------------------------
-const CASE_MPC = SUITE["case studies"]["PredictiveController"]
+## ----------------------------------------------------------------------------------------
+## ---------------------- CASE STUDIES ----------------------------------------------------
+## ----------------------------------------------------------------------------------------
+const CASE_MPC = SUITE["CASE STUDIES"]["PredictiveController"]
 
 ## ----------------- Case study: CSTR without feedforward ------------------------
 G = [ tf(1.90, [18, 1]) tf(1.90, [18, 1]);
@@ -135,7 +138,7 @@ transcription = MultipleShooting()
 mpc_ipopt_ms = setconstraint!(LinMPC(model; optim, transcription), ymin=[45, -Inf])
 JuMP.unset_time_limit_sec(mpc_ipopt_ms.optim) 
 
-samples, evals = 500, 1
+samples, evals = 5000, 1
 CASE_MPC["CSTR"]["LinMPC"]["Without feedforward"]["OSQP"]["SingleShooting"] = 
     @benchmarkable(test_mpc($mpc_osqp_ss, $plant); 
         samples=samples, evals=evals
@@ -209,7 +212,7 @@ transcription = MultipleShooting()
 mpc_d_ipopt_ms = setconstraint!(LinMPC(model_d; optim, transcription), ymin=[45, -Inf])
 JuMP.unset_time_limit_sec(mpc_d_ipopt_ms.optim)
 
-samples, evals = 500, 1
+samples, evals = 5000, 1
 CASE_MPC["CSTR"]["LinMPC"]["With feedforward"]["OSQP"]["SingleShooting"] = 
     @benchmarkable(test_mpc_d($mpc_d_osqp_ss, $plant); 
         samples=samples, evals=evals
@@ -285,7 +288,7 @@ JuMP.unset_time_limit_sec(nmpc_madnlp_ss.optim)
 # MadNLP_QNopt = MadNLP.QuasiNewtonOptions(; max_history=42)
 # JuMP.set_attribute(nmpc_madnlp_ms.optim, "quasi_newton_options", MadNLP_QNopt)
 
-samples, evals, seconds = 50, 1, 15*60
+samples, evals, seconds = 100, 1, 15*60
 CASE_MPC["Pendulum"]["NonLinMPC"]["Noneconomic"]["Ipopt"]["SingleShooting"] = 
     @benchmarkable(
         sim!($nmpc_ipopt_ss, $N, $ry; plant=$plant, x_0=$x_0, x̂_0=$x̂_0),
@@ -336,7 +339,7 @@ JuMP.unset_time_limit_sec(empc_madnlp_ss.optim)
 
 # TODO: test EMPC with MadNLP and MultipleShooting, see comment above.
 
-samples, evals, seconds = 50, 1, 15*60
+samples, evals, seconds = 100, 1, 15*60
 CASE_MPC["Pendulum"]["NonLinMPC"]["Economic"]["Ipopt"]["SingleShooting"] = 
     @benchmarkable(
         sim!($empc_ipopt_ss, $N, $ry; plant=$plant2, x_0=$x_0, x̂_0=$x̂_0),
@@ -388,7 +391,7 @@ JuMP.unset_time_limit_sec(nmpc2_ipopt_ms.optim)
 # TODO: test custom constraints with MadNLP and SingleShooting, see comment above.
 # TODO: test custom constraints with MadNLP and MultipleShooting, see comment above.
 
-samples, evals, seconds = 50, 1, 15*60
+samples, evals, seconds = 100, 1, 15*60
 CASE_MPC["Pendulum"]["NonLinMPC"]["Custom constraints"]["Ipopt"]["SingleShooting"] = 
     @benchmarkable(
         sim!($nmpc2_ipopt_ss, $N, $ry; plant=$plant2, x_0=$x_0, x̂_0=$x̂_0),
@@ -458,7 +461,7 @@ mpc3_ipopt_ms = LinMPC(kf; Hp, Hc, Mwt, Nwt, Cwt, optim, transcription)
 mpc3_ipopt_ms = setconstraint!(mpc3_ipopt_ms; umin, umax)
 JuMP.unset_time_limit_sec(mpc3_ipopt_ms.optim)
 
-samples, evals = 500, 1
+samples, evals = 5000, 1
 CASE_MPC["Pendulum"]["LinMPC"]["Successive linearization"]["OSQP"]["SingleShooting"] = 
     @benchmarkable(
         sim2!($mpc3_osqp_ss, $model, $N, $ry, $plant, $x_0, $x̂_0, $y_step),

benchmark/benchmarks.jl

@@ -4,8 +4,8 @@ using JuMP, OSQP, DAQP, Ipopt, MadNLP
 
 const SUITE = BenchmarkGroup(["ModelPredictiveControl"])
 
-SUITE["unit tests"]   = BenchmarkGroup(["allocation-free", "allocations", "single call"])
-SUITE["case studies"] = BenchmarkGroup(["performance", "speed" ,"integration"])
+SUITE["UNIT TESTS"]   = BenchmarkGroup(["allocation-free", "allocations", "single call"])
+SUITE["CASE STUDIES"] = BenchmarkGroup(["performance", "speed" ,"integration"])
 
 include("0_bench_setup.jl")
 include("1_bench_sim_model.jl")