More accurate benchmarks

Author: MilesCranmer

benchmark/benchmarks.jl

@@ -3,48 +3,19 @@ using DynamicExpressions: copy_node
 
 include("benchmark_utils.jl")
 
-const v_PACKAGE_VERSION = try
-    VersionNumber(DynamicExpressions.PACKAGE_VERSION)
-catch
-    VersionNumber("v0.0.0")
-end
-
 const SUITE = BenchmarkGroup()
 
 function benchmark_evaluation()
     suite = BenchmarkGroup()
     operators = OperatorEnum(;
         binary_operators=[+, -, /, *], unary_operators=[cos, exp], enable_autodiff=true
     )
-    simple_tree = Node(
-        2,
-        Node(
-            1,
-            Node(
-                3,
-                Node(1, Node(; val=1.0f0), Node(; feature=2)),
-                Node(2, Node(; val=-1.0f0)),
-            ),
-            Node(1, Node(; feature=3), Node(; feature=4)),
-        ),
-        Node(
-            4,
-            Node(
-                3,
-                Node(1, Node(; val=1.0f0), Node(; feature=2)),
-                Node(2, Node(; val=-1.0f0)),
-            ),
-            Node(1, Node(; feature=3), Node(; feature=4)),
-        ),
-    )
     for T in (ComplexF32, ComplexF64, Float32, Float64)
-        if !(T <: Real) && v_PACKAGE_VERSION < v"0.5.0" && v_PACKAGE_VERSION != v"0.0.0"
+        if !(T <: Real) && PACKAGE_VERSION < v"0.5.0" && PACKAGE_VERSION != v"0.0.0"
             continue
         end
         suite[T] = BenchmarkGroup()
 
-        evals = 10
-        samples = 1_000
         n = 1_000
 
         #! format: off
@@ -53,25 +24,36 @@ function benchmark_evaluation()
                 continue
             end
             extra_key = turbo ? "_turbo" : ""
+            eval_tree_array(
+                gen_random_tree_fixed_size(20, operators, 5, T),
+                randn(MersenneTwister(0), T, 5, n),
+                operators;
+                turbo=turbo
+            )
             suite[T]["evaluation$(extra_key)"] = @benchmarkable(
-                eval_tree_array(tree, X, $operators; turbo=$turbo),
-                evals=evals,
-                samples=samples,
-                seconds=5.0,
+                [eval_tree_array(tree, X, $operators; turbo=$turbo) for tree in trees],
                 setup=(
                     X=randn(MersenneTwister(0), $T, 5, $n);
-                    tree=convert(Node{$T}, copy_node($simple_tree))
+                    treesize=20;
+                    ntrees=100;
+                    trees=[gen_random_tree_fixed_size(treesize, $operators, 5, $T) for _ in 1:ntrees]
                 )
             )
             if T <: Real
+                eval_grad_tree_array(
+                    gen_random_tree_fixed_size(20, operators, 5, T),
+                    randn(MersenneTwister(0), T, 5, n),
+                    operators;
+                    variable=true,
+                    turbo=turbo
+                )
                 suite[T]["derivative$(extra_key)"] = @benchmarkable(
-                    eval_grad_tree_array(tree, X, $operators; variable=true, turbo=$turbo),
-                    evals=evals,
-                    samples=samples,
-                    seconds=5.0,
+                    [eval_grad_tree_array(tree, X, $operators; variable=true, turbo=$turbo) for tree in trees],
                     setup=(
                         X=randn(MersenneTwister(0), $T, 5, $n);
-                        tree=convert(Node{$T}, copy_node($simple_tree))
+                        treesize=20;
+                        ntrees=100;
+                        trees=[gen_random_tree_fixed_size(treesize, $operators, 5, $T) for _ in 1:ntrees]
                     )
                 )
             end