Basic max_sample_size tests

Author: PragTob

test/benchee/benchmark/runner_test.exs

@@ -8,9 +8,12 @@ defmodule Benchee.Benchmark.RunnerTest do
   alias Benchee.Test.FakeBenchmarkPrinter
   alias Benchee.Test.FakeProgressPrinter
 
+  # times are in ns, so this is 40ms
+  @small_time 40_000_000
+
   @config %Configuration{
     parallel: 1,
-    time: 40_000_000,
+    time: @small_time,
     warmup: 20_000_000,
     inputs: nil,
     pre_check: false,
@@ -475,6 +478,28 @@ defmodule Benchee.Benchmark.RunnerTest do
         |> Benchmark.collect(FakeBenchmarkPrinter)
       end
     end
+
+    test "max_sample_size is abided by across all times" do
+      max_sample_size = 3
+
+      %Suite{scenarios: [scenario]} =
+        %Suite{
+          configuration: %Configuration{
+            max_sample_size: max_sample_size,
+            # each one of those should easily be able to gather the samples in time
+            time: @small_time,
+            memory_time: @small_time,
+            reduction_time: @small_time
+          }
+        }
+        |> Benchmark.benchmark("some benchmak", fn -> 2 end)
+        |> Benchmark.collect(FakeBenchmarkPrinter)
+
+      # if it starts going flakey might need to do <= which is in the spirit of max
+      assert length(scenario.run_time_data.samples) == max_sample_size
+      assert length(scenario.memory_usage_data.samples) == max_sample_size
+      assert length(scenario.reductions_data.samples) == max_sample_size
+    end
   end
 
   # hooks are pretty much their own beast

test/benchee_test.exs

@@ -214,22 +214,22 @@ defmodule BencheeTest do
     refute output =~ ~r/fast/
   end
 
+  @disable_all_output_options [
+    print: [
+      fast_warning: false,
+      benchmarking: false,
+      configuration: false
+    ],
+    formatters: []
+  ]
   test "integration disabling all output configs and formatters we're left with an empty output" do
     output =
       capture_io(fn ->
         Benchee.run(
           %{"Blitz" => fn -> 0 end},
-          Keyword.merge(
-            @test_configuration,
-            time: 0.001,
-            warmup: 0,
-            print: [
-              fast_warning: false,
-              benchmarking: false,
-              configuration: false
-            ],
-            formatters: []
-          )
+          @test_configuration
+          |> Keyword.merge(time: 0.001, warmup: 0)
+          |> Keyword.merge(@disable_all_output_options)
         )
       end)
 
@@ -492,6 +492,31 @@ defmodule BencheeTest do
     assert length(occurrences) == 2
   end
 
+  test "max_sample_size & multiple inputs with very fast functions" do
+    max_sample_size = 3
+
+    configuration =
+      @test_configuration
+      |> Keyword.merge(@disable_all_output_options)
+      |> Keyword.merge(
+        max_sample_size: max_sample_size,
+        inputs: %{"number_one" => 1, :symbole_one => :one}
+      )
+
+    suite =
+      Benchee.run(
+        %{
+          "identity" => fn i -> i end
+        },
+        configuration
+      )
+
+    Enum.each(suite.scenarios, fn scenario ->
+      # if this becomes flakey, change to <=
+      assert scenario.run_time_data.statistics.sample_size == max_sample_size
+    end)
+  end
+
   test ".run returns the suite intact" do
     capture_io(fn ->
       suite =