Reasoning and benchmark for the default to max_sample_size 1M.

Author: PragTob

lib/benchee/configuration.ex

@@ -40,6 +40,14 @@ defmodule Benchee.Configuration do
             measure_function_call_overhead: false,
             title: nil,
             profile_after: false,
+            # Why a million? Well, it's a nice number :P
+            # Seriously, though: That's usually enough samples to be significant enough as a
+            # result.
+            # At the same time, at least on my machine it takes less than one second for
+            # `Statistex` to calculate the statistics for a list that large.
+            # It also generates less than 1GB in data (some of which is garbage collected/
+            # not necessarily all in RAM at the same time) - which seems reasonable enough.
+            # see `samples/statistics_performance.exs` and also maybe run it yourself.
             max_sample_size: 1_000_000
 
   @typedoc """

samples/sort_performance.exs

@@ -1,30 +1,55 @@
 list_10k = 1..10_000 |> Enum.to_list() |> Enum.shuffle()
 list_100k = 1..100_000 |> Enum.to_list() |> Enum.shuffle()
+list_1M = 1..1_000_000 |> Enum.to_list() |> Enum.shuffle()
 
-Benchee.run(%{"10k" => fn -> Enum.sort(list_10k) end, "100k" => fn -> Enum.sort(list_100k) end})
+Benchee.run(
+  [
+    {"10k", fn -> Enum.sort(list_10k) end},
+    {"100k", fn -> Enum.sort(list_100k) end},
+    {"1M", fn -> Enum.sort(list_1M) end}
+  ],
+  memory_time: 2
+)
 
+# tobi@qiqi:~/github/benchee(max-sample-size)$ mix run samples/sort_performance.exs
 # Operating System: Linux
-# CPU Information: Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz
-# Number of Available Cores: 8
-# Available memory: 15.61 GB
-# Elixir 1.8.1
-# Erlang 21.3.2
+# CPU Information: AMD Ryzen 9 5900X 12-Core Processor
+# Number of Available Cores: 24
+# Available memory: 31.26 GB
+# Elixir 1.18.3
+# Erlang 27.3.2
+# JIT enabled: true
 
 # Benchmark suite executing with the following configuration:
 # warmup: 2 s
 # time: 5 s
-# memory time: 0 ns
+# memory time: 2 s
+# reduction time: 0 ns
 # parallel: 1
 # inputs: none specified
-# Estimated total run time: 14 s
+# Estimated total run time: 27 s
 
-# Benchmarking 100k...
-# Benchmarking 10k...
+# Benchmarking 10k ...
+# Benchmarking 100k ...
+# Benchmarking 1M ...
+# Calculating statistics...
+# Formatting results...
 
 # Name           ips        average  deviation         median         99th %
-# 10k         722.79        1.38 ms    ±10.21%        1.37 ms        1.77 ms
-# 100k         56.70       17.64 ms     ±6.38%       17.34 ms       22.84 ms
+# 10k        1237.97        0.81 ms    ±28.70%        0.73 ms        1.42 ms
+# 100k         74.50       13.42 ms    ±40.13%       10.96 ms       31.42 ms
+# 1M            6.17      162.05 ms    ±34.38%      146.08 ms      328.83 ms
 
 # Comparison:
-# 10k         722.79
-# 100k         56.70 - 12.75x slower +16.25 ms
+# 10k        1237.97
+# 100k         74.50 - 16.62x slower +12.62 ms
+# 1M            6.17 - 200.61x slower +161.24 ms
+
+# Memory usage statistics:
+
+# Name    Memory usage
+# 10k          1.42 MB
+# 100k        18.59 MB - 13.09x memory usage +17.17 MB
+# 1M         218.22 MB - 153.62x memory usage +216.80 MB
+
+# **All measurements for memory usage were the same**

samples/statistics_performance.exs

@@ -0,0 +1,55 @@
+list_10k = 1..10_000 |> Enum.to_list() |> Enum.shuffle()
+list_100k = 1..100_000 |> Enum.to_list() |> Enum.shuffle()
+list_1M = 1..1_000_000 |> Enum.to_list() |> Enum.shuffle()
+
+Benchee.run(
+  [
+    {"10k", fn -> Statistex.statistics(list_10k) end},
+    {"100k", fn -> Statistex.statistics(list_100k) end},
+    {"1M", fn -> Statistex.statistics(list_1M) end}
+  ],
+  memory_time: 2
+)
+
+# tobi@qiqi:~/github/benchee(max-sample-size)$ mix run samples/sort_performance.exs
+# Operating System: Linux
+# CPU Information: AMD Ryzen 9 5900X 12-Core Processor
+# Number of Available Cores: 24
+# Available memory: 31.26 GB
+# Elixir 1.18.3
+# Erlang 27.3.2
+# JIT enabled: true
+
+# Benchmark suite executing with the following configuration:
+# warmup: 2 s
+# time: 5 s
+# memory time: 2 s
+# reduction time: 0 ns
+# parallel: 1
+# inputs: none specified
+# Estimated total run time: 27 s
+
+# Benchmarking 10k ...
+# Benchmarking 100k ...
+# Benchmarking 1M ...
+# Calculating statistics...
+# Formatting results...
+
+# Name           ips        average  deviation         median         99th %
+# 10k        1237.97        0.81 ms    ±28.70%        0.73 ms        1.42 ms
+# 100k         74.50       13.42 ms    ±40.13%       10.96 ms       31.42 ms
+# 1M            6.17      162.05 ms    ±34.38%      146.08 ms      328.83 ms
+
+# Comparison:
+# 10k        1237.97
+# 100k         74.50 - 16.62x slower +12.62 ms
+# 1M            6.17 - 200.61x slower +161.24 ms
+
+# Memory usage statistics:
+
+# Name    Memory usage
+# 10k          1.42 MB
+# 100k        18.59 MB - 13.09x memory usage +17.17 MB
+# 1M         218.22 MB - 153.62x memory usage +216.80 MB
+
+# **All measurements for memory usage were the same**