Actually map over new statistics and explain why we do the mapping

Author: PragTob

lib/benchee/statistics.ex

@@ -25,6 +25,9 @@ defmodule Benchee.Statistics do
     :relative_more,
     :relative_less,
     :absolute_difference,
+    :outliers,
+    :lower_outlier_bound,
+    :upper_outlier_bound,
     sample_size: 0
   ]
 
@@ -85,6 +88,9 @@ defmodule Benchee.Statistics do
           relative_more: float | nil | :infinity,
           relative_less: float | nil | :infinity,
           absolute_difference: float | nil,
+          outliers: [number],
+          lower_outlier_bound: number,
+          upper_outlier_bound: number,
           sample_size: integer
         }
 
@@ -115,7 +121,7 @@ defmodule Benchee.Statistics do
       ...>     input: "Input"
       ...>   }
       ...> ]
-      ...>
+      ...> 
       ...> suite = %Benchee.Suite{scenarios: scenarios}
       ...> statistics(suite, Benchee.Test.FakeProgressPrinter)
       %Benchee.Suite{
@@ -137,7 +143,10 @@ defmodule Benchee.Statistics do
                 mode: [500, 400],
                 minimum: 200,
                 maximum: 900,
-                sample_size: 9
+                sample_size: 9,
+                outliers: [],
+                lower_outlier_bound: 100.0,
+                upper_outlier_bound: 900.0
               }
             },
             memory_usage_data: %Benchee.CollectionData{
@@ -153,7 +162,10 @@ defmodule Benchee.Statistics do
                 mode: [500, 400],
                 minimum: 200,
                 maximum: 900,
-                sample_size: 9
+                sample_size: 9,
+                outliers: [],
+                lower_outlier_bound: 100.0,
+                upper_outlier_bound: 900.0
               }
             }
           }
@@ -243,6 +255,14 @@ defmodule Benchee.Statistics do
     |> convert_from_statistex
   end
 
+  # It might seem silly to maintain and map statistex to our own struct,
+  # but this gives benchee more control  and makes it safer to upgrade and change.
+  # Also, we don't expose changes in statistex versions automatically to plugins.
+  #
+  # As an example right now it's being discussed in statistex to add an `m2` statistic that holds
+  # no value for benchee (as it's ony used to calculate variance).
+  #
+  # We also manually add `ips` related stats (see `add_ips/1`) so differences are sufficient.
   defp convert_from_statistex(statistex_statistics) do
     %__MODULE__{
       average: statistex_statistics.average,
@@ -253,7 +273,10 @@ defmodule Benchee.Statistics do
       mode: statistex_statistics.mode,
       minimum: statistex_statistics.minimum,
       maximum: statistex_statistics.maximum,
-      sample_size: statistex_statistics.sample_size
+      sample_size: statistex_statistics.sample_size,
+      outliers: statistex_statistics.outliers,
+      lower_outlier_bound: statistex_statistics.lower_outlier_bound,
+      upper_outlier_bound: statistex_statistics.upper_outlier_bound
     }
   end
 

test/fixtures/escript/mix.lock

@@ -1,4 +1,4 @@
 %{
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
-  "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
+  "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"},
 }