WIP: (known failure) try to remove repeatedly identifying outliers

From: #5 (comment)

Need to think through it again/and or check some more samples
and test it against that. Getting different bounds/outliers
right now although I think they're right.

Author: PragTob

lib/statistex.ex

@@ -96,18 +96,16 @@ defmodule Statistex do
 
   The statistics themselves are described in the individual samples that can be used to calculate individual values.
 
-  `Argumenterror` is raised if the given list is empty.
+  `ArgumentError` is raised if the given list is empty.
 
   ## Options
 
-  In a `percentiles` options arguments for the calculation of percentiles (see `percentiles/2`) can
+  With a `percentiles` options arguments for the calculation of percentiles (see `percentiles/2`) can
   be given. The percentiles 25th, 50th (median) and 75th are always calculated.
 
-  The option `exclude_outliers` can be set to `:once`, `:repeatedly` or `nil`,
-  `nil` is the default. If this option set to `:once` the outliers are excluded
-  and the statistics are calculated with the rest of the samples. The value
-  `:repeatedly` repeats the outlier exclusion until the samples no longer
-  contains outliers.
+  The option `exclude_outliers` can be set to `true`, `false`. Defaults to `false`.
+  If this option is set to `true` the outliers are excluded
+  and the statistics are calculated with the rest of the samples.
 
   ## Examples
 
@@ -167,62 +165,72 @@ defmodule Statistex do
   def statistics(samples, configuration) do
     samples = Enum.sort(samples)
 
-    minimum = hd(samples)
-    maximum = List.last(samples)
-
-    percentiles = calculate_percentiles(samples, configuration)
+    # these statistics are required to do the outlier calculations
+    %{minimum: minimum, maximum: maximum, percentiles: percentiles} =
+      base_statistics(samples, configuration)
 
     outlier_bounds =
       do_outlier_bounds(samples, percentiles: percentiles, minimum: minimum, maximum: maximum)
 
+    # make sure rest remains sorted and so can be used again to ok results
     {outliers, rest} = do_outliers(samples, outlier_bounds: outlier_bounds)
 
     if exclude_outliers?(configuration) and Enum.any?(outliers) do
-      configuration =
-        configuration
-        |> Keyword.update!(:exclude_outliers, fn
-          :once -> :stop
-          :repeatedly -> :repeatedly
-        end)
-        |> Keyword.update(:acc_outliers, outliers, fn list -> list ++ outliers end)
-
-      statistics(rest, configuration)
+      # figure out to avoid double sorting
+      rest = Enum.sort(rest)
+      # need to recalculate with the outliers removed
+      %{minimum: minimum, maximum: maximum, percentiles: percentiles} =
+        base_statistics(rest, configuration)
+
+      create_full_statistics(rest, minimum, maximum, percentiles, outliers, outlier_bounds)
     else
-      outliers = outliers ++ Keyword.get(configuration, :acc_outliers, [])
-
-      total = total(samples)
-      sample_size = length(samples)
-      average = average(samples, total: total, sample_size: sample_size)
-      variance = variance(samples, average: average, sample_size: sample_size)
-
-      frequency_distribution = frequency_distribution(samples)
-
-      standard_deviation = standard_deviation(samples, variance: variance)
-
-      standard_deviation_ratio =
-        standard_deviation_ratio(samples, standard_deviation: standard_deviation)
-
-      %__MODULE__{
-        total: total,
-        average: average,
-        variance: variance,
-        standard_deviation: standard_deviation,
-        standard_deviation_ratio: standard_deviation_ratio,
-        median: median(samples, percentiles: percentiles),
-        percentiles: percentiles,
-        frequency_distribution: frequency_distribution,
-        mode: mode(samples, frequency_distribution: frequency_distribution),
-        minimum: minimum,
-        maximum: maximum,
-        outlier_bounds: outlier_bounds,
-        outliers: outliers,
-        sample_size: sample_size
-      }
+      create_full_statistics(samples, minimum, maximum, percentiles, outliers, outlier_bounds)
     end
   end
 
+  defp base_statistics(samples, configuration) do
+    minimum = hd(samples)
+    maximum = List.last(samples)
+
+    percentiles = calculate_percentiles(samples, configuration)
+
+    %{minimum: minimum, maximum: maximum, percentiles: percentiles}
+  end
+
   defp exclude_outliers?(configuration) do
-    Keyword.get(configuration, :exclude_outliers) in [:once, :repeatedly]
+    Access.get(configuration, :exclude_outliers) == true
+  end
+
+  # maybe make argument a map
+  defp create_full_statistics(samples, minimum, maximum, percentiles, outliers, outlier_bounds) do
+    total = total(samples)
+    sample_size = length(samples)
+    average = average(samples, total: total, sample_size: sample_size)
+    variance = variance(samples, average: average, sample_size: sample_size)
+
+    frequency_distribution = frequency_distribution(samples)
+
+    standard_deviation = standard_deviation(samples, variance: variance)
+
+    standard_deviation_ratio =
+      standard_deviation_ratio(samples, standard_deviation: standard_deviation)
+
+    %__MODULE__{
+      total: total,
+      average: average,
+      variance: variance,
+      standard_deviation: standard_deviation,
+      standard_deviation_ratio: standard_deviation_ratio,
+      median: median(samples, percentiles: percentiles),
+      percentiles: percentiles,
+      frequency_distribution: frequency_distribution,
+      mode: mode(samples, frequency_distribution: frequency_distribution),
+      minimum: minimum,
+      maximum: maximum,
+      outlier_bounds: outlier_bounds,
+      outliers: outliers,
+      sample_size: sample_size
+    }
   end
 
   @doc """

test/statistex_test.exs

@@ -67,7 +67,7 @@ defmodule Statistex.StatistexTest do
 
     test "returns Statistex struct with excluded outliers once" do
       assert Statistex.statistics([50, 50, 450, 450, 450, 500, 500, 500, 600, 900],
-               exclude_outliers: :once
+               exclude_outliers: true
              ) ==
                %Statistex{
                  total: 3450,
@@ -81,33 +81,13 @@ defmodule Statistex.StatistexTest do
                  mode: [500, 450],
                  minimum: 450,
                  maximum: 600,
+                 # check with other sources what is right and what isn't, I fear we may have calculated outliers twice before
                  outlier_bounds: {450, 575.0},
+                 # Either sort them or make the test ignorant of order
                  outliers: [600, 50, 50, 900],
                  sample_size: 7
                }
     end
-
-    test "returns Statistex struct with excluded outliers repeatedly" do
-      assert Statistex.statistics([50, 50, 450, 450, 450, 500, 500, 500, 600, 900],
-               exclude_outliers: :repeatedly
-             ) ==
-               %Statistex{
-                 total: 2850,
-                 average: 475.0,
-                 variance: 750.0,
-                 standard_deviation: 27.386127875258307,
-                 standard_deviation_ratio: 0.05765500605317538,
-                 median: 475.0,
-                 percentiles: %{25 => 450.0, 50 => 475.0, 75 => 500.0},
-                 frequency_distribution: %{450 => 3, 500 => 3},
-                 mode: [500, 450],
-                 minimum: 450,
-                 maximum: 500,
-                 outlier_bounds: {450, 500},
-                 outliers: [50, 50, 900, 600],
-                 sample_size: 6
-               }
-    end
   end
 
   describe "property testing as we might get loads of data" do