WIP

Author: nalimilan

src/extras.jl

@@ -233,20 +233,28 @@ Provide the default label format for the `cut(x, ngroups)` method.
 quantile_formatter(from, to, i; leftclosed, rightclosed) =
     string("Q", i, ": ", leftclosed ? "[" : "(", from, ", ", to, rightclosed ? "]" : ")")
 
-function _quantile!(v::AbstractVector, ps::AbstractVector)
-    n = length(v)
-    n > 0 || throw(ArgumentError("cannot compute quantiles of empty data vector"))
-    return map(ps) do p
-        i = clamp(ceil(Int, n*p), 1, n) + firstindex(v) - 1
-        q = v[i]
-        # Take next distinct value even if quantile falls in a series of duplicated values
-        @inbounds for j in (i+1):lastindex(v)
-            q_prev = q
-            q = v[j]
-            q_prev != q && break
+"""Find first value in data which is greater than each quantile in ``qs``."""
+function find_breaks(v::AbstractVector, qs::AbstractVector)
+    n = length(qs)
+    breaks = similar(v, n)
+    n == 0 && return breaks
+
+    i = 1
+    q = qs[1]
+    @inbounds for x in v
+        if x > q
+            breaks[i] = x
+            i += 1
+            i > n && break
+            q = qs[i]
         end
-        return q
     end
+    # if last values in x are equal to q, breaks were not initialized
+    for i in i:n
+        breaks[i] = q
+    end
+    @show breaks
+    return breaks
 end
 
 """
@@ -279,14 +287,16 @@ function cut(x::AbstractArray, ngroups::Integer;
              labels::Union{AbstractVector{<:SupportedTypes},Function}=quantile_formatter,
              allowempty::Bool=false)
     ngroups >= 1 || throw(ArgumentError("ngroups must be strictly positive (got $ngroups)"))
-    xnm = eltype(x) >: Missing ? sort!(collect(skipmissing(x))) : sort(x)
-    min_x, max_x = first(xnm), last(xnm)
+    sorted_x = eltype(x) >: Missing ? sort!(collect(skipmissing(x))) : sort(x)
+    min_x, max_x = first(sorted_x), last(sorted_x)
     if (min_x isa Number && isnan(min_x)) ||
         (max_x isa Number && isnan(max_x))
         throw(ArgumentError("NaN values are not allowed in input vector"))
     end
-    qs = _quantile!(xnm, (1:(ngroups-1))/ngroups)
-    breaks = [min_x; qs; max_x]
+    qs = quantile!(sorted_x, (1:(ngroups-1))/ngroups, sorted=true)
+    @show qs, min_x, max_x
+    breaks = [min_x; find_breaks(sorted_x, qs); max_x]
+    @show breaks
     if !allowempty && !allunique(@view breaks[1:end-1])
         throw(ArgumentError("cannot compute $ngroups quantiles due to " *
                             "too many duplicated values in `x`. " *

test/15_extras.jl

@@ -361,4 +361,51 @@ end
     @test levels(x) == ["Q1: [-Inf, 3.0)", "Q2: [3.0, 5.0]"]
 end
 
+@testset "cut in corner cases" begin
+    # In this case, cut(x, quantile(x, (0:36)/36)) in R generates
+    # an empty "(143,172]" level
+    # and qcut(x, 36) in Polars misses that level.
+    # Our approach uses different breaks at 143 and 182
+    x = [23, 23, 60, 76, 84, 95, 101, 108, 111, 133, 137, 143, 143, 143, 182,
+         206, 214, 241, 258, 262, 280, 289, 303, 312, 321, 323, 352, 353, 354,
+         368, 369, 373, 374, 384, 385, 386, 387, 392, 405, 406, 410, 421, 430,
+         430, 431, 442, 464, 474, 478, 479, 496, 516, 530, 534, 549, 554, 568,
+         575, 589, 590, 591, 592, 595, 596, 603, 625, 632, 632, 638, 640, 640,
+         645, 648, 690, 704, 748, 758, 771, 772, 803, 835, 839, 853, 869, 873,
+         874, 887, 911, 920, 923, 928, 933, 943, 945, 945, 947, 951, 965, 978, 980]
+
+    @test cut(x, 36) ==
+        ["Q17: [442, 474)", "Q8: [280, 312)", "Q2: [76, 101)", "Q9: [312, 323)",
+         "Q14: [387, 406)", "Q30: [835, 869)", "Q17: [442, 474)", "Q35: [947, 965)",
+         "Q2: [76, 101)", "Q11: [354, 373)", "Q32: [887, 923)", "Q12: [373, 385)",
+         "Q24: [603, 638)", "Q29: [772, 835)", "Q24: [603, 638)", "Q15: [406, 430)",
+         "Q11: [354, 373)", "Q23: [592, 603)", "Q3: [101, 133)", "Q16: [430, 442)",
+         "Q34: [933, 947)", "Q27: [648, 748)", "Q28: [748, 772)", "Q28: [748, 772)",
+         "Q21: [568, 589)", "Q18: [474, 496)", "Q32: [887, 923)", "Q11: [354, 373)",
+         "Q7: [241, 280)", "Q3: [101, 133)", "Q19: [496, 534)", "Q13: [385, 387)",
+         "Q36: [965, 980]", "Q33: [923, 933)", "Q16: [430, 442)", "Q36: [965, 980]",
+         "Q27: [648, 748)", "Q24: [603, 638)", "Q32: [887, 923)", "Q4: [133, 182)",
+         "Q22: [589, 592)", "Q1: [23, 76)", "Q5: [182, 206)", "Q28: [748, 772)",
+         "Q30: [835, 869)", "Q31: [869, 887)", "Q22: [589, 592)", "Q15: [406, 430)",
+         "Q31: [869, 887)", "Q19: [496, 534)", "Q26: [640, 648)", "Q8: [280, 312)",
+         "Q18: [474, 496)", "Q14: [387, 406)", "Q7: [241, 280)", "Q30: [835, 869)",
+         "Q3: [101, 133)", "Q9: [312, 323)", "Q4: [133, 182)", "Q24: [603, 638)",
+         "Q20: [534, 568)", "Q25: [638, 640)", "Q20: [534, 568)", "Q23: [592, 603)",
+         "Q12: [373, 385)", "Q27: [648, 748)", "Q29: [772, 835)", "Q6: [206, 241)",
+         "Q34: [933, 947)", "Q16: [430, 442)", "Q26: [640, 648)", "Q15: [406, 430)",
+         "Q12: [373, 385)", "Q26: [640, 648)", "Q19: [496, 534)", "Q4: [133, 182)",
+         "Q34: [933, 947)", "Q31: [869, 887)", "Q10: [323, 354)", "Q21: [568, 589)",
+         "Q4: [133, 182)", "Q7: [241, 280)", "Q35: [947, 965)", "Q14: [387, 406)",
+         "Q18: [474, 496)", "Q34: [933, 947)", "Q20: [534, 568)", "Q22: [589, 592)",
+         "Q33: [923, 933)", "Q10: [323, 354)", "Q13: [385, 387)", "Q1: [23, 76)",
+         "Q36: [965, 980]", "Q2: [76, 101)", "Q23: [592, 603)", "Q6: [206, 241)",
+         "Q1: [23, 76)", "Q10: [323, 354)", "Q4: [133, 182)", "Q8: [280, 312)"]
+
+    @test cut([0, 1, 1, 1, 1], 2) ==
+        ["Q1: [0, 1)", "Q2: [1, 1]", "Q2: [1, 1]", "Q2: [1, 1]", "Q2: [1, 1]"]
+
+    @test cut([1, 1, 1, 1, 2], 2) ==
+        ["Q1: [1, 2)", "Q1: [1, 2)", "Q1: [1, 2)", "Q1: [1, 2)", "Q2: [2, 2]"]
+end
+
 end