Fix tdigest

Author: LiaCastaneda

datafusion/core/tests/dataframe/dataframe_functions.rs

@@ -510,7 +510,7 @@ async fn test_fn_approx_percentile_cont() -> Result<()> {
     +-------------------------------------------------------------------------------------+
     | approx_percentile_cont(Float64(0.1),Int32(2)) WITHIN GROUP [test.b DESC NULLS LAST] |
     +-------------------------------------------------------------------------------------+
-    | 69                                                                                  |
+    | 100                                                                                 |
     +-------------------------------------------------------------------------------------+
     ");
 

datafusion/functions-aggregate-common/src/tdigest.rs

@@ -443,6 +443,13 @@ impl TDigest {
                 return self.max();
             }
 
+            // If rank reaches the last unit of weight, return max directly.
+            // Without this, interpolation at the last centroid boundary can
+            // produce p90 > p99 on sparse data (e.g. 10 values).
+            if rank >= self.count - 1.0 {
+                return self.max();
+            }
+
             pos = 0;
             t = self.count;
 
@@ -735,6 +742,23 @@ mod tests {
         assert_state_roundtrip!(t);
     }
 
+    // On sparse data, higher quantiles must not return lower values than lower quantiles.
+    #[test]
+    fn test_sparse_dataset_quantile_ordering() {
+        let values = vec![1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0, 50.0, 100.0, 1000.0];
+        let t = TDigest::new(100);
+        let t = t.merge_unsorted_f64(values);
+
+        let p50 = t.estimate_quantile(0.5);
+        let p90 = t.estimate_quantile(0.9);
+        let p99 = t.estimate_quantile(0.99);
+
+        assert!(p50 <= p90, "p50 ({p50}) should be <= p90 ({p90})");
+        assert!(p90 <= p99, "p90 ({p90}) should be <= p99 ({p99})");
+        assert_eq!(p90, 1000.0, "p90 should be max on boundary rank");
+        assert_eq!(p99, 1000.0, "p99 should be max on boundary rank");
+    }
+
     #[test]
     fn test_size() {
         let t = TDigest::new(10);