move+hide+improve confidence interval computation

Author: jan-elastic

x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ConfidenceIntervalDoubleEvaluator.java renamed to x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/approximate/ConfidenceIntervalDoubleEvaluator.java

@@ -31,8 +31,8 @@
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.expression.function.scalar.approximate.ConfidenceInterval;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong;
-import org.elasticsearch.xpack.esql.expression.function.scalar.math.ConfidenceInterval;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvContains;
 import org.elasticsearch.xpack.esql.expression.function.scalar.random.Random;

x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ConfidenceIntervalIntEvaluator.java renamed to x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/approximate/ConfidenceIntervalIntEvaluator.java

@@ -118,7 +118,6 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Atan2;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cbrt;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Ceil;
-import org.elasticsearch.xpack.esql.expression.function.scalar.math.ConfidenceInterval;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.CopySign;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cos;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cosh;
@@ -371,7 +370,6 @@ private static FunctionDefinition[][] functions() {
                 def(Atan2.class, Atan2::new, "atan2"),
                 def(Cbrt.class, Cbrt::new, "cbrt"),
                 def(Ceil.class, Ceil::new, "ceil"),
-                def(ConfidenceInterval.class, ConfidenceInterval::new, "confidence_interval"),
                 def(Cos.class, Cos::new, "cos"),
                 def(Cosh.class, Cosh::new, "cosh"),
                 def(E.class, E::new, "e"),

x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ConfidenceIntervalLongEvaluator.java renamed to x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/approximate/ConfidenceIntervalLongEvaluator.java

@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingWritables;
+import org.elasticsearch.xpack.esql.expression.function.scalar.approximate.ConfidenceInterval;
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.ClampMax;
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.ClampMin;
@@ -26,7 +27,6 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Atan2;
-import org.elasticsearch.xpack.esql.expression.function.scalar.math.ConfidenceInterval;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.CopySign;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.E;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Hypot;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/approximate/Approximate.java

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+package org.elasticsearch.xpack.esql.expression.function.scalar.approximate;
 
 import org.apache.commons.math3.distribution.NormalDistribution;
 import org.apache.commons.math3.stat.descriptive.moment.Mean;
@@ -49,6 +49,8 @@ public class ConfidenceInterval extends EsqlScalarFunction {
         ConfidenceInterval::new
     );
 
+    private static final NormalDistribution normal = new NormalDistribution();
+
     private final Expression bestEstimate;
     private final Expression estimates;
 
@@ -174,7 +176,7 @@ static void process(IntBlock.Builder builder, @Position int position, IntBlock b
         Number[] confidenceInterval = computeConfidenceInterval(bestEstimate, estimates);
         builder.beginPositionEntry();
         for (Number v : confidenceInterval) {
-            builder.appendInt(v.intValue());
+            builder.appendInt((int) Math.round(v.doubleValue()));
         }
         builder.endPositionEntry();
     }
@@ -195,12 +197,12 @@ static void process(LongBlock.Builder builder, @Position int position, LongBlock
         Number[] confidenceInterval = computeConfidenceInterval(bestEstimate, estimates);
         builder.beginPositionEntry();
         for (Number v : confidenceInterval) {
-            builder.appendLong(v.longValue());
+            builder.appendLong(Math.round(v.doubleValue()));
         }
         builder.endPositionEntry();
     }
 
-    private static Number[] computeConfidenceInterval(Number bestEstimate, Number[] estimates) {
+    public static Number[] computeConfidenceInterval(Number bestEstimate, Number[] estimates) {
         Mean estimatesMean = new Mean();
         StandardDeviation estimatesStdDev = new StandardDeviation(false);
         Skewness estimatesSkew = new Skewness();
@@ -209,26 +211,18 @@ private static Number[] computeConfidenceInterval(Number bestEstimate, Number[]
             estimatesStdDev.increment(estimate.doubleValue());
             estimatesSkew.increment(estimate.doubleValue());
         }
-
-        double mm = estimatesMean.getResult();
         double sm = estimatesStdDev.getResult();
-
         if (sm == 0.0) {
-            return new Number[] { bestEstimate, bestEstimate, bestEstimate };
+            return new Number[] { bestEstimate, bestEstimate };
         }
-
-        double a = estimatesSkew.getResult() / 6;
-
-        NormalDistribution norm = new NormalDistribution(0, 1);
-
+        double mm = estimatesMean.getResult();
         double z0 = (bestEstimate.doubleValue() - mm) / sm;
-        double dz = norm.inverseCumulativeProbability((1 + 0.95) / 2);  // for 95% confidence interval
-        double zl = z0 - dz;
-        double zu = z0 + dz;
-
-        sm /= Math.sqrt(estimatesMean.getN());
-
-        return new Number[] { mm + sm * (z0 + zl / (1 - Math.min(0.8, a * zl))), mm + sm * (z0 + zu / (1 - Math.min(0.8, a * zu))), };
+        double dz = normal.inverseCumulativeProbability((1 + 0.95) / 2);  // for 95% confidence interval; TODO make configurable
+        double a = estimatesSkew.getResult() / (6 * Math.sqrt(estimates.length));
+        double zl = z0 + (z0 - dz) / (1 - Math.min(a * (z0 - dz), 0.9));
+        double zu = z0 + (z0 + dz) / (1 - Math.min(a * (z0 + dz), 0.9));
+        double scale = Math.max(1 / Math.sqrt(estimates.length), z0 < 0 ? z0 / zl : z0 / zu);
+        return new Number[] { mm + scale * sm * zl, mm + sm * scale * zu };
     }
 
     @Override