Merge remote-tracking branch 'origin/main' into otlp-tsdb-empty-endpoint

Author: felixbarny

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

docs/changelog/133397.yaml

@@ -0,0 +1,5 @@
+pr: 133397
+summary: Push down loading of singleton dense double based field types to the …
+area: "Codec"
+type: enhancement
+issues: []

docs/changelog/133601.yaml

@@ -0,0 +1,8 @@
+pr: 133601
+summary: Fix bug in topn
+area: ES|QL
+type: bug
+issues:
+ - 133600
+ - 133574
+ - 133607

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java

@@ -771,6 +771,8 @@ public interface EntitlementChecker {
 
     void check$java_io_File$createNewFile(Class<?> callerClass, File file);
 
+    void check$java_io_File$$createTempFile(Class<?> callerClass, String prefix, String suffix);
+
     void check$java_io_File$$createTempFile(Class<?> callerClass, String prefix, String suffix, File directory);
 
     void check$java_io_File$delete(Class<?> callerClass, File file);

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java

@@ -97,6 +97,17 @@ static void fileCreateTempFile() throws IOException {
         File.createTempFile("prefix", "suffix", readWriteDir().toFile());
     }
 
+    @EntitlementTest(expectedAccess = ALWAYS_ALLOWED)
+    static void fileCreateTempFileSystemTempDirectory() throws IOException {
+        File.createTempFile("prefix", "suffix");
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_ALLOWED)
+    static void fileCreateTempFileNullDirectory() throws IOException {
+        // null directory = system temp directory
+        File.createTempFile("prefix", "suffix", null);
+    }
+
     @EntitlementTest(expectedAccess = PLUGINS)
     static void fileDelete() throws IOException {
         var toDelete = EntitledActions.createTempFileForWrite();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ElasticsearchEntitlementChecker.java

@@ -1489,9 +1489,20 @@ public void checkSelectorProviderOpenSocketChannel(Class<?> callerClass, Selecto
         policyChecker.checkFileWrite(callerClass, file);
     }
 
+    @Override
+    public void check$java_io_File$$createTempFile(Class<?> callerClass, String prefix, String suffix) {
+        policyChecker.checkCreateTempFile(callerClass);
+    }
+
     @Override
     public void check$java_io_File$$createTempFile(Class<?> callerClass, String prefix, String suffix, File directory) {
-        policyChecker.checkFileWrite(callerClass, directory);
+        // A null value for the directory parameter means using the temp directory (java.io.tmpdir,
+        // aka org.elasticsearch.env.Environment#tmpDir, aka PathLookup#TEMP).
+        if (directory == null) {
+            policyChecker.checkCreateTempFile(callerClass);
+        } else {
+            policyChecker.checkFileWrite(callerClass, directory);
+        }
     }
 
     @Override

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java

@@ -77,6 +77,11 @@ public Buckets negativeBuckets() {
         return EmptyBuckets.INSTANCE;
     }
 
+    @Override
+    public double sum() {
+        return 0;
+    }
+
     @Override
     public long ramBytesUsed() {
         return 0;

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java

@@ -47,7 +47,7 @@
  */
 public interface ExponentialHistogram extends Accountable {
 
-    // TODO(b/128622): support min/max/sum/count storage and merging.
+    // TODO(b/128622): support min/max storage and merging.
     // TODO(b/128622): Add special positive and negative infinity buckets
     // to allow representation of explicit bucket histograms with open boundaries.
 
@@ -93,6 +93,15 @@ public interface ExponentialHistogram extends Accountable {
      */
     Buckets negativeBuckets();
 
+    /**
+     * Returns the sum of all values represented by this histogram.
+     * Note that even if histograms are cumulative, the sum is not guaranteed to be monotonically increasing,
+     * because histograms support negative values.
+     *
+     * @return the sum, guaranteed to be zero for empty histograms
+     */
+    double sum();
+
     /**
      * Represents a bucket range of an {@link ExponentialHistogram}, either the positive or the negative range.
      */

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java

@@ -123,6 +123,7 @@ private void mergeValuesToHistogram() {
         }
 
         valueBuffer.reset();
+        valueBuffer.setSum(rawValuesSum());
         int scale = valueBuffer.scale();
 
         // Buckets must be provided with their indices in ascending order.
@@ -161,6 +162,14 @@ private void mergeValuesToHistogram() {
         valueCount = 0;
     }
 
+    private double rawValuesSum() {
+        double sum = 0;
+        for (int i = 0; i < valueCount; i++) {
+            sum += rawValueBuffer[i];
+        }
+        return sum;
+    }
+
     private static long estimateBaseSize(int numBuckets) {
         return SHALLOW_SIZE + RamEstimationUtil.estimateDoubleArray(numBuckets);
     };

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java

@@ -150,6 +150,7 @@ public void add(ExponentialHistogram toAdd) {
             buffer = FixedCapacityExponentialHistogram.create(bucketLimit, circuitBreaker);
         }
         buffer.setZeroBucket(zeroBucket);
+        buffer.setSum(a.sum() + b.sum());
 
         // We attempt to bring everything to the scale of A.
         // This might involve increasing the scale for B, which would increase its indices.