Merge branch 'main' into cps-expression-rewriter

Author: n1v0lg

docs/changelog/130485.yaml

@@ -0,0 +1,5 @@
+pr: 130485
+summary: Add `RerankRequestChunker`
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/134080.yaml

@@ -0,0 +1,5 @@
+pr: 134080
+summary: Added Google Model Garden Anthropic Completion and Chat Completion support to the Inference Plugin
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/135051.yaml

@@ -3,3 +3,13 @@ summary: Ban Limit + `MvExpand` before remote Enrich
 area: ES|QL
 type: bug
 issues: []
+highlight:
+  title: Prevent LIMIT + MV_EXPAND before remote ENRICH
+  body: |-
+    Queries using LIMIT followed by MV_EXPAND before a remote ENRICH can produce incorrect results due to distributed execution semantics.
+    These queries are now unsupported and produce an error. Example:
+    [source,yaml]
+    ----------------------------
+    FROM *:events | SORT @timestamp | LIMIT 2 | MV_EXPAND ip | ENRICH _remote:clientip_policy ON ip
+    ----------------------------
+    To avoid this error, reorder your query, for example by moving ENRICH earlier in the pipeline.

docs/changelog/135204.yaml

@@ -0,0 +1,6 @@
+pr: 135204
+summary: Make `_tsid` available in metadata
+area: ES|QL
+type: enhancement
+issues:
+ - 133205

docs/changelog/135337.yaml

@@ -0,0 +1,5 @@
+pr: 135337
+summary: Do not pass `ProjectMetadata` to lazy index permissions builder
+area: Security
+type: enhancement
+issues: []

docs/changelog/135402.yaml

@@ -0,0 +1,5 @@
+pr: 135402
+summary: Improve TSDB ingestion by hashing dimensions only once, using a new auto-populeted `index.dimensions` private index setting
+area: TSDB
+type: enhancement
+issues: []

libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java

@@ -24,6 +24,7 @@
 import java.util.stream.Collectors;
 
 public class Utils {
+    private static final FileSystem JRT_FS = FileSystems.getFileSystem(URI.create("jrt:/"));
 
     // TODO Currently ServerProcessBuilder is using --add-modules=ALL-MODULE-PATH, should this rather
     // reflect below excludes (except for java.desktop which requires a special handling)?
@@ -47,9 +48,9 @@ public class Utils {
         && m.contains(".internal.") == false
         && m.contains(".incubator.") == false;
 
-    private static Map<String, Set<String>> findModuleExports(FileSystem fs) throws IOException {
+    public static Map<String, Set<String>> findModuleExports() throws IOException {
         var modulesExports = new HashMap<String, Set<String>>();
-        try (var stream = Files.walk(fs.getPath("modules"))) {
+        try (var stream = Files.walk(JRT_FS.getPath("modules"))) {
             stream.filter(p -> p.getFileName().toString().equals("module-info.class")).forEach(x -> {
                 try (var is = Files.newInputStream(x)) {
                     var md = ModuleDescriptor.read(is);
@@ -74,21 +75,20 @@ public interface JdkModuleConsumer {
     }
 
     public static void walkJdkModules(JdkModuleConsumer c) throws IOException {
-        walkJdkModules(DEFAULT_MODULE_PREDICATE, c);
+        walkJdkModules(DEFAULT_MODULE_PREDICATE, Utils.findModuleExports(), c);
     }
 
-    public static void walkJdkModules(Predicate<String> modulePredicate, JdkModuleConsumer c) throws IOException {
-        FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/"));
-
-        var moduleExports = Utils.findModuleExports(fs);
-        try (var stream = Files.walk(fs.getPath("modules"))) {
-            var modules = stream.filter(x -> x.toString().endsWith(".class"))
-                .collect(Collectors.groupingBy(x -> x.subpath(1, 2).toString()));
+    public static void walkJdkModules(Predicate<String> modulePredicate, Map<String, Set<String>> exportsByModule, JdkModuleConsumer c)
+        throws IOException {
+        try (var stream = Files.walk(JRT_FS.getPath("modules"))) {
+            var modules = stream.filter(
+                x -> x.toString().endsWith(".class") && x.getFileName().toString().equals("module-info.class") == false
+            ).collect(Collectors.groupingBy(x -> x.subpath(1, 2).toString()));
 
             for (var kv : modules.entrySet()) {
                 var moduleName = kv.getKey();
                 if (modulePredicate.test(moduleName)) {
-                    var thisModuleExports = moduleExports.get(moduleName);
+                    var thisModuleExports = exportsByModule.get(moduleName);
                     c.accept(moduleName, kv.getValue(), thisModuleExports);
                 }
             }

libs/entitlement/tools/jdk-api-extractor/src/main/java/org/elasticsearch/entitlement/tools/jdkapi/JdkApiExtractor.java

@@ -24,6 +24,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -62,21 +63,34 @@ public static void main(String[] args) throws IOException {
         validateArgs(args);
         boolean deprecationsOnly = optionalArgs(args).anyMatch(DEPRECATIONS_ONLY::equals);
 
-        Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass = new TreeMap<>();
-        Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass = new TreeMap<>();
-        Map<String, Set<AccessibleMethod>> deprecationsByClass = new TreeMap<>();
+        final Map<String, String> moduleNameByClass = new HashMap<>();
+        final Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass = new TreeMap<>();
+        final Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass = new TreeMap<>();
+        final Map<String, Set<AccessibleMethod>> deprecationsByClass = new TreeMap<>();
 
+        final Map<String, Set<String>> exportsByModule = Utils.findModuleExports();
+        // 1st: map class names to module names (including later excluded modules) for lookup in 2nd step
+        Utils.walkJdkModules(m -> true, exportsByModule, (moduleName, moduleClasses, moduleExports) -> {
+            for (var classFile : moduleClasses) {
+                String prev = moduleNameByClass.put(internalClassName(classFile, moduleName), moduleName);
+                if (prev != null) {
+                    throw new IllegalStateException("Class " + classFile + " is in both modules " + prev + " and " + moduleName);
+                }
+            }
+        });
+
+        var visitor = new AccessibleClassVisitor(
+            moduleNameByClass,
+            exportsByModule,
+            accessibleImplementationsByClass,
+            accessibleForOverridesByClass,
+            deprecationsByClass
+        );
         Predicate<String> modulePredicate = Utils.DEFAULT_MODULE_PREDICATE.or(
             m -> optionalArgs(args).anyMatch(INCLUDE_INCUBATOR::equals) && m.contains(".incubator.")
         );
-
-        Utils.walkJdkModules(modulePredicate, (moduleName, moduleClasses, moduleExports) -> {
-            var visitor = new AccessibleClassVisitor(
-                moduleExports,
-                accessibleImplementationsByClass,
-                accessibleForOverridesByClass,
-                deprecationsByClass
-            );
+        // 2nd: calculate accessible implementations of classes in included modules
+        Utils.walkJdkModules(modulePredicate, exportsByModule, (moduleName, moduleClasses, moduleExports) -> {
             for (var classFile : moduleClasses) {
                 // skip if class was already visited earlier due to a dependency on it
                 if (accessibleImplementationsByClass.containsKey(internalClassName(classFile, moduleName))) {
@@ -91,7 +105,18 @@ public static void main(String[] args) throws IOException {
             }
         });
 
-        writeFile(Path.of(args[0]), deprecationsOnly ? deprecationsByClass : accessibleImplementationsByClass);
+        // finally, skip some implementations we're not interested in
+        Predicate<Map.Entry<String, Set<AccessibleMethod>>> predicate = entry -> {
+            if (entry.getKey().startsWith("com/sun/") && entry.getKey().contains("/internal/")) {
+                // skip com.sun.*.internal classes as they are not part of the supported JDK API
+                // even if methods override some publicly visible API
+                return false;
+            }
+            // skip classes that are not part of included modules, but checked due to dependencies
+            String moduleName = moduleNameByClass.get(entry.getKey());
+            return modulePredicate.test(moduleName);
+        };
+        writeFile(Path.of(args[0]), deprecationsOnly ? deprecationsByClass : accessibleImplementationsByClass, predicate);
     }
 
     private static String internalClassName(Path clazz, String moduleName) {
@@ -132,9 +157,17 @@ private static boolean isWritableOutputPath(String pathStr) {
     }
 
     @SuppressForbidden(reason = "cli tool printing to standard err/out")
-    private static void writeFile(Path path, Map<String, Set<AccessibleMethod>> methods) throws IOException {
+    private static void writeFile(
+        Path path,
+        Map<String, Set<AccessibleMethod>> methods,
+        Predicate<Map.Entry<String, Set<AccessibleMethod>>> predicate
+    ) throws IOException {
         System.out.println("Writing result for " + Runtime.version() + " to " + path.toAbsolutePath());
-        Files.write(path, () -> methods.entrySet().stream().flatMap(AccessibleMethod::toLines).iterator(), StandardCharsets.UTF_8);
+        Files.write(
+            path,
+            () -> methods.entrySet().stream().filter(predicate).flatMap(AccessibleMethod::toLines).iterator(),
+            StandardCharsets.UTF_8
+        );
     }
 
     record AccessibleMethod(String method, String descriptor, boolean isPublic, boolean isFinal, boolean isStatic) {
@@ -163,7 +196,8 @@ static Stream<CharSequence> toLines(Map.Entry<String, Set<AccessibleMethod>> ent
     }
 
     static class AccessibleClassVisitor extends ClassVisitor {
-        private final Set<String> moduleExports;
+        private final Map<String, String> moduleNameByClass;
+        private final Map<String, Set<String>> exportsByModule;
         private final Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass;
         private final Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass;
         private final Map<String, Set<AccessibleMethod>> deprecationsByClass;
@@ -179,13 +213,15 @@ static class AccessibleClassVisitor extends ClassVisitor {
         private boolean isExported;
 
         AccessibleClassVisitor(
-            Set<String> moduleExports,
+            Map<String, String> moduleNameByClass,
+            Map<String, Set<String>> exportsByModule,
             Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass,
             Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass,
             Map<String, Set<AccessibleMethod>> deprecationsByClass
         ) {
             super(ASM9);
-            this.moduleExports = moduleExports;
+            this.moduleNameByClass = moduleNameByClass;
+            this.exportsByModule = exportsByModule;
             this.accessibleImplementationsByClass = accessibleImplementationsByClass;
             this.accessibleForOverridesByClass = accessibleForOverridesByClass;
             this.deprecationsByClass = deprecationsByClass;
@@ -214,7 +250,7 @@ public void visit(int version, int access, String name, String signature, String
             }
             // only initialize local state AFTER visiting all dependencies above!
             super.visit(version, access, name, signature, superName, interfaces);
-            this.isExported = moduleExports.contains(getPackageName(name));
+            this.isExported = getModuleExports(getModuleName(name)).contains(getPackageName(name));
             this.className = name;
             this.isPublicClass = (access & ACC_PUBLIC) != 0;
             this.isFinalClass = (access & ACC_FINAL) != 0;
@@ -224,6 +260,22 @@ public void visit(int version, int access, String name, String signature, String
             this.deprecations = newSortedSet();
         }
 
+        private String getModuleName(String name) {
+            String module = moduleNameByClass.get(name);
+            if (module == null) {
+                throw new IllegalStateException("Unknown module for class: " + name);
+            }
+            return module;
+        }
+
+        private Set<String> getModuleExports(String module) {
+            Set<String> exports = exportsByModule.get(module);
+            if (exports == null) {
+                throw new IllegalStateException("Unknown exports for module: " + module);
+            }
+            return exports;
+        }
+
         @Override
         public void visitEnd() {
             super.visitEnd();

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java

@@ -56,12 +56,21 @@ public class ExponentialHistogramMerger implements Accountable, Releasable {
     /**
      * Creates a new instance with the specified bucket limit.
      *
-     * @param bucketLimit the maximum number of buckets the result histogram is allowed to have
+     * @param bucketLimit the maximum number of buckets the result histogram is allowed to have, must be at least 4
      * @param circuitBreaker the circuit breaker to use to limit memory allocations
      */
     public static ExponentialHistogramMerger create(int bucketLimit, ExponentialHistogramCircuitBreaker circuitBreaker) {
         circuitBreaker.adjustBreaker(BASE_SIZE);
-        return new ExponentialHistogramMerger(bucketLimit, circuitBreaker);
+        boolean success = false;
+        try {
+            ExponentialHistogramMerger result = new ExponentialHistogramMerger(bucketLimit, circuitBreaker);
+            success = true;
+            return result;
+        } finally {
+            if (success == false) {
+                circuitBreaker.adjustBreaker(-BASE_SIZE);
+            }
+        }
     }
 
     private ExponentialHistogramMerger(int bucketLimit, ExponentialHistogramCircuitBreaker circuitBreaker) {
@@ -70,6 +79,10 @@ private ExponentialHistogramMerger(int bucketLimit, ExponentialHistogramCircuitB
 
     // Only intended for testing, using this in production means an unnecessary reduction of precision
     private ExponentialHistogramMerger(int bucketLimit, int maxScale, ExponentialHistogramCircuitBreaker circuitBreaker) {
+        // We need at least four buckets to represent any possible distribution
+        if (bucketLimit < 4) {
+            throw new IllegalArgumentException("The bucket limit must be at least 4");
+        }
         this.bucketLimit = bucketLimit;
         this.maxScale = maxScale;
         this.circuitBreaker = circuitBreaker;
@@ -124,11 +137,16 @@ public ReleasableExponentialHistogram getAndClear() {
         return retVal;
     }
 
-    // TODO(b/128622): this algorithm is very efficient if b has roughly as many buckets as a
-    // However, if b is much smaller we still have to iterate over all buckets of a which is very wasteful.
-    // This can be optimized by buffering multiple histograms to accumulate first,
-    // then in O(log(n)) turn them into a single, merged histogram.
-    // (n is the number of buffered buckets)
+    // This algorithm is very efficient if B has roughly as many buckets as A.
+    // However, if B is much smaller we still have to iterate over all buckets of A.
+    // This can be optimized by buffering the buckets of small histograms and only merging them when we have enough buckets.
+    // The buffered histogram buckets would first be merged with each other, and then be merged with accumulator.
+    //
+    // However, benchmarks of a PoC implementation have shown that this only brings significant improvements
+    // if the accumulator size is 500+ and the merged histograms are smaller than 50 buckets
+    // and otherwise slows down the merging.
+    // It would be possible to only enable the buffering for small histograms,
+    // but the optimization seems not worth the added complexity at this point.
 
     /**
      * Merges the given histogram into the current result.

libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGeneratorTests.java

@@ -33,7 +33,7 @@ public class ExponentialHistogramGeneratorTests extends ExponentialHistogramTest
 
     public void testVeryLargeValue() {
         double value = Double.MAX_VALUE / 10;
-        ExponentialHistogram histo = createAutoReleasedHistogram(1, value);
+        ExponentialHistogram histo = createAutoReleasedHistogram(4, value);
         long index = histo.positiveBuckets().iterator().peekIndex();
         int scale = histo.scale();