Merge branch 'main' into fix-test-framework-pom

Author: breskeby

docs/changelog/134080.yaml

@@ -0,0 +1,5 @@
+pr: 134080
+summary: Added Google Model Garden Anthropic Completion and Chat Completion support to the Inference Plugin
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/135051.yaml

@@ -3,3 +3,13 @@ summary: Ban Limit + `MvExpand` before remote Enrich
 area: ES|QL
 type: bug
 issues: []
+highlight:
+  title: Prevent LIMIT + MV_EXPAND before remote ENRICH
+  body: |-
+    Queries using LIMIT followed by MV_EXPAND before a remote ENRICH can produce incorrect results due to distributed execution semantics.
+    These queries are now unsupported and produce an error. Example:
+    [source,yaml]
+    ----------------------------
+    FROM *:events | SORT @timestamp | LIMIT 2 | MV_EXPAND ip | ENRICH _remote:clientip_policy ON ip
+    ----------------------------
+    To avoid this error, reorder your query, for example by moving ENRICH earlier in the pipeline.

docs/changelog/135204.yaml

@@ -0,0 +1,6 @@
+pr: 135204
+summary: Make `_tsid` available in metadata
+area: ES|QL
+type: enhancement
+issues:
+ - 133205

docs/changelog/135337.yaml

@@ -0,0 +1,5 @@
+pr: 135337
+summary: Do not pass `ProjectMetadata` to lazy index permissions builder
+area: Security
+type: enhancement
+issues: []

docs/reference/enrich-processor/geoip-processor.md

@@ -206,14 +206,18 @@ PUT my_ip_locations
 
 If you can’t [automatically update](#geoip-automatic-updates) your IP geolocation databases from the Elastic endpoint, you have a few other options:
 
-* [Use a proxy endpoint](#use-proxy-geoip-endpoint)
+* [Use a reverse proxy endpoint](#use-proxy-geoip-endpoint)
 * [Use a custom endpoint](#use-custom-geoip-endpoint)
 * [Manually update your IP geolocation databases](#manually-update-geoip-databases)
 
 $$$use-proxy-geoip-endpoint$$$
-**Use a proxy endpoint**
+**Use a reverse proxy endpoint**
 
-If you can’t connect directly to the Elastic GeoIP endpoint, consider setting up a secure proxy. You can then specify the proxy endpoint URL in the [`ingest.geoip.downloader.endpoint`](#ingest-geoip-downloader-endpoint) setting of each node’s `elasticsearch.yml` file.
+If you can’t connect directly to the Elastic GeoIP endpoint, consider setting up a secure reverse proxy. You can then specify the reverse proxy endpoint URL in the [`ingest.geoip.downloader.endpoint`](#ingest-geoip-downloader-endpoint) setting of each node’s `elasticsearch.yml` file.
+
+:::{note}
+True HTTP proxy support for GeoIP database downloads is not currently available in {{es}}.
+:::
 
 In a strict setup the following domains may need to be added to the allowed domains list:
 

docs/reference/query-languages/esql/_snippets/functions/description/count_distinct.md

@@ -24,6 +24,7 @@
 import java.util.stream.Collectors;
 
 public class Utils {
+    private static final FileSystem JRT_FS = FileSystems.getFileSystem(URI.create("jrt:/"));
 
     // TODO Currently ServerProcessBuilder is using --add-modules=ALL-MODULE-PATH, should this rather
     // reflect below excludes (except for java.desktop which requires a special handling)?
@@ -47,9 +48,9 @@ public class Utils {
         && m.contains(".internal.") == false
         && m.contains(".incubator.") == false;
 
-    private static Map<String, Set<String>> findModuleExports(FileSystem fs) throws IOException {
+    public static Map<String, Set<String>> findModuleExports() throws IOException {
         var modulesExports = new HashMap<String, Set<String>>();
-        try (var stream = Files.walk(fs.getPath("modules"))) {
+        try (var stream = Files.walk(JRT_FS.getPath("modules"))) {
             stream.filter(p -> p.getFileName().toString().equals("module-info.class")).forEach(x -> {
                 try (var is = Files.newInputStream(x)) {
                     var md = ModuleDescriptor.read(is);
@@ -74,21 +75,20 @@ public interface JdkModuleConsumer {
     }
 
     public static void walkJdkModules(JdkModuleConsumer c) throws IOException {
-        walkJdkModules(DEFAULT_MODULE_PREDICATE, c);
+        walkJdkModules(DEFAULT_MODULE_PREDICATE, Utils.findModuleExports(), c);
     }
 
-    public static void walkJdkModules(Predicate<String> modulePredicate, JdkModuleConsumer c) throws IOException {
-        FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/"));
-
-        var moduleExports = Utils.findModuleExports(fs);
-        try (var stream = Files.walk(fs.getPath("modules"))) {
-            var modules = stream.filter(x -> x.toString().endsWith(".class"))
-                .collect(Collectors.groupingBy(x -> x.subpath(1, 2).toString()));
+    public static void walkJdkModules(Predicate<String> modulePredicate, Map<String, Set<String>> exportsByModule, JdkModuleConsumer c)
+        throws IOException {
+        try (var stream = Files.walk(JRT_FS.getPath("modules"))) {
+            var modules = stream.filter(
+                x -> x.toString().endsWith(".class") && x.getFileName().toString().equals("module-info.class") == false
+            ).collect(Collectors.groupingBy(x -> x.subpath(1, 2).toString()));
 
             for (var kv : modules.entrySet()) {
                 var moduleName = kv.getKey();
                 if (modulePredicate.test(moduleName)) {
-                    var thisModuleExports = moduleExports.get(moduleName);
+                    var thisModuleExports = exportsByModule.get(moduleName);
                     c.accept(moduleName, kv.getValue(), thisModuleExports);
                 }
             }

docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json

@@ -24,6 +24,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -62,21 +63,34 @@ public static void main(String[] args) throws IOException {
         validateArgs(args);
         boolean deprecationsOnly = optionalArgs(args).anyMatch(DEPRECATIONS_ONLY::equals);
 
-        Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass = new TreeMap<>();
-        Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass = new TreeMap<>();
-        Map<String, Set<AccessibleMethod>> deprecationsByClass = new TreeMap<>();
+        final Map<String, String> moduleNameByClass = new HashMap<>();
+        final Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass = new TreeMap<>();
+        final Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass = new TreeMap<>();
+        final Map<String, Set<AccessibleMethod>> deprecationsByClass = new TreeMap<>();
 
+        final Map<String, Set<String>> exportsByModule = Utils.findModuleExports();
+        // 1st: map class names to module names (including later excluded modules) for lookup in 2nd step
+        Utils.walkJdkModules(m -> true, exportsByModule, (moduleName, moduleClasses, moduleExports) -> {
+            for (var classFile : moduleClasses) {
+                String prev = moduleNameByClass.put(internalClassName(classFile, moduleName), moduleName);
+                if (prev != null) {
+                    throw new IllegalStateException("Class " + classFile + " is in both modules " + prev + " and " + moduleName);
+                }
+            }
+        });
+
+        var visitor = new AccessibleClassVisitor(
+            moduleNameByClass,
+            exportsByModule,
+            accessibleImplementationsByClass,
+            accessibleForOverridesByClass,
+            deprecationsByClass
+        );
         Predicate<String> modulePredicate = Utils.DEFAULT_MODULE_PREDICATE.or(
             m -> optionalArgs(args).anyMatch(INCLUDE_INCUBATOR::equals) && m.contains(".incubator.")
         );
-
-        Utils.walkJdkModules(modulePredicate, (moduleName, moduleClasses, moduleExports) -> {
-            var visitor = new AccessibleClassVisitor(
-                moduleExports,
-                accessibleImplementationsByClass,
-                accessibleForOverridesByClass,
-                deprecationsByClass
-            );
+        // 2nd: calculate accessible implementations of classes in included modules
+        Utils.walkJdkModules(modulePredicate, exportsByModule, (moduleName, moduleClasses, moduleExports) -> {
             for (var classFile : moduleClasses) {
                 // skip if class was already visited earlier due to a dependency on it
                 if (accessibleImplementationsByClass.containsKey(internalClassName(classFile, moduleName))) {
@@ -91,7 +105,18 @@ public static void main(String[] args) throws IOException {
             }
         });
 
-        writeFile(Path.of(args[0]), deprecationsOnly ? deprecationsByClass : accessibleImplementationsByClass);
+        // finally, skip some implementations we're not interested in
+        Predicate<Map.Entry<String, Set<AccessibleMethod>>> predicate = entry -> {
+            if (entry.getKey().startsWith("com/sun/") && entry.getKey().contains("/internal/")) {
+                // skip com.sun.*.internal classes as they are not part of the supported JDK API
+                // even if methods override some publicly visible API
+                return false;
+            }
+            // skip classes that are not part of included modules, but checked due to dependencies
+            String moduleName = moduleNameByClass.get(entry.getKey());
+            return modulePredicate.test(moduleName);
+        };
+        writeFile(Path.of(args[0]), deprecationsOnly ? deprecationsByClass : accessibleImplementationsByClass, predicate);
     }
 
     private static String internalClassName(Path clazz, String moduleName) {
@@ -132,9 +157,17 @@ private static boolean isWritableOutputPath(String pathStr) {
     }
 
     @SuppressForbidden(reason = "cli tool printing to standard err/out")
-    private static void writeFile(Path path, Map<String, Set<AccessibleMethod>> methods) throws IOException {
+    private static void writeFile(
+        Path path,
+        Map<String, Set<AccessibleMethod>> methods,
+        Predicate<Map.Entry<String, Set<AccessibleMethod>>> predicate
+    ) throws IOException {
         System.out.println("Writing result for " + Runtime.version() + " to " + path.toAbsolutePath());
-        Files.write(path, () -> methods.entrySet().stream().flatMap(AccessibleMethod::toLines).iterator(), StandardCharsets.UTF_8);
+        Files.write(
+            path,
+            () -> methods.entrySet().stream().filter(predicate).flatMap(AccessibleMethod::toLines).iterator(),
+            StandardCharsets.UTF_8
+        );
     }
 
     record AccessibleMethod(String method, String descriptor, boolean isPublic, boolean isFinal, boolean isStatic) {
@@ -163,7 +196,8 @@ static Stream<CharSequence> toLines(Map.Entry<String, Set<AccessibleMethod>> ent
     }
 
     static class AccessibleClassVisitor extends ClassVisitor {
-        private final Set<String> moduleExports;
+        private final Map<String, String> moduleNameByClass;
+        private final Map<String, Set<String>> exportsByModule;
         private final Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass;
         private final Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass;
         private final Map<String, Set<AccessibleMethod>> deprecationsByClass;
@@ -179,13 +213,15 @@ static class AccessibleClassVisitor extends ClassVisitor {
         private boolean isExported;
 
         AccessibleClassVisitor(
-            Set<String> moduleExports,
+            Map<String, String> moduleNameByClass,
+            Map<String, Set<String>> exportsByModule,
             Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass,
             Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass,
             Map<String, Set<AccessibleMethod>> deprecationsByClass
         ) {
             super(ASM9);
-            this.moduleExports = moduleExports;
+            this.moduleNameByClass = moduleNameByClass;
+            this.exportsByModule = exportsByModule;
             this.accessibleImplementationsByClass = accessibleImplementationsByClass;
             this.accessibleForOverridesByClass = accessibleForOverridesByClass;
             this.deprecationsByClass = deprecationsByClass;
@@ -214,7 +250,7 @@ public void visit(int version, int access, String name, String signature, String
             }
             // only initialize local state AFTER visiting all dependencies above!
             super.visit(version, access, name, signature, superName, interfaces);
-            this.isExported = moduleExports.contains(getPackageName(name));
+            this.isExported = getModuleExports(getModuleName(name)).contains(getPackageName(name));
             this.className = name;
             this.isPublicClass = (access & ACC_PUBLIC) != 0;
             this.isFinalClass = (access & ACC_FINAL) != 0;
@@ -224,6 +260,22 @@ public void visit(int version, int access, String name, String signature, String
             this.deprecations = newSortedSet();
         }
 
+        private String getModuleName(String name) {
+            String module = moduleNameByClass.get(name);
+            if (module == null) {
+                throw new IllegalStateException("Unknown module for class: " + name);
+            }
+            return module;
+        }
+
+        private Set<String> getModuleExports(String module) {
+            Set<String> exports = exportsByModule.get(module);
+            if (exports == null) {
+                throw new IllegalStateException("Unknown exports for module: " + module);
+            }
+            return exports;
+        }
+
         @Override
         public void visitEnd() {
             super.visitEnd();