Merge remote-tracking branch 'carlosdelest/feature/esql_dense_vector_support' into feature/esql_dense_vector_support

Author: carlosdelest

build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy

@@ -5,10 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
-import java.nio.file.Path
-
 class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
-    def "works"() {
+    def "basic functionality"() {
         given:
         file("src/main/java/com/example/Example.java") << """
             package com.example;
@@ -54,12 +52,70 @@ class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
 
         def location = Map.of(
             "module", "com.example",
-            "representative_class", Path.of("com", "example", "Example.class").toString()
+            "representative_class", "com/example/Example.class"
         )
         def expectedOutput = Map.of(
             "component", "example-component",
             "locations", List.of(location)
         )
         new ObjectMapper().readValue(output, Map.class) == expectedOutput
     }
+
+    def "dependencies"() {
+        buildFile << """
+        import org.elasticsearch.gradle.plugin.GenerateTestBuildInfoTask;
+
+        plugins {
+            id 'java'
+            id 'elasticsearch.test-build-info'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+
+        dependencies {
+            // We pin to specific versions here because they are known to have the properties we want to test.
+            // We're not actually running this code.
+            implementation "org.ow2.asm:asm:9.7.1" // has module-info.class
+            implementation "junit:junit:4.13" // has Automatic-Module-Name, and brings in hamcrest which does not
+        }
+
+        tasks.withType(GenerateTestBuildInfoTask.class) {
+            componentName = 'example-component'
+            outputFile = new File('build/generated-build-info/plugin-test-build-info.json')
+        }
+        """
+
+        when:
+        def result = gradleRunner('generateTestBuildInfo').build()
+        def task = result.task(":generateTestBuildInfo")
+
+
+        then:
+        task.outcome == TaskOutcome.SUCCESS
+
+        def output = file("build/generated-build-info/plugin-test-build-info.json")
+        output.exists() == true
+
+        def locationFromModuleInfo = Map.of(
+            "module", "org.objectweb.asm",
+            "representative_class", 'org/objectweb/asm/AnnotationVisitor.class'
+        )
+        def locationFromManifest = Map.of(
+            "module", "junit",
+            "representative_class", 'junit/textui/TestRunner.class'
+        )
+        def locationFromJarFileName = Map.of(
+            "module", "hamcrest.core",
+            "representative_class", 'org/hamcrest/BaseDescription.class'
+        )
+        def expectedOutput = Map.of(
+            "component", "example-component",
+            "locations", List.of(locationFromModuleInfo, locationFromManifest, locationFromJarFileName)
+        )
+
+        def value = new ObjectMapper().readValue(output, Map.class)
+        value == expectedOutput
+    }
 }

build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateTestBuildInfoTask.java

@@ -41,7 +41,7 @@
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.CodeSource;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
@@ -209,17 +209,16 @@ private String extractModuleNameFromJar(File file, JarFile jarFile) throws IOExc
      * @return a {@link StringBuilder} with the {@code META-INF/versions/<version number>} if it exists; otherwise null
      */
     private static StringBuilder versionDirectoryIfExists(JarFile jarFile) {
+        Comparator<Integer> numericOrder = Integer::compareTo;
         List<Integer> versions = jarFile.stream()
             .filter(je -> je.getName().startsWith(META_INF_VERSIONS_PREFIX) && je.getName().endsWith("/module-info.class"))
             .map(
                 je -> Integer.parseInt(
                     je.getName().substring(META_INF_VERSIONS_PREFIX.length(), je.getName().length() - META_INF_VERSIONS_PREFIX.length())
                 )
             )
+            .sorted(numericOrder.reversed())
             .toList();
-        versions = new ArrayList<>(versions);
-        versions.sort(Integer::compareTo);
-        versions = versions.reversed();
         int major = Runtime.version().feature();
         StringBuilder path = new StringBuilder(META_INF_VERSIONS_PREFIX);
         for (int version : versions) {
@@ -300,7 +299,10 @@ private String extractClassNameFromDirectory(File dir) throws IOException {
             public @NotNull FileVisitResult visitFile(@NotNull Path candidate, @NotNull BasicFileAttributes attrs) {
                 String name = candidate.getFileName().toString(); // Just the part after the last dir separator
                 if (name.endsWith(".class") && (name.equals("module-info.class") || name.contains("$")) == false) {
-                    result = candidate.toAbsolutePath().toString().substring(dir.getAbsolutePath().length() + 1);
+                    result = candidate.toAbsolutePath()
+                        .toString()
+                        .substring(dir.getAbsolutePath().length() + 1)
+                        .replace(File.separatorChar, '/');
                     return TERMINATE;
                 } else {
                     return CONTINUE;
@@ -316,20 +318,24 @@ private String extractClassNameFromDirectory(File dir) throws IOException {
      * if it exists or the preset one derived from the jar task
      */
     private String extractModuleNameFromDirectory(File dir) throws IOException {
-        List<File> files = new ArrayList<>(List.of(dir));
-        while (files.isEmpty() == false) {
-            File find = files.removeFirst();
-            if (find.exists()) {
-                if (find.getName().equals("module-info.class")) {
-                    try (InputStream inputStream = new FileInputStream(find)) {
-                        return extractModuleNameFromModuleInfo(inputStream);
+        var visitor = new SimpleFileVisitor<Path>() {
+            private String result = getModuleName().getOrNull();
+
+            @Override
+            public @NotNull FileVisitResult visitFile(@NotNull Path candidate, @NotNull BasicFileAttributes attrs) throws IOException {
+                String name = candidate.getFileName().toString(); // Just the part after the last dir separator
+                if (name.equals("module-info.class")) {
+                    try (InputStream inputStream = new FileInputStream(candidate.toFile())) {
+                        result = extractModuleNameFromModuleInfo(inputStream);
+                        return TERMINATE;
                     }
-                } else if (find.isDirectory()) {
-                    files.addAll(Arrays.asList(find.listFiles()));
+                } else {
+                    return CONTINUE;
                 }
             }
-        }
-        return getModuleName().getOrNull();
+        };
+        Files.walkFileTree(dir.toPath(), visitor);
+        return visitor.result;
     }
 
     /**

docs/changelog/128259.yaml

@@ -0,0 +1,6 @@
+pr: 128259
+summary: Added geometry validation for GEO types to exit early on invalid latitudes
+area: Geo
+type: bug
+issues:
+ - 128234

docs/changelog/128263.yaml

@@ -0,0 +1,5 @@
+pr: 128263
+summary: Allow lookup join on mixed numeric fields
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/128320.yaml

@@ -0,0 +1,5 @@
+pr: 128320
+summary: Use new source loader when lower `docId` is accessed
+area: Codec
+type: bug
+issues: []

muted-tests.yml

@@ -477,6 +477,24 @@ tests:
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceIT
   method: testSymlinkUpdateTriggerReload
   issue: https://github.com/elastic/elasticsearch/issues/128369
+- class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT
+  method: testEqualityAndOther {semantic_text}
+  issue: https://github.com/elastic/elasticsearch/issues/128414
+- class: org.elasticsearch.index.query.MultiMatchQueryBuilderTests
+  method: testToQuery
+  issue: https://github.com/elastic/elasticsearch/issues/128416
+- class: org.elasticsearch.xpack.ml.integration.InferenceIngestIT
+  method: testPipelineIngestWithModelAliases
+  issue: https://github.com/elastic/elasticsearch/issues/128417
+- class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
+  method: testCCSClusterDetailsWhereAllShardsSkippedInCanMatch
+  issue: https://github.com/elastic/elasticsearch/issues/128418
+- class: org.elasticsearch.xpack.esql.action.ForkIT
+  method: testProfile
+  issue: https://github.com/elastic/elasticsearch/issues/128377
+- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithFiltersIT
+  method: testTimestampFilterFromQuery
+  issue: https://github.com/elastic/elasticsearch/issues/127332
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java

@@ -29,11 +29,13 @@ class ConcurrentSegmentSourceProvider implements SourceProvider {
     private final SourceLoader sourceLoader;
     private final StoredFieldLoader storedFieldLoader;
     private final Map<Object, Leaf> leaves = ConcurrentCollections.newConcurrentMap();
+    private final boolean isStoredSource;
 
-    ConcurrentSegmentSourceProvider(SourceLoader loader, boolean loadSource) {
+    ConcurrentSegmentSourceProvider(SourceLoader loader, boolean isStoredSource) {
         this.sourceLoader = loader;
         // we force a sequential reader here since it is used during query execution where documents are scanned sequentially
-        this.storedFieldLoader = StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields(), true);
+        this.storedFieldLoader = StoredFieldLoader.create(isStoredSource, sourceLoader.requiredStoredFields(), true);
+        this.isStoredSource = isStoredSource;
     }
 
     @Override
@@ -44,6 +46,14 @@ public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
             leaf = new Leaf(sourceLoader.leaf(ctx.reader(), null), storedFieldLoader.getLoader(ctx, null));
             var existing = leaves.put(id, leaf);
             assert existing == null : "unexpected source provider [" + existing + "]";
+        } else if (isStoredSource == false && doc < leaf.doc) {
+            // When queries reference the same runtime field in multiple clauses, each clause re-reads the values from the source in
+            // increasing docId order. So the last docId accessed by the first clause is higher than the first docId read by the second
+            // clause. This is okay for stored source, as stored fields do not restrict the order that docIds that can be accessed.
+            // But with synthetic source, field values may come from doc values, which require than docIds only be read in increasing order.
+            // To handle this, we detect lower docIds and create a new doc value reader for each clause.
+            leaf = new Leaf(sourceLoader.leaf(ctx.reader(), null), storedFieldLoader.getLoader(ctx, null));
+            leaves.put(id, leaf);
         }
         return leaf.getSource(ctx, doc);
     }

x-pack/plugin/build.gradle

@@ -116,6 +116,8 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("esql/40_tsdb/from index pattern unsupported counter", "TODO: support for subset of metric fields")
   task.skipTest("esql/40_unsupported_types/unsupported", "TODO: support for subset of metric fields")
   task.skipTest("esql/40_unsupported_types/unsupported with sort", "TODO: support for subset of metric fields")
+  task.skipTest("service_accounts/10_basic/Test get service accounts", "Enterprise Search service account removed, invalidating the current tests")
+  task.skipTest("service_accounts/10_basic/Test service account tokens", "Enterprise Search service account removed, invalidating the current tests")
   task.skipTest("ml/3rd_party_deployment/Test clear deployment cache", "Deprecated route removed")
   task.skipTest("ml/3rd_party_deployment/Test start and stop deployment with cache", "Deprecated route removed")
   task.skipTest("ml/3rd_party_deployment/Test start and stop multiple deployments", "Deprecated route removed")

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtilsTests.java

@@ -63,7 +63,7 @@ public void testAddAuthorizationInfoWithApiKey() throws IOException {
     }
 
     public void testAddAuthorizationInfoWithServiceAccount() throws IOException {
-        String account = "elastic/" + randomFrom("kibana", "fleet-server", "enterprise-search-server");
+        String account = "elastic/" + randomFrom("kibana", "fleet-server");
         User user = new User(account);
         AuthenticationTestBuilder builder = AuthenticationTestHelper.builder().serviceAccount(user);
         Authentication authentication = builder.build();

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SpatialCoordinateTypes.java

@@ -12,6 +12,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.utils.GeographyValidator;
 import org.elasticsearch.geometry.utils.GeometryValidator;
 import org.elasticsearch.geometry.utils.WellKnownBinary;
 import org.elasticsearch.geometry.utils.WellKnownText;
@@ -32,6 +33,11 @@ public long pointAsLong(double x, double y) {
             int longitudeEncoded = encodeLongitude(x);
             return (((long) latitudeEncoded) << 32) | (longitudeEncoded & 0xFFFFFFFFL);
         }
+
+        public GeometryValidator validator() {
+            // We validate the lat/lon values, and ignore any z values
+            return GeographyValidator.instance(true);
+        }
     },
     CARTESIAN {
 
@@ -67,6 +73,10 @@ public long pointAsLong(double x, double y) {
         }
     };
 
+    protected GeometryValidator validator() {
+        return GeometryValidator.NOOP;
+    }
+
     public abstract Point longAsPoint(long encoded);
 
     public abstract long pointAsLong(double x, double y);
@@ -77,7 +87,7 @@ public long wkbAsLong(BytesRef wkb) {
     }
 
     public Point wkbAsPoint(BytesRef wkb) {
-        Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length);
+        Geometry geometry = WellKnownBinary.fromWKB(validator(), false, wkb.bytes, wkb.offset, wkb.length);
         if (geometry instanceof Point point) {
             return point;
         } else {
@@ -101,26 +111,18 @@ public BytesRef wktToWkb(String wkt) {
         // TODO: we should be able to transform WKT to WKB without building the geometry
         // we should as well use different validator for cartesian and geo?
         try {
-            Geometry geometry = WellKnownText.fromWKT(GeometryValidator.NOOP, false, wkt);
+            Geometry geometry = WellKnownText.fromWKT(validator(), false, wkt);
             return new BytesRef(WellKnownBinary.toWKB(geometry, ByteOrder.LITTLE_ENDIAN));
         } catch (Exception e) {
             throw new IllegalArgumentException("Failed to parse WKT: " + e.getMessage(), e);
         }
     }
 
-    public Geometry wktToGeometry(String wkt) {
-        try {
-            return WellKnownText.fromWKT(GeometryValidator.NOOP, false, wkt);
-        } catch (Exception e) {
-            throw new IllegalArgumentException("Failed to parse WKT: " + e.getMessage(), e);
-        }
-    }
-
     public String wkbToWkt(BytesRef wkb) {
         return WellKnownText.fromWKB(wkb.bytes, wkb.offset, wkb.length);
     }
 
     public Geometry wkbToGeometry(BytesRef wkb) {
-        return WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length);
+        return WellKnownBinary.fromWKB(validator(), false, wkb.bytes, wkb.offset, wkb.length);
     }
 }