Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed (#92957)

9.5 will include several changes related to vector search. An extensive list is available at https://github.com/apache/lucene/milestone/4 .

Author: javanna

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.7.0
-lucene            = 9.4.2
+lucene            = 9.5.0-snapshot-d19c3e2e0ed
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 19.0.1+10@afdd2e245b014143b62ccb916125e3ce
@@ -21,12 +21,12 @@ netty             = 4.1.86.Final
 
 commons_lang3                   = 3.9
 
+antlr4            = 4.11.1
 # when updating this version, you need to ensure compatibility with:
 #  - modules/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
 #  - x-pack/plugin/security
 bouncycastle=1.64
-
 # used by security and idp (need to be in sync due to cross-dependency in testing)
 opensaml = 4.0.1
 

distribution/src/config/jvm.options

@@ -73,3 +73,6 @@
 
 ## GC logging
 -Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,level,pid,tags:filecount=32,filesize=64m
+
+# temporarily disable the Panama-based MMapDirectory
+-Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.4.2
-:lucene_version_path:   9_4_2
+:lucene_version:        9.5.0
+:lucene_version_path:   9_5_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/92957.yaml

@@ -0,0 +1,5 @@
+pr: 92957
+summary: Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed
+area: Search
+type: upgrade
+issues: []

gradle/verification-metadata.xml

@@ -21,7 +21,6 @@
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
 import org.apache.lucene.geo.Point;
-import org.apache.lucene.spatial3d.geom.GeoArea;
 import org.apache.lucene.spatial3d.geom.GeoPoint;
 import org.apache.lucene.spatial3d.geom.GeoPolygon;
 import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
@@ -133,23 +132,13 @@ private void assertHexRing(int res, String h3Address, String[] children) {
     public void testNoChildrenIntersecting() {
         String[] h3Addresses = H3.getStringRes0Cells();
         String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
-        // Once testIssue91915 is fixed, put upper limit of the loop to H3.MAX_H3_RES
-        for (int i = 1; i <= 10; i++) {
+        for (int i = 1; i <= H3.MAX_H3_RES; i++) {
             h3Addresses = H3.h3ToChildren(h3Address);
             assertIntersectingChildren(h3Address, h3Addresses);
             h3Address = RandomPicks.randomFrom(random(), h3Addresses);
         }
     }
 
-    public void testIssue91915() {
-        GeoPolygon polygon1 = getGeoPolygon("8ec82ea0650155f");
-        GeoPolygon polygon2 = getGeoPolygon("8ec82ea06501447");
-        // these polygons are disjoint but due to https://github.com/apache/lucene/issues/11883
-        // they are reported as intersects. Once this is fixed this test will fail, we should adjust
-        // testNoChildrenIntersecting
-        assertEquals("see https://github.com/elastic/elasticsearch/issues/91915", GeoArea.OVERLAPS, polygon1.getRelationship(polygon2));
-    }
-
     private void assertIntersectingChildren(String h3Address, String[] children) {
         int size = H3.h3ToNotIntersectingChildrenSize(h3Address);
         for (int i = 0; i < size; i++) {

libs/h3/src/test/java/org/elasticsearch/h3/ParentChildNavigationTests.java

@@ -435,6 +435,22 @@ public XContentBuilder value(byte value) throws IOException {
         return this;
     }
 
+    public XContentBuilder array(String name, byte[] values) throws IOException {
+        return field(name).values(values);
+    }
+
+    private XContentBuilder values(byte[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (byte b : values) {
+            value(b);
+        }
+        endArray();
+        return this;
+    }
+
     ////////////////////////////////////////////////////////////////////////////
     // Double
     //////////////////////////////////

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java

@@ -17,7 +17,7 @@ esplugin {
 dependencies {
   api "org.apache.lucene:lucene-expressions:${versions.lucene}"
   runtimeOnly "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  runtimeOnly 'org.antlr:antlr4-runtime:4.5.1-1'
+  runtimeOnly "org.antlr:antlr4-runtime:${versions.antlr4}"
   runtimeOnly 'org.ow2.asm:asm:7.2'
   runtimeOnly 'org.ow2.asm:asm-commons:7.2'
   runtimeOnly 'org.ow2.asm:asm-tree:7.2'

modules/lang-expression/build.gradle

@@ -34,7 +34,7 @@ configurations {
 }
 
 dependencies {
-    api 'org.antlr:antlr4-runtime:4.5.3'
+    api "org.antlr:antlr4-runtime:${versions.antlr4}"
     api 'org.ow2.asm:asm-util:7.2'
     api 'org.ow2.asm:asm-tree:7.2'
     api 'org.ow2.asm:asm-commons:7.2'
@@ -174,7 +174,7 @@ configurations {
 }
 
 dependencies {
-    regenerate 'org.antlr:antlr4:4.5.3'
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String grammarPath = 'src/main/antlr'
@@ -274,7 +274,7 @@ configurations {
 }
 
 dependencies {
-    regenerate 'org.antlr:antlr4:4.5.3'
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String suggestGrammarPath = 'src/main/antlr'

modules/lang-painless/build.gradle

@@ -13,7 +13,7 @@
     requires org.elasticsearch.server;
     requires org.elasticsearch.xcontent;
 
-    requires antlr4.runtime;
+    requires org.antlr.antlr4.runtime;
     requires org.apache.lucene.core;
     requires org.objectweb.asm;
     requires org.objectweb.asm.commons;