Merge branch 'main' into tasks/rcs-rm-isCrossClusterSearchEnabled

Author: JeremyDahlgren

gradle/verification-metadata.xml

@@ -2150,24 +2150,24 @@
             <sha256 value="5c8551990307a032336d98ddaed549a39a689f07d4d4c6b950601bf22b3d6a1b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-format" version="16.1.0">
-         <artifact name="arrow-format-16.1.0.jar">
-            <sha256 value="ad97e0fc72e193b1de3cbce4818d1ff16e81673fd523d001e8d2774bde40ee6c" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-format" version="18.3.0">
+         <artifact name="arrow-format-18.3.0.jar">
+            <sha256 value="728297d7757d192053ce071cc59e76c8a03ff4f5e430177fa97e96b29dd2de1b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-memory-core" version="16.1.0">
-         <artifact name="arrow-memory-core-16.1.0.jar">
-            <sha256 value="da7af1a1a899bd5a1b6c71284243b9f3c0e1098f0cb10cd7be4b8b455ced79dd" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-memory-core" version="18.3.0">
+         <artifact name="arrow-memory-core-18.3.0.jar">
+            <sha256 value="32ed0719bf2ba42becc3a88c95722851cf1bb88b37d06c95654a1152bed6ef2e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-memory-unsafe" version="16.1.0">
-         <artifact name="arrow-memory-unsafe-16.1.0.jar">
-            <sha256 value="6534eded25f2c30593416a294c1047f0b017baa9906d98f6f3270737b076c745" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-memory-unsafe" version="18.3.0">
+         <artifact name="arrow-memory-unsafe-18.3.0.jar">
+            <sha256 value="e9652f322d7be306c400ea0d775b3114a11b48d2afe23e8b8566f13c469d5483" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-vector" version="16.1.0">
-         <artifact name="arrow-vector-16.1.0.jar">
-            <sha256 value="c5837b3aa24dfd93759f57bc5759b9a8fbb5bf3912d55994d70cabb904436aab" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-vector" version="18.3.0">
+         <artifact name="arrow-vector-18.3.0.jar">
+            <sha256 value="b37eda92daccaffc12abf5ed425db1d0bdb3edea150a6ca856f1cf4292442299" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.avro" name="avro" version="1.7.4">

test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java

@@ -894,7 +894,6 @@ private void initGiantTextField(int docs) throws IOException {
                 }
             }
         }
-
         logger.info("loading many documents with one big text field - docs per bulk {}", docsPerBulk);
 
         int fieldSize = Math.toIntExact(ByteSizeValue.ofMb(5).getBytes());
@@ -1064,6 +1063,15 @@ private void bulk(String name, String bulk) throws IOException {
         );
         Response response = client().performRequest(request);
         assertThat(entityAsMap(response), matchesMap().entry("errors", false).extraOk());
+
+        /*
+         * Flush after each bulk to clear the test-time seenSequenceNumbers Map in
+         * TranslogWriter. Without this the server will OOM from time to time keeping
+         * stuff around to run assertions on.
+         */
+        request = new Request("POST", "/" + name + "/_flush");
+        response = client().performRequest(request);
+        assertThat(entityAsMap(response), matchesMap().entry("_shards", matchesMap().extraOk().entry("failed", 0)).extraOk());
     }
 
     private void initIndex(String name, String bulk) throws IOException {

x-pack/plugin/esql/arrow/build.gradle

@@ -12,9 +12,9 @@ dependencies {
   compileOnly project(':x-pack:plugin:esql:compute')
   compileOnly project(':x-pack:plugin:esql-core')
   compileOnly project(':x-pack:plugin:mapper-version')
-  implementation('org.apache.arrow:arrow-vector:16.1.0')
-  implementation('org.apache.arrow:arrow-format:16.1.0')
-  implementation('org.apache.arrow:arrow-memory-core:16.1.0')
+  implementation('org.apache.arrow:arrow-vector:18.3.0')
+  implementation('org.apache.arrow:arrow-format:18.3.0')
+  implementation('org.apache.arrow:arrow-memory-core:18.3.0')
   implementation('org.checkerframework:checker-qual:3.42.0')
   implementation('com.google.flatbuffers:flatbuffers-java:23.5.26')
   // Needed for the json arrow serialization, and loaded even if we don't use it.
@@ -25,7 +25,7 @@ dependencies {
   runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
 
   testImplementation project(':test:framework')
-  testImplementation('org.apache.arrow:arrow-memory-unsafe:16.1.0')
+  testImplementation('org.apache.arrow:arrow-memory-unsafe:18.3.0')
   testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}")
 }
 
@@ -38,18 +38,8 @@ tasks.named("dependencyLicenses").configure {
 tasks.named("thirdPartyAudit").configure {
   ignoreViolations(
     // uses sun.misc.Unsafe. Only used in tests.
-    'org.apache.arrow.memory.util.hash.SimpleHasher',
-    'org.apache.arrow.memory.util.hash.MurmurHasher',
     'org.apache.arrow.memory.util.MemoryUtil',
     'org.apache.arrow.memory.util.MemoryUtil$1',
-    'org.apache.arrow.vector.DecimalVector',
-    'org.apache.arrow.vector.BaseFixedWidthVector',
-    'org.apache.arrow.vector.util.DecimalUtility',
-    'org.apache.arrow.vector.Decimal256Vector',
-    'org.apache.arrow.vector.util.VectorAppender',
-    'org.apache.arrow.memory.ArrowBuf',
-    'org.apache.arrow.vector.BitVectorHelper',
-    'org.apache.arrow.memory.util.ByteFunctionHelpers',
   )
   ignoreMissingClasses(
     'org.apache.commons.codec.binary.Hex'

x-pack/plugin/esql/qa/server/single-node/build.gradle

@@ -16,17 +16,17 @@ dependencies {
   javaRestTestImplementation project(xpackModule('esql'))
   yamlRestTestImplementation project(xpackModule('esql:qa:server'))
 
-  javaRestTestImplementation('org.apache.arrow:arrow-vector:16.1.0')
-  javaRestTestImplementation('org.apache.arrow:arrow-format:16.1.0')
-  javaRestTestImplementation('org.apache.arrow:arrow-memory-core:16.1.0')
+  javaRestTestImplementation('org.apache.arrow:arrow-vector:18.3.0')
+  javaRestTestImplementation('org.apache.arrow:arrow-format:18.3.0')
+  javaRestTestImplementation('org.apache.arrow:arrow-memory-core:18.3.0')
   javaRestTestImplementation('org.checkerframework:checker-qual:3.42.0')
   javaRestTestImplementation('com.google.flatbuffers:flatbuffers-java:23.5.26')
   javaRestTestImplementation("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   javaRestTestImplementation("com.fasterxml.jackson.core:jackson-core:${versions.jackson}")
   javaRestTestImplementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
   javaRestTestImplementation("org.slf4j:slf4j-api:${versions.slf4j}")
   javaRestTestImplementation("org.slf4j:slf4j-nop:${versions.slf4j}")
-  javaRestTestImplementation('org.apache.arrow:arrow-memory-unsafe:16.1.0')
+  javaRestTestImplementation('org.apache.arrow:arrow-memory-unsafe:18.3.0')
 
   clusterPlugins project(':plugins:mapper-size')
   clusterPlugins project(':plugins:mapper-murmur3')