Merge branch 'main' into feature/esql-knn-function-minimal-support

Author: carlosdelest

docs/changelog/127613.yaml

@@ -0,0 +1,5 @@
+pr: 127613
+summary: Threadpool merge executor is aware of available disk space
+area: Engine
+type: feature
+issues: []

muted-tests.yml

@@ -498,6 +498,21 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test073RunEsAsDifferentUserAndGroupWithoutBindMounting
   issue: https://github.com/elastic/elasticsearch/issues/128996
+- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
+  method: test {lookup-join.LookupJoinOnTimeSeriesIndex ASYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/129078
+- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
+  method: test {lookup-join.LookupJoinOnTimeSeriesIndex SYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/129082
+- class: org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT
+  method: test {p0=upgraded_cluster/70_ilm/Test Lifecycle Still There And Indices Are Still Managed}
+  issue: https://github.com/elastic/elasticsearch/issues/129097
+- class: org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT
+  method: test {p0=upgraded_cluster/90_ml_data_frame_analytics_crud/Get mixed cluster outlier_detection job}
+  issue: https://github.com/elastic/elasticsearch/issues/129098
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test081SymlinksAreFollowedWithEnvironmentVariableFiles
+  issue: https://github.com/elastic/elasticsearch/issues/128867
 
 # Examples:
 #

qa/vector/build.gradle

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+apply plugin: 'elasticsearch.java'
+apply plugin: 'elasticsearch.build'
+
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /lucene-.*/, to: 'lucene'
+}
+
+tasks.named('forbiddenApisMain').configure {
+  enabled = false
+}
+
+dependencies {
+  api "org.apache.lucene:lucene-core:${versions.lucene}"
+  api "org.apache.lucene:lucene-queries:${versions.lucene}"
+  api "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  implementation project(':libs:logging')
+  implementation project(':server')
+}
+/**
+ * Task to run the KnnIndexTester with the provided parameters.
+ */
+tasks.register("checkVec", JavaExec) {
+  group = "Execution"
+  description = "Runs KnnIndexTester with the provided parameters to validate recall and performance."
+  classpath = sourceSets.main.runtimeClasspath
+  mainClass.set("org.elasticsearch.test.knn.KnnIndexTester")
+  // Configure logging to console
+  systemProperty "es.logger.out", "console"
+  systemProperty "es.logger.level", "INFO"  // Change to DEBUG if needed
+
+  if (buildParams.getRuntimeJavaVersion().map { it.majorVersion.toInteger() }.get() >= 21) {
+    jvmArgs '-Xms4g', '-Xmx4g', '--add-modules=jdk.incubator.vector', '--enable-native-access=ALL-UNNAMED', '-Djava.util.concurrent.ForkJoinPool.common.parallelism=8', '-XX:+UnlockDiagnosticVMOptions', '-XX:+DebugNonSafepoints', '-XX:+HeapDumpOnOutOfMemoryError'
+  }
+}
+
+tasks.register("checkVecHelp", JavaExec) {
+  group = "Help"
+  description = "Prints help for the KnnIndexTester task."
+  classpath = sourceSets.main.runtimeClasspath
+  mainClass.set("org.elasticsearch.test.knn.KnnIndexTester")
+  args = ["--help"]
+  doLast {
+    println """
+    =============================================================================
+    KnnIndexTester Help
+    =============================================================================
+
+    Run with Gradle:
+    ----------------
+    # Using default configuration file
+    ./gradlew :qa:vector:checkVec
+
+    # Using custom configuration file
+    ./gradlew :qa:vector:checkVec --args="path/to/your/config.json"
+
+    # Adjust heap size
+    ./gradlew :qa:vector:checkVec -Dorg.gradle.jvmargs="-Xmx8g" --args="path/to/your/config.json"
+
+    # Set environment variable for more extensive JVM options
+    export GRADLE_OPTS="-Xmx8g -XX:+UseG1GC -XX:MaxGCPauseMillis=100"
+    ./gradlew :qa:vector:checkVec
+
+
+    Run directly with Java:
+    ----------------------
+    # Generate classpath (run once to create the file)
+    ./gradlew :qa:vector:printClasspath
+
+    # Then use the classpath file with java
+    java -cp "\$(cat build/vector_classpath.txt)" \\
+         --add-modules=jdk.incubator.vector \\
+         --enable-native-access=ALL-UNNAMED \\
+         -Djava.util.concurrent.ForkJoinPool.common.parallelism=8 \\
+         -Xmx4g \\
+         -Xms4g \\\\
+         org.elasticsearch.test.knn.KnnIndexTester path/to/your/config.json
+    """
+  }
+}
+
+tasks.register("printClasspath") {
+  group = "Help"
+  description = "Prints the classpath needed to run KnnIndexTester directly with java"
+
+  doLast {
+    def classpathFile = new File("${buildDir}/vector_classpath.txt")
+    classpathFile.parentFile.mkdirs()
+    classpathFile.text = sourceSets.main.runtimeClasspath.asPath
+    println "Classpath written to: ${classpathFile.absolutePath}"
+  }
+}