Merge remote-tracking branch 'upstream/main' into offset-numeric-arrays-index-version-fix

Author: jordan-powers

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -61,7 +61,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-plugins");
         map.put(LegacyRestTestBasePlugin.class, ":qa:system-indices");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:verify-version-constants");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-die-with-dignity");

docs/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.doc.DocSnippetTask
 import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT
 
@@ -2059,3 +2060,29 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
     setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events)
   }
 }
+
+tasks.register('verifyDocsLuceneVersion') {
+  doFirst {
+    File docsVersionsFile = file('Versions.asciidoc')
+    List<String> versionLines = docsVersionsFile.readLines('UTF-8')
+    String docsLuceneVersion = null
+    for (String line : versionLines) {
+      if (line.startsWith(':lucene_version:')) {
+        docsLuceneVersion = line.split()[1]
+      }
+    }
+    if (docsLuceneVersion == null) {
+      throw new GradleException('Could not find lucene version in docs version file')
+    }
+    String expectedLuceneVersion = VersionProperties.lucene
+    // remove potential -snapshot-{gitrev} suffix
+    expectedLuceneVersion -= ~/-snapshot-[0-9a-f]+$/
+    if (docsLuceneVersion != expectedLuceneVersion) {
+      throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]")
+    }
+  }
+}
+
+tasks.named('check') {
+  dependsOn 'verifyDocsLuceneVersion'
+}

docs/changelog/125239.yaml

@@ -0,0 +1,6 @@
+pr: 125239
+summary: Adding common rerank options to Perform Inference API
+area: Machine Learning
+type: enhancement
+issues:
+ - 111273

qa/verify-version-constants/build.gradle

@@ -7,57 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
-dependencies {
-  testImplementation project(':modules:rest-root')
-}
-
 buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
-  def baseCluster = testClusters.register(baseName) {
-      version = bwcVersion.toString()
-      setting 'xpack.security.enabled', 'true'
-      user username: 'admin', password: 'admin-password', role: 'superuser'
-  }
-
-  tasks.register("${baseName}#integTest", StandaloneRestIntegTestTask) {
-    useCluster baseCluster
-    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    nonInputProperties.systemProperty('tests.clustername', "${->baseCluster.get().getName()}")
-  }
-
-  tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn "${baseName}#integTest"
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty 'tests.cluster_version', bwcVersion
   }
 }
-
-tasks.register("verifyDocsLuceneVersion") {
-  doFirst {
-    File docsVersionsFile = layout.settingsDirectory.file('docs/Versions.asciidoc').asFile
-    List<String> versionLines = docsVersionsFile.readLines('UTF-8')
-    String docsLuceneVersion = null
-    for (String line : versionLines) {
-      if (line.startsWith(':lucene_version:')) {
-        docsLuceneVersion = line.split()[1]
-      }
-    }
-    if (docsLuceneVersion == null) {
-      throw new GradleException('Could not find lucene version in docs version file')
-    }
-    String expectedLuceneVersion = VersionProperties.lucene
-    // remove potential -snapshot-{gitrev} suffix
-    expectedLuceneVersion -= ~/-snapshot-[0-9a-f]+$/
-    if (docsLuceneVersion != expectedLuceneVersion) {
-      throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]")
-    }
-  }
-}
-
-tasks.named("check").configure {
-  dependsOn "verifyDocsLuceneVersion"
-}

qa/verify-version-constants/src/javaRestTest/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.qa.verify_version_constants;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class VerifyVersionConstantsIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(System.getProperty("tests.cluster_version"))
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    public void testLuceneVersionConstant() throws IOException, ParseException {
+        Response response = client().performRequest(new Request("GET", "/"));
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+
+        String luceneVersionString = objectPath.evaluate("version.lucene_version").toString();
+        org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString);
+
+        IndexVersion indexVersion = getIndexVersion();
+        assertThat(indexVersion.luceneVersion(), equalTo(luceneVersion));
+    }
+
+    private IndexVersion getIndexVersion() throws IOException {
+        IndexVersion indexVersion = null;
+
+        Request request = new Request("GET", "_nodes");
+        request.addParameter("filter_path", "nodes.*.index_version,nodes.*.name");
+        Response response = client().performRequest(request);
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        Map<String, Object> nodeMap = objectPath.evaluate("nodes");
+        for (String id : nodeMap.keySet()) {
+            Number ix = objectPath.evaluate("nodes." + id + ".index_version");
+            IndexVersion version;
+            if (ix != null) {
+                version = IndexVersion.fromId(ix.intValue());
+            } else {
+                // it doesn't have index version (pre 8.11) - just infer it from the release version
+                version = parseLegacyVersion(System.getProperty("tests.cluster_version")).map(x -> IndexVersion.fromId(x.id()))
+                    .orElse(IndexVersions.MINIMUM_COMPATIBLE);
+            }
+
+            if (indexVersion == null) {
+                indexVersion = version;
+            } else {
+                String name = objectPath.evaluate("nodes." + id + ".name");
+                assertThat("Node " + name + " has a different index version to other nodes", version, Matchers.equalTo(indexVersion));
+            }
+        }
+
+        assertThat("Index version could not be read", indexVersion, notNullValue());
+        return indexVersion;
+    }
+
+    @Override
+    public boolean preserveClusterUponCompletion() {
+        /*
+         * We don't perform any writes to the cluster so there won't be anything
+         * to clean up. Also, our cleanup code is really only compatible with
+         * *write* compatible versions but this runs with *index* compatible
+         * versions.
+         */
+        return true;
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java

@@ -155,6 +155,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_AGGREGATE_METRIC_DOUBLE_LITERAL_8_19 = def(8_841_0_12);
     public static final TransportVersion INFERENCE_MODEL_REGISTRY_METADATA_8_19 = def(8_841_0_13);
     public static final TransportVersion INTRODUCE_LIFECYCLE_TEMPLATE_8_19 = def(8_841_0_14);
+    public static final TransportVersion RERANK_COMMON_OPTIONS_ADDED_8_19 = def(8_841_0_15);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0 = def(9_000_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01);
     public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02);
@@ -201,6 +202,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INDEXING_STATS_INCLUDES_RECENT_WRITE_LOAD = def(9_034_0_00);
     public static final TransportVersion ESQL_AGGREGATE_METRIC_DOUBLE_LITERAL = def(9_035_0_00);
     public static final TransportVersion INDEX_METADATA_INCLUDES_RECENT_WRITE_LOAD = def(9_036_0_00);
+    public static final TransportVersion RERANK_COMMON_OPTIONS_ADDED = def(9_037_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -85,6 +85,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion V_8_3_0 = def(8_03_00_99, Version.LUCENE_9_2_0);
     public static final IndexVersion V_8_4_0 = def(8_04_00_99, Version.LUCENE_9_3_0);
     public static final IndexVersion V_8_5_0 = def(8_05_00_99, Version.LUCENE_9_4_1);
+    public static final IndexVersion V_8_5_3 = def(8_05_03_99, Version.LUCENE_9_4_2);
     public static final IndexVersion V_8_6_0 = def(8_06_00_99, Version.LUCENE_9_4_2);
     public static final IndexVersion V_8_7_0 = def(8_07_00_99, Version.LUCENE_9_5_0);
     public static final IndexVersion V_8_8_0 = def(8_08_00_99, Version.LUCENE_9_6_0);

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -91,18 +91,22 @@ default boolean hideFromConfigurationApi() {
     /**
      * Perform inference on the model.
      *
-     * @param model        The model
-     * @param query        Inference query, mainly for re-ranking
-     * @param input        Inference input
-     * @param stream       Stream inference results
-     * @param taskSettings Settings in the request to override the model's defaults
-     * @param inputType    For search, ingest etc
-     * @param timeout      The timeout for the request
-     * @param listener     Inference result listener
+     * @param model           The model
+     * @param query           Inference query, mainly for re-ranking
+     * @param returnDocuments For re-ranking task type, whether to return documents
+     * @param topN            For re-ranking task type, how many docs to return
+     * @param input           Inference input
+     * @param stream          Stream inference results
+     * @param taskSettings    Settings in the request to override the model's defaults
+     * @param inputType       For search, ingest etc
+     * @param timeout         The timeout for the request
+     * @param listener        Inference result listener
      */
     void infer(
         Model model,
         @Nullable String query,
+        @Nullable Boolean returnDocuments,
+        @Nullable Integer topN,
         List<String> input,
         boolean stream,
         Map<String, Object> taskSettings,

server/src/main/java/org/elasticsearch/inference/InferenceService.java

@@ -288,6 +288,12 @@ public T version(Version version) {
         return cast(this);
     }
 
+    @Override
+    public T version(String version) {
+        this.version = Version.fromString(version);
+        return cast(this);
+    }
+
     public Version getVersion() {
         return inherit(() -> parent.getVersion(), version);
     }