Add rolling upgrade and mixed cluster tests for system indices/inference API

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java

@@ -11,12 +11,17 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
+import org.apache.http.HttpHost;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.test.XContentTestUtils.JsonMapView;
 import org.elasticsearch.test.rest.RestTestLegacyFeatures;
 
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_INDEX_VERSION;
@@ -30,6 +35,38 @@ public SystemIndicesUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
         super(upgradedNodes);
     }
 
+    public void testSystemIndicesDescriptorsAppearInUpgradedCluster() throws Exception {
+        if (isUpgradedCluster()) {
+            // TODO/BUG? If we do not restart the cluster, only the last upgraded node has non-empty mappings_versions.
+            // The (2) nodes that upgraded first have empty mappings_versions, until we restart them.
+            getUpgradeCluster().restart(false);
+
+            try (
+                RestClient newNodeClient = buildClient(
+                    restClientSettings(),
+                    parseClusterHosts(getUpgradeCluster().getHttpAddresses()).toArray(HttpHost[]::new)
+                )
+            ) {
+
+                final Request request = new Request("GET", "_cluster/state");
+                final Response response = newNodeClient.performRequest(request);
+
+                var responseData = responseAsMap(response);
+
+                var nodesVersions = (List<?>) responseData.get("nodes_versions");
+                var nodeToMappings = new HashMap<String, Map<?, ?>>();
+                for (int i = 0; i < nodesVersions.size(); ++i) {
+                    var entry = (Map<?, ?>) nodesVersions.get(i);
+                    nodeToMappings.put(entry.get("node_id").toString(), (Map<?, ?>) entry.get("mappings_versions"));
+                }
+                var masterNode = responseData.get("master_node").toString();
+                var masterNodeMappings = nodeToMappings.get(masterNode);
+
+                assertFalse(masterNodeMappings.isEmpty());
+            }
+        }
+    }
+
     @SuppressWarnings("unchecked")
     public void testSystemIndicesUpgrades() throws Exception {
         final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct "

test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java

@@ -137,6 +137,9 @@ public class RestTestLegacyFeatures implements FeatureSpecification {
     // YAML
     public static final NodeFeature REST_ELASTIC_PRODUCT_HEADER_PRESENT = new NodeFeature("action.rest.product_header_present");
 
+    // Inference
+    public static final NodeFeature OPEN_AI_EMBEDDINGS_ADDED = new NodeFeature("inference.open_ai_embeddings_added");
+
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
         return Map.ofEntries(
@@ -184,7 +187,8 @@ public Map<NodeFeature, Version> getHistoricalFeatures() {
             entry(NEW_DATA_STREAMS_INDEX_NAME_FORMAT, Version.V_7_11_0),
             entry(DISABLE_FIELD_NAMES_FIELD_REMOVED, Version.V_8_0_0),
             entry(ML_NLP_SUPPORTED, Version.V_8_0_0),
-            entry(MAPPINGS_UPGRADE_SERVICE_USES_MAPPINGS_VERSION, Version.V_8_11_0)
+            entry(MAPPINGS_UPGRADE_SERVICE_USES_MAPPINGS_VERSION, Version.V_8_11_0),
+            entry(OPEN_AI_EMBEDDINGS_ADDED, Version.V_8_12_0)
         );
     }
 }

x-pack/plugin/inference/qa/mixed-cluster/build.gradle

@@ -18,9 +18,10 @@ dependencies {
   )
 }
 
-// inference is available in 8.11 or later
+// Inference API added in 8.11, but we generate BwC as far back as 8.9.0 to test for mixed clusters
+// with nodes where index descriptors were non-existing/used the older Version-based versioning
 def supportedVersion = bwcVersion -> {
-  return bwcVersion.onOrAfter(Version.fromString("8.11.0"));
+  return bwcVersion.onOrAfter(Version.fromString("8.9.0"));
 }
 
 BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName ->

x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/OpenAIServiceMixedIT.java

@@ -8,22 +8,26 @@
 package org.elasticsearch.xpack.inference.qa.mixed;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.test.rest.RestTestLegacyFeatures;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.qa.mixed.MixedClusterSpecTestCase.bwcVersion;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
 
 public class OpenAIServiceMixedIT extends BaseMixedTestCase {
 
@@ -110,6 +114,24 @@ public void testOpenAiCompletions() throws IOException {
         assertCompletionInference(inferenceId);
     }
 
+    public void testMixedClusterNotFullySupportingOpenAiCompletionsRejectsApiCalls() throws IOException {
+        assumeFalse(
+            "Old nodes must be before support for OpenAi inference was added",
+            oldClusterHasFeature(RestTestLegacyFeatures.OPEN_AI_EMBEDDINGS_ADDED)
+        );
+        final String inferenceId = "mixed-cluster-completions";
+
+        var modelConfig = chatCompletionsConfig(getUrl(openAiChatCompletionsServer));
+
+        String endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.COMPLETION, inferenceId);
+        var request = new Request("PUT", endpoint);
+        request.setJsonEntity(modelConfig);
+        var responseException = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        var response = responseException.getResponse();
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(400));
+        assertThat(entityAsMap(response).get("error").toString(), startsWith("no handler found for uri [_inference/completion/"));
+    }
+
     void assertCompletionInference(String inferenceId) throws IOException {
         openAiChatCompletionsServer.enqueue(new MockResponse().setResponseCode(200).setBody(chatCompletionsResponse()));
         var inferenceMap = inference(inferenceId, TaskType.COMPLETION, "some text");

x-pack/plugin/inference/qa/rolling-upgrade/build.gradle

@@ -22,8 +22,9 @@ dependencies {
   javaRestTestImplementation(testArtifact(project(":qa:rolling-upgrade"), "javaRestTest"))
 }
 
-// Inference API added in 8.11
-BuildParams.bwcVersions.withWireCompatible(v -> v.after("8.11.0")) { bwcVersion, baseName ->
+// Inference API added in 8.11, but we generate BwC as far back as 8.9.0 to test for upgrades
+// from a cluster where system index descriptors were non-existing/used the older Version-based versioning
+BuildParams.bwcVersions.withWireCompatible(v -> v.after("8.9.0")) { bwcVersion, baseName ->
   tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
     usesBwcDistribution(bwcVersion)
     systemProperty("tests.old_cluster_version", bwcVersion)

x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.test.rest.RestTestLegacyFeatures;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -110,6 +111,30 @@ public void testOpenAiEmbeddings() throws IOException {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public void testOpenAiInferenceCreateAfterUpgradeFromNonSupportedVersion() throws IOException {
+        assumeFalse(
+            "Old cluster must be before a supported version",
+            oldClusterHasFeature(RestTestLegacyFeatures.OPEN_AI_EMBEDDINGS_ADDED)
+        );
+
+        final String upgradedClusterId = "upgraded-cluster-embeddings";
+        var testTaskType = TaskType.TEXT_EMBEDDING;
+
+        if (isUpgradedCluster()) {
+            String inferenceConfig = embeddingConfigWithModelInServiceSettings(getUrl(openAiEmbeddingsServer));
+            openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
+            put(upgradedClusterId, inferenceConfig, testTaskType);
+
+            var configs = (List<Map<String, Object>>) get(testTaskType, upgradedClusterId).get("endpoints");
+            assertThat(configs, hasSize(1));
+
+            assertEmbeddingInference(upgradedClusterId);
+
+            delete(upgradedClusterId);
+        }
+    }
+
     void assertEmbeddingInference(String inferenceId) throws IOException {
         openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
         var inferenceMap = inference(inferenceId, TaskType.TEXT_EMBEDDING, "some text");