From 4844d5dee713cba7997a23ec80d4950e809c5d85 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 8 Oct 2025 12:11:42 +0100 Subject: [PATCH 1/5] [ML] Fix and unmute Transform upgrade test (#132995) (cherry picked from commit 208ae578e089b6f40ba4339cab883d72631e40eb) # Conflicts: # muted-tests.yml # x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java --- .../TransportUpgradeTransformsAction.java | 3 +-- .../IndexBasedTransformConfigManager.java | 19 +++++++++++-------- .../upgrades/MLModelDeploymentsUpgradeIT.java | 4 ++-- .../upgrades/TransformSurvivesUpgradeIT.java | 17 ++++++++++------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java index b91cf6d3bb5b6..f8d6c399f9fda 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java @@ -29,7 +29,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.ml.utils.TransportVersionUtils; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.transform.TransformMetadata; import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction; @@ -107,7 +106,7 @@ protected void masterOperation(Task ignoredTask, Request request, ClusterState s } // do not allow in mixed clusters - if (TransportVersionUtils.isMinTransportVersionSameAsCurrent(state) == false) { + if (state.nodes().isMixedVersionCluster()) { listener.onFailure( new ElasticsearchStatusException("Cannot upgrade transforms while cluster upgrade is in progress.", RestStatus.CONFLICT) ); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 25630fb24772a..0393a98c9263d 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -295,16 +295,19 @@ public void deleteOldIndices(ActionListener listener) { // use the transform context as we access system indexes try (ThreadContext.StoredContext ctx = client.threadPool().getThreadContext().stashWithOrigin(TRANSFORM_ORIGIN)) { - indicesToDelete.addAll( - Arrays.asList( - indexNameExpressionResolver.concreteIndexNames( - state, - IndicesOptions.lenientExpandHidden(), - TransformInternalIndexConstants.INDEX_NAME_PATTERN - ) - ) + var matchingIndexes = indexNameExpressionResolver.concreteIndices( + state, + IndicesOptions.lenientExpandHidden(), + TransformInternalIndexConstants.INDEX_NAME_PATTERN ); + for (var index : matchingIndexes) { + var meta = state.getMetadata().indexMetadata(index); + if (meta.isSystem() == false) { // ignore system indices as these are automatically managed + indicesToDelete.add(meta.getIndex().getName()); + } + } + indicesToDelete.addAll( Arrays.asList( indexNameExpressionResolver.concreteIndexNames( diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java index 83ea9071eca90..7f486266c1a7f 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java @@ -72,10 +72,10 @@ public void setUpLogging() throws IOException { request.setJsonEntity(""" { "persistent": { - "logger.org.elasticsearch.xpack.ml.inference": "TRACE", + "logger.org.elasticsearch.xpack.ml.inference": "DEBUG", "logger.org.elasticsearch.xpack.ml.inference.assignments": "DEBUG", "logger.org.elasticsearch.xpack.ml.process": "DEBUG", - "logger.org.elasticsearch.xpack.ml.action": "TRACE" + "logger.org.elasticsearch.xpack.ml.action": "DEBUG" } } """); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 87a9911bd80b6..2e98ebc57cc0f 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -9,6 +9,7 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -131,11 +132,11 @@ private void createAndStartContinuousTransform() throws Exception { assertBusy(() -> { var stateAndStats = getTransformStats(CONTINUOUS_TRANSFORM_ID); + assertThat((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats), equalTo(ENTITIES.size())); assertThat( - ((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats)).longValue(), - equalTo(ENTITIES.size()) + ((Integer) XContentMapValues.extractValue("stats.documents_processed", stateAndStats)).longValue(), + equalTo(totalDocsWritten) ); - assertThat((Integer) XContentMapValues.extractValue("stats.documents_processed", stateAndStats), equalTo(totalDocsWritten)); // Even if we get back to started, we may periodically get set back to `indexing` when triggered. // Though short lived due to no changes on the source indices, it could result in flaky test behavior assertThat(stateAndStats.get("state"), oneOf("started", "indexing")); @@ -237,10 +238,12 @@ private void verifyUpgradeFailsIfMixedCluster() { if (isOriginalClusterCurrent()) { return; } - final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); - - Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); - assertThat(ex.getMessage(), containsString("All nodes must be the same version")); + var oldestVersion = Version.fromString(UPGRADE_FROM_VERSION); + if (oldestVersion.onOrAfter(Version.V_9_3_0)) { + final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); + Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); + assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); + } } private void verifyUpgrade() throws IOException { From 1285a8634119290d37156eb9fbfa5b6f98a34017 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 10 Oct 2025 13:28:48 +0100 Subject: [PATCH 2/5] Update TransformSurvivesUpgradeIT.java --- .../upgrades/TransformSurvivesUpgradeIT.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 2e98ebc57cc0f..319db79c68433 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -97,7 +97,6 @@ public void testTransformRollingUpgrade() throws Exception { lastCheckpoint = 2; } verifyContinuousTransformHandlesData(lastCheckpoint); - verifyUpgradeFailsIfMixedCluster(); } case UPGRADED -> { client().performRequest(waitForYellow); @@ -233,19 +232,6 @@ private void verifyContinuousTransformHandlesData(long expectedLastCheckpoint) t }); } - private void verifyUpgradeFailsIfMixedCluster() { - // upgrade tests by design are also executed with the same version, this check must be skipped in this case, see gh#39102. - if (isOriginalClusterCurrent()) { - return; - } - var oldestVersion = Version.fromString(UPGRADE_FROM_VERSION); - if (oldestVersion.onOrAfter(Version.V_9_3_0)) { - final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); - Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); - assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); - } - } - private void verifyUpgrade() throws IOException { final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); Response response = client().performRequest(upgradeTransformRequest); From 5c641c8828235ac890baaa3724487f52771917ef Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 10 Oct 2025 12:35:51 +0000 Subject: [PATCH 3/5] [CI] Auto commit changes from spotless --- .../org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 319db79c68433..ab26223bce387 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -9,7 +9,6 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -36,7 +35,6 @@ import static org.elasticsearch.xpack.test.rest.XPackRestTestConstants.TRANSFORM_INTERNAL_INDEX_PREFIX; import static org.elasticsearch.xpack.test.rest.XPackRestTestConstants.TRANSFORM_INTERNAL_INDEX_PREFIX_DEPRECATED; import static org.elasticsearch.xpack.test.rest.XPackRestTestConstants.TRANSFORM_TASK_NAME; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; From 1d5c9a5c1910727fbd334e3efeac15884dba9c43 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 10 Oct 2025 14:05:19 +0100 Subject: [PATCH 4/5] fix --- .../transform/persistence/IndexBasedTransformConfigManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 0393a98c9263d..360ef28c9815c 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -302,7 +302,7 @@ public void deleteOldIndices(ActionListener listener) { ); for (var index : matchingIndexes) { - var meta = state.getMetadata().indexMetadata(index); + var meta = state.getMetadata().getIndexSafe(index); if (meta.isSystem() == false) { // ignore system indices as these are automatically managed indicesToDelete.add(meta.getIndex().getName()); } From d75a689a6aa9f7f2addfaa12db47ea59f53e5131 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 10 Oct 2025 14:25:52 +0100 Subject: [PATCH 5/5] f --- .../transform/action/TransportUpgradeTransformsAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java index f8d6c399f9fda..b91cf6d3bb5b6 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.ml.utils.TransportVersionUtils; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.transform.TransformMetadata; import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction; @@ -106,7 +107,7 @@ protected void masterOperation(Task ignoredTask, Request request, ClusterState s } // do not allow in mixed clusters - if (state.nodes().isMixedVersionCluster()) { + if (TransportVersionUtils.isMinTransportVersionSameAsCurrent(state) == false) { listener.onFailure( new ElasticsearchStatusException("Cannot upgrade transforms while cluster upgrade is in progress.", RestStatus.CONFLICT) );