wip testing

.buildkite/pipelines/pull-request/pr-upgrade.yml

@@ -0,0 +1,6 @@
+steps:
+  - label: pr-upgrade
+    command: ".buildkite/scripts/run-pr-upgrade-tests.sh"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      memory: "4G"

.buildkite/scripts/run-pr-upgrade-tests.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+set -euo pipefail
+
+if [[ -z "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]]; then
+   echo "Not a pull request, skipping PR upgrade tests."
+   exit 0
+fi
+
+# Identify the merge base of the current commit (branch) and the base branch of the pull request.
+# PR upgrade tests are run from the merge base to the current commit.
+BASE_COMMIT=$(git merge-base $BUILDKITE_PULL_REQUEST_BASE_BRANCH $BUILDKITE_COMMIT)
+
+VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
+
+echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
+
+cat <<EOF | buildkite-agent pipeline upload
+steps:
+    - label: pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH
+      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
+      timeout_in_minutes: 300
+      agents:
+        provider: gcp
+        image: family/elasticsearch-ubuntu-2004
+        machineType: n1-standard-32
+        buildDirectory: /dev/shm/bk
+        preemptible: true
+      retry:
+        automatic:
+          - exit_status: "-1"
+            limit: 3
+            signal_reason: none
+          - signal_reason: agent_stop
+            limit: 3
+EOF

build-tools-internal/src/main/groovy/elasticsearch.bc-upgrade-test.gradle

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+tasks.register("bcUpgradeTest", StandaloneRestIntegTestTask) {
+  // We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
+  usesBwcDistribution(Version.fromString("0.0.0"))
+  systemProperty("tests.old_cluster_version", "0.0.0")
+  onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
+}

modules/ingest-geoip/qa/full-cluster-restart/build.gradle

@@ -7,11 +7,12 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
-
+apply plugin: 'elasticsearch.bc-upgrade-test'
 
 dependencies {
   javaRestTestImplementation project(':test:fixtures:geoip-fixture')

qa/full-cluster-restart/build.gradle

@@ -13,6 +13,7 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
 apply plugin: 'elasticsearch.bwc-test'
+apply plugin: 'elasticsearch.bc-upgrade-test'
 
 buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
   tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java

@@ -134,7 +134,11 @@ public boolean isRunningAgainstOldCluster() {
         return requestedUpgradeStatus == OLD;
     }
 
-    public static String getOldClusterVersion() {
+    /**
+     * The version of the "old" (initial) cluster. It is an opaque string, do not even think about parsing it for version
+     * comparison. Use (test) cluster features and {@link ParameterizedFullClusterRestartTestCase#oldClusterHasFeature} instead.
+     */
+    protected static String getOldClusterVersion() {
         return System.getProperty("tests.bwc.main.version", OLD_CLUSTER_VERSION);
     }
 

qa/rolling-upgrade/build.gradle

@@ -14,6 +14,7 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-test-artifact-base'
 apply plugin: 'elasticsearch.bwc-test'
 apply plugin: 'elasticsearch.fwc-test'
+apply plugin: 'elasticsearch.bc-upgrade-test'
 
 testArtifacts {
   registerTestArtifactFromSourceSet(sourceSets.javaRestTest)
@@ -26,18 +27,6 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
   }
 }
 
-tasks.register("bcUpgradeTest", StandaloneRestIntegTestTask) {
-  // We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
-  usesBwcDistribution(Version.fromString("0.0.0"))
-  systemProperty("tests.old_cluster_version", "0.0.0")
-  onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
-  filter {
-    // filter tests initially for quicker iterations
-    // TODO remove once expanding the test set to other modules
-    includeTestsMatching("org.elasticsearch.upgrades.IndexingIT")
-  }
-}
-
 tasks.withType(Test).configureEach {
   // CI doesn't like it when there's multiple clusters running at once
   maxParallelForks = 1

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java

@@ -30,10 +30,9 @@ public abstract class AbstractRollingUpgradeTestCase extends ParameterizedRollin
     private static final ElasticsearchCluster cluster = buildCluster();
 
     private static ElasticsearchCluster buildCluster() {
-        Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION);
         var cluster = ElasticsearchCluster.local()
             .distribution(DistributionType.DEFAULT)
-            .version(getOldClusterTestVersion())
+            .version(getOldClusterVersion())
             .nodes(NODE_NUM)
             .setting("path.repo", new Supplier<>() {
                 @Override
@@ -47,7 +46,7 @@ public String get() {
 
         // Avoid triggering bogus assertion when serialized parsed mappings don't match with original mappings, because _source key is
         // inconsistent
-        if (oldVersion.before(Version.fromString("8.18.0"))) {
+        if (Version.fromString(getOldClusterVersion()).before(Version.fromString("8.18.0"))) {
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper");
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService");
         }

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeWithSecurityTestCase.java

@@ -35,10 +35,9 @@ public abstract class AbstractRollingUpgradeWithSecurityTestCase extends Paramet
     private static final ElasticsearchCluster cluster = buildCluster();
 
     private static ElasticsearchCluster buildCluster() {
-        Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION);
         var cluster = ElasticsearchCluster.local()
             .distribution(DistributionType.DEFAULT)
-            .version(getOldClusterTestVersion())
+            .version(getOldClusterVersion())
             .nodes(NODE_NUM)
             .user(USER, PASS)
             .setting("xpack.security.autoconfiguration.enabled", "false")
@@ -52,7 +51,7 @@ public String get() {
 
         // Avoid triggering bogus assertion when serialized parsed mappings don't match with original mappings, because _source key is
         // inconsistent
-        if (oldVersion.before(Version.fromString("8.18.0"))) {
+        if (Version.fromString(getOldClusterVersion()).before(Version.fromString("8.18.0"))) {
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper");
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService");
         }

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java

@@ -82,7 +82,7 @@ public DenseVectorMappingUpdateIT(@Name("upgradedNodes") int upgradedNodes) {
     }
 
     public void testDenseVectorMappingUpdateOnOldCluster() throws IOException {
-        if (getOldClusterTestVersion().after(Version.V_8_7_0.toString())) {
+        if (oldClusterHasFeature("gte_v8.7.1")) {
             String indexName = "test_index";
             if (isOldCluster()) {
                 Request createIndex = new Request("PUT", "/" + indexName);

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -33,8 +34,12 @@
 
 public class FileSettingsUpgradeIT extends ParameterizedRollingUpgradeTestCase {
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // Remove this rule entirely
     private static final RunnableTestRuleAdapter versionLimit = new RunnableTestRuleAdapter(
-        () -> assumeTrue("Only valid when upgrading from pre-file settings", getOldClusterTestVersion().before(new Version(8, 4, 0)))
+        () -> assumeTrue(
+            "Only valid when upgrading from pre-file settings",
+            Version.fromString(getOldClusterVersion()).before(new Version(8, 4, 0))
+        )
     );
 
     private static final String settingsJSON = """
@@ -54,7 +59,7 @@ public class FileSettingsUpgradeIT extends ParameterizedRollingUpgradeTestCase {
 
     private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
-        .version(getOldClusterTestVersion())
+        .version(getOldClusterVersion())
         .nodes(NODE_NUM)
         .setting("path.repo", new Supplier<>() {
             @Override

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsUsageRollingUpgradeIT.java

@@ -30,7 +30,7 @@ public LogsUsageRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
     }
 
     public void testUsage() throws Exception {
-        assumeTrue("logsdb.prior_logs_usage only gets set in 8.x", getOldClusterTestVersion().before("9.0.0"));
+        assumeFalse("logsdb.prior_logs_usage only gets set in 8.x", oldClusterHasFeature("gte_v9.0.0"));
         String dataStreamName = "logs-mysql-error";
         if (isOldCluster()) {
             bulkIndex(dataStreamName, 4, 256, Instant.now());

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java

@@ -28,6 +28,7 @@
 
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.IntStream;
 
@@ -36,7 +37,7 @@
 
 public abstract class ParameterizedRollingUpgradeTestCase extends ESRestTestCase {
     protected static final int NODE_NUM = 3;
-    protected static final String OLD_CLUSTER_VERSION = System.getProperty("tests.old_cluster_version");
+    private static final String OLD_CLUSTER_VERSION = System.getProperty("tests.old_cluster_version");
     private static final Set<Integer> upgradedNodes = new HashSet<>();
     private static TestFeatureService oldClusterTestFeatureService = null;
     private static boolean upgradeFailed = false;
@@ -127,11 +128,6 @@ public static void resetNodes() {
         upgradeFailed = false;
     }
 
-    @Deprecated // Use the new testing framework and oldClusterHasFeature(feature) instead
-    protected static String getOldClusterVersion() {
-        return OLD_CLUSTER_VERSION;
-    }
-
     protected static boolean oldClusterHasFeature(String featureId) {
         assert oldClusterTestFeatureService != null;
         return oldClusterTestFeatureService.clusterHasFeature(featureId);
@@ -146,12 +142,23 @@ protected static IndexVersion getOldClusterIndexVersion() {
         return oldIndexVersion;
     }
 
-    protected static Version getOldClusterTestVersion() {
-        return Version.fromString(OLD_CLUSTER_VERSION);
+    /**
+     * The version of the "old" (initial) cluster. It is an opaque string, do not even think about parsing it for version
+     * comparison. Use (test) cluster features and {@link ParameterizedRollingUpgradeTestCase#oldClusterHasFeature} instead.
+     */
+    protected static String getOldClusterVersion() {
+        return System.getProperty("tests.bwc.main.version", OLD_CLUSTER_VERSION);
     }
 
     protected static boolean isOldClusterVersion(String nodeVersion) {
-        return OLD_CLUSTER_VERSION.equals(nodeVersion);
+        return getOldClusterVersion().equals(nodeVersion);
+    }
+
+    protected static boolean isOldClusterVersion(String nodeVersion, String buildHash) {
+        if (Objects.equals(System.getProperty("tests.bwc.refspec.main"), buildHash)) {
+            return true;
+        }
+        return isOldClusterVersion(nodeVersion);
     }
 
     protected static boolean isOldCluster() {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java

@@ -137,7 +137,8 @@ private List<String> getUpgradedNodeIds() throws IOException {
         List<String> upgradedNodes = new ArrayList<>();
         for (Map.Entry<String, Map<String, Object>> nodeInfoEntry : nodes.entrySet()) {
             String nodeVersion = extractValue(nodeInfoEntry.getValue(), "version");
-            if (isOldClusterVersion(nodeVersion) == false) {
+            String nodeBuildHash = extractValue(nodeInfoEntry.getValue(), "build_hash");
+            if (isOldClusterVersion(nodeVersion, nodeBuildHash) == false) {
                 upgradedNodes.add(nodeInfoEntry.getKey());
             }
         }

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java

@@ -27,7 +27,7 @@ public SourceModeRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
     }
 
     public void testConfigureStoredSourceBeforeIndexCreationLegacy() throws IOException {
-        assumeTrue("testing deprecation warnings and deprecation migrations", getOldClusterTestVersion().before("9.0.0"));
+        assumeFalse("testing deprecation warnings and deprecation migrations", oldClusterHasFeature("gte_v9.0.0"));
         String templateName = "logs@custom";
         if (isOldCluster()) {
             var storedSourceMapping = """
@@ -56,7 +56,7 @@ public void testConfigureStoredSourceBeforeIndexCreationLegacy() throws IOExcept
     }
 
     public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOException {
-        assumeTrue("testing deprecation warnings and deprecation migrations", getOldClusterTestVersion().before("9.0.0"));
+        assumeFalse("testing deprecation warnings and deprecation migrations", oldClusterHasFeature("gte_v9.0.0"));
         String templateName = "logs@custom";
         if (isOldCluster()) {
             var storedSourceMapping = """