Adding BWC test for remote publication enabled cluster (#20221)

* Adding BWC test for remote publication enabled cluster

Signed-off-by: Harsh Garg <[email protected]>

Author: gargharsh3134

CHANGELOG.md

@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support for HTTP/3 (server side) ([#20017](https://github.com/opensearch-project/OpenSearch/pull/20017))
 - Add circuit breaker support for gRPC transport to prevent out-of-memory errors ([#20203](https://github.com/opensearch-project/OpenSearch/pull/20203))
 - Add index-level-encryption support for snapshots and remote-store ([#20095](https://github.com/opensearch-project/OpenSearch/pull/20095))
+- Adding BackWardCompatibility test for remote publication enabled cluster ([#20221](https://github.com/opensearch-project/OpenSearch/pull/20221))
+
 
 ### Changed
 - Handle custom metadata files in subdirectory-store ([#20157](https://github.com/opensearch-project/OpenSearch/pull/20157))

TESTING.md

@@ -395,6 +395,8 @@ Backwards compatibility tests exist to test upgrading from each supported versio
 
 The test can be run for any versions which the current version will be compatible with. Tests are run for released versions download the distributions from the artifact repository, see [DistributionDownloadPlugin](./buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java) for the repository location. Tests are run for versions that are not yet released automatically check out the branch and build from source to get the distributions, see [BwcVersions](./buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java) and [distribution/bwc/build.gradle](./distribution/bwc/build.gradle) for more information.
 
+**Note:** Rolling upgrade tests run in two modes: with remote publication disabled (default) and with remote publication enabled (using a `-remote` suffix cluster). The remote publication tests verify cluster state persistence to remote storage during upgrades. See [Remote Cluster State documentation](https://docs.opensearch.org/latest/tuning-your-cluster/availability-and-recovery/remote-store/remote-cluster-state/) for more information.
+
 The minimum JDK versions for runtime and compiling need to be installed, and environment variables `JAVAx_HOME`, such as `JAVA8_HOME`, pointing to the JDK installations are required to run the tests against unreleased versions, since the distributions are created by building from source. The required JDK versions for each branch are located at [.ci/java-versions.properties](.ci/java-versions.properties), see [BwcSetupExtension](./buildSrc/src/main/java/org/opensearch/gradle/internal/BwcSetupExtension.java) for more information.
 
 To run all the backwards compatibility tests use:

qa/rolling-upgrade/build.gradle

@@ -37,6 +37,23 @@ apply plugin: 'opensearch.standalone-test'
 apply from : "$rootDir/gradle/bwc-test.gradle"
 apply plugin: 'opensearch.rest-resources'
 
+def configureTestTask(task, clusterName, bwcVersionStr, suite, excludeRemoteTest, extraConfig = {}) {
+  task.configure {
+    useCluster project, testClusters."${clusterName}"
+    systemProperty 'tests.upgrade_from_version', bwcVersionStr
+    systemProperty 'tests.rest.suite', suite
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${clusterName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${clusterName}".getName()}")
+
+    if (excludeRemoteTest) {
+      exclude '**/RemotePublicationClusterStateIT.class'
+    }
+
+    extraConfig.delegate = delegate
+    extraConfig()
+  }
+}
+
 for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
   /*
    * The goal here is to:
@@ -47,7 +64,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
    *  <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster'
    *  <li>upgrade one more node
    *  <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' again
-   *  <li>updgrade the last node
+   *  <li>upgrade the last node
    *  <li>run tests with systemProperty 'tests.rest.suite', 'upgraded_cluster'
    * </ul>
    */
@@ -58,7 +75,6 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
     "${baseName}" {
       versions = [bwcVersionStr, project.version]
       numberOfNodes = 3
-
       setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
       setting 'http.content_type.required', 'true'
@@ -67,57 +83,85 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
 
   tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
     dependsOn processTestResources
-    useCluster project, testClusters."${baseName}"
     mustRunAfter(precommit)
-    doFirst {
-      delete("${buildDir}/cluster/shared/repo/${baseName}")
+    configureTestTask(it, baseName, bwcVersionStr, 'old_cluster', true) {
+      doFirst { delete("${buildDir}/cluster/shared/repo/${baseName}") }
     }
-    systemProperty 'tests.upgrade_from_version', bwcVersionStr
-    systemProperty 'tests.rest.suite', 'old_cluster'
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
   }
 
   tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#oldClusterTest"
-    useCluster project, testClusters."${baseName}"
-    doFirst {
-      testClusters."${baseName}".nextNodeToNextVersion()
+    configureTestTask(it, baseName, bwcVersionStr, 'mixed_cluster', true) {
+      systemProperty 'tests.first_round', 'true'
+      doFirst { testClusters."${baseName}".nextNodeToNextVersion() }
     }
-    systemProperty 'tests.rest.suite', 'mixed_cluster'
-    systemProperty 'tests.upgrade_from_version', bwcVersionStr
-    systemProperty 'tests.first_round', 'true'
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
   }
 
   tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#oneThirdUpgradedTest"
-    useCluster project, testClusters."${baseName}"
-    doFirst {
-      testClusters."${baseName}".nextNodeToNextVersion()
+    configureTestTask(it, baseName, bwcVersionStr, 'mixed_cluster', true) {
+      systemProperty 'tests.first_round', 'false'
+      doFirst { testClusters."${baseName}".nextNodeToNextVersion() }
     }
-    systemProperty 'tests.rest.suite', 'mixed_cluster'
-    systemProperty 'tests.upgrade_from_version', bwcVersionStr
-    systemProperty 'tests.first_round', 'false'
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
   }
 
   tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#twoThirdsUpgradedTest"
-    doFirst {
-      testClusters."${baseName}".nextNodeToNextVersion()
+    configureTestTask(it, baseName, bwcVersionStr, 'upgraded_cluster', true) {
+      doFirst { testClusters."${baseName}".nextNodeToNextVersion() }
     }
-    useCluster project, testClusters."${baseName}"
-    systemProperty 'tests.rest.suite', 'upgraded_cluster'
-    systemProperty 'tests.upgrade_from_version', bwcVersionStr
+  }
+
+  // Remote publication cluster
+  String remoteBaseName = "${baseName}-remote"
+
+  testClusters {
+    "${remoteBaseName}" {
+      versions = [bwcVersionStr, project.version]
+      numberOfNodes = 3
+      setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+      setting 'path.repo', "${buildDir}/cluster/shared/repo/${remoteBaseName}"
+      setting 'http.content_type.required', 'true'
+      setting 'node.attr.remote_store.routing_table.repository', 'my-fs-repository'
+      setting 'node.attr.remote_store.state.repository', 'my-fs-repository'
+      setting 'node.attr.remote_store.repository.my-fs-repository.type', 'fs'
+      setting 'node.attr.remote_store.repository.my-fs-repository.settings.location', "${buildDir}/cluster/shared/repo/${remoteBaseName}/remote_state"
+      setting 'cluster.remote_store.state.enabled', 'true'
+      setting 'cluster.remote_store.publication.enabled', 'true'
+    }
+  }
+
+  tasks.register("${remoteBaseName}#oldClusterTest", StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#upgradedClusterTest"
+    configureTestTask(it, remoteBaseName, bwcVersionStr, 'old_cluster', false) {
+      doFirst { delete("${buildDir}/cluster/shared/repo/${remoteBaseName}") }
+    }
+  }
+
+  tasks.register("${remoteBaseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) {
+    dependsOn "${remoteBaseName}#oldClusterTest"
+    configureTestTask(it, remoteBaseName, bwcVersionStr, 'mixed_cluster', false) {
+      systemProperty 'tests.first_round', 'true'
+      doFirst { testClusters."${remoteBaseName}".nextNodeToNextVersion() }
+    }
+  }
 
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
+  tasks.register("${remoteBaseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) {
+    dependsOn "${remoteBaseName}#oneThirdUpgradedTest"
+    configureTestTask(it, remoteBaseName, bwcVersionStr, 'mixed_cluster', false) {
+      systemProperty 'tests.first_round', 'false'
+      doFirst { testClusters."${remoteBaseName}".nextNodeToNextVersion() }
+    }
+  }
+
+  tasks.register("${remoteBaseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
+    dependsOn "${remoteBaseName}#twoThirdsUpgradedTest"
+    configureTestTask(it, remoteBaseName, bwcVersionStr, 'upgraded_cluster', false) {
+      doFirst { testClusters."${remoteBaseName}".nextNodeToNextVersion() }
+    }
   }
 
   tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn tasks.named("${baseName}#upgradedClusterTest")
+    dependsOn tasks.named("${remoteBaseName}#upgradedClusterTest")
   }
 }