Pg cleanup

cloud/common/vars/dependencies.groovy

@@ -191,7 +191,7 @@ void installExecutorDependencies(String testExecutorType) {
     }
 }
 
-void installProviderDependencies(Map libraries, String operator, String provider) {
+void installProviderDependencies(Map libraries, String operator, String provider, String platformVersion = '') {
     // PSMDB requires Google and Azure CLIs, regardless of the provider.
     // Rancher requires Google CLI to create cluster as GCE instances are used.
 
@@ -204,12 +204,24 @@ void installProviderDependencies(Map libraries, String operator, String provider
         installAzureCLI()
         libraries.azure.auth()
     }
+
+    if (provider == 'eks') {
+        installEksctl()
+    }
+
+    if (provider == 'doks') {
+        installDoctl()
+    }
+
+    if (provider == 'openshift') {
+        installOpenshiftClient(platformVersion ?: 'latest')
+    }
 }
 
-void prepareNode(Map libraries, String testExecutorType,String operator, String provider) {
+void prepareNode(Map libraries, String testExecutorType, String operator, String provider, String platformVersion = '') {
     install()
     installExecutorDependencies(testExecutorType)
-    installProviderDependencies(libraries, operator, provider)
+    installProviderDependencies(libraries, operator, provider, platformVersion)
 }
 
 return this

cloud/common/vars/eks.groovy

@@ -19,6 +19,12 @@ void createCluster(Map clusterCfg) {
     def clusterSuffix = clusterCfg.clusterSuffix
     def clusterFullName = "${clusterCfg.clusterName}-${clusterSuffix}"
 
+    def preBootstrapCommands = clusterCfg.hugepages ? """  preBootstrapCommands:
+    - "echo 'vm.nr_hugepages=1024' >> /etc/sysctl.conf"
+    - "echo 'vm.hugetlb_shm_group=26' >> /etc/sysctl.conf"
+    - "sysctl -p"
+""" : ""
+
     timeout(time: 30, unit: 'MINUTES') {
         sh """
             timestamp="\$(date +%s)"
@@ -52,7 +58,7 @@ nodeGroups:
     - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
   instancesDistribution:
     instanceTypes: ["m5.xlarge", "m5.2xlarge"] # At least two instance types should be specified
-  tags:
+${preBootstrapCommands}  tags:
     'iit-billing-tag': 'jenkins-eks'
     'delete-cluster-after-hours': '6'
     'team': 'cloud'
@@ -130,7 +136,11 @@ void shutdownCluster(Map clusterCfg) {
                     for loadbal in \$LOADBALS; do
                         aws elb delete-load-balancer --load-balancer-name \$loadbal --region ${region}
                     done
-                    eksctl delete cluster -f cluster-${clusterSuffix}.yaml --wait --force --disable-nodegroup-eviction || true
+                    if [ -f cluster-${clusterSuffix}.yaml ]; then
+                        eksctl delete cluster -f cluster-${clusterSuffix}.yaml --wait --force --disable-nodegroup-eviction || true
+                    else
+                        eksctl delete cluster --name ${clusterFullName} --region ${region} --wait --force --disable-nodegroup-eviction || true
+                    fi
 
                     VPC_DESC=\$(aws ec2 describe-vpcs --vpc-id \$VPC_ID --region ${region} || true)
                     if [ -n "\$VPC_DESC" ]; then

cloud/common/vars/gcloud.groovy

@@ -38,13 +38,21 @@ void createCluster(Map clusterCfg) {
                 "GKE_RELEASE_CHANNEL=${clusterCfg.platformChannel}",
                 "GKE_REGION=${clusterCfg.zone}",
                 "PLATFORM_VER=${clusterCfg.platformVersion}",
-                "MACHINE_TYPE=${clusterCfg.machineType}"
+                "MACHINE_TYPE=${clusterCfg.machineType}",
+                "HUGEPAGES=${clusterCfg.hugepages ? '1' : ''}"
             ]) {
                 sh '''
                     export KUBECONFIG=/tmp/$CLUSTER_NAME-$CLUSTER_SUFFIX
                     maxRetries=15
                     exitCode=1
 
+                    SYSTEM_CONFIG_FLAG=""
+                    SYSTEM_CONFIG_FILE="$WORKSPACE/hugepages-config-$CLUSTER_SUFFIX.yaml"
+                    if [[ -n "$HUGEPAGES" ]]; then
+                        printf 'linuxConfig:\n  hugepageConfig:\n    hugepage_size2m: 1024\n' > "$SYSTEM_CONFIG_FILE"
+                        SYSTEM_CONFIG_FLAG="--system-config-from-file=$SYSTEM_CONFIG_FILE"
+                    fi
+
                     while [[ $exitCode != 0 && $maxRetries > 0 ]]; do
                         gcloud container clusters create $CLUSTER_NAME-$CLUSTER_SUFFIX \
                             --release-channel $GKE_RELEASE_CHANNEL \
@@ -63,13 +71,15 @@ void createCluster(Map clusterCfg) {
                             --logging=NONE \
                             --no-enable-managed-prometheus \
                             --workload-pool=cloud-dev-112233.svc.id.goog \
+                            $SYSTEM_CONFIG_FLAG \
                             --quiet &&\
                         kubectl create clusterrolebinding cluster-admin-binding1 --clusterrole=cluster-admin --user=$(gcloud config get-value core/account)
                         exitCode=$?
                         if [[ $exitCode == 0 ]]; then break; fi
                         (( maxRetries -- ))
                         sleep 1
                     done
+                    rm -f "$SYSTEM_CONFIG_FILE"
                     if [[ $exitCode != 0 ]]; then exit $exitCode; fi
 
                     CURRENT_TIME=$(date --rfc-3339=seconds)

cloud/common/vars/tests.groovy

@@ -1,5 +1,11 @@
 void loadCloudSecret(String operator) {
-    def credentialsId = operator in ["pg-operator", "pxc-operator"] ? "cloud-secret-file" : "cloud-secret-file-${operator}"
+    def credentialsByOperator = [
+        'pg-operator'   : 'cloud-secret-file',
+        'pxc-operator'  : 'cloud-secret-file',
+        'ps-operator'   : 'cloud-secret-file-ps',
+        'psmdb-operator': 'cloud-secret-file-psmdb'
+    ]
+    def credentialsId = credentialsByOperator[operator] ?: "cloud-secret-file-${operator}"
 
     withCredentials([
         file(
@@ -114,21 +120,26 @@ void printTestVariables(Map testVariables) {
 }
 
 String getReleaseParamName(String imageName, String pillarVersion, String operator) {
-    def operatorImages = [
+    def pgVersionKey = pillarVersion?.replace('-postgis', '')
+    def pgPostgresKey = pillarVersion?.endsWith('-postgis') ? "IMAGE_POSTGIS${pgVersionKey}" : "IMAGE_POSTGRESQL${pgVersionKey}"
+
+    def versionedImages = [
         "psmdb-operator": [
             IMAGE_MONGOD: "IMAGE_MONGOD${pillarVersion}"
         ],
         "ps-operator": [
-            IMAGE_MYSQL: "IMAGE_MYSQL${pillarVersion}"
+            IMAGE_MYSQL : "IMAGE_MYSQL${pillarVersion}",
+            IMAGE_BACKUP: "IMAGE_BACKUP${pillarVersion}",
+            IMAGE_ROUTER: "IMAGE_ROUTER${pillarVersion}"
         ],
         "pxc-operator": [
             IMAGE_PXC   : "IMAGE_PXC${pillarVersion}",
             IMAGE_BACKUP: "IMAGE_BACKUP${pillarVersion}"
         ],
         "pg-operator": [
-            IMAGE_POSTGRESQL: "IMAGE_POSTGRESQL${pillarVersion}",
-            IMAGE_PGBOUNCER : "IMAGE_PGBOUNCER${pillarVersion}",
-            IMAGE_BACKREST  : "IMAGE_BACKREST${pillarVersion}"
+            IMAGE_POSTGRESQL: pgPostgresKey,
+            IMAGE_PGBOUNCER : "IMAGE_PGBOUNCER${pgVersionKey}",
+            IMAGE_BACKREST  : "IMAGE_BACKREST${pgVersionKey}"
         ]
     ]
 
@@ -138,7 +149,7 @@ String getReleaseParamName(String imageName, String pillarVersion, String operat
         return "IMAGE_POSTGIS${pillarVersion}"
     }
 
-    return operatorImages[operator?.toLowerCase()]?.get(imageName) ?: imageName
+    return versionedImages[operator?.toLowerCase()]?.get(imageName) ?: imageName
 }
 
 Boolean isReleaseRun(Map testVariables) {
@@ -445,6 +456,10 @@ String getExportedVariablesForTests(Map testVariables, String clusterSuffix) {
         exports << 'export COLUMNS=200'
     }
 
+    if (testVariables.test_executor_type == "kuttl") {
+        exports << 'export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"'
+    }
+
     testVariables.extra_envs?.each { key, value ->
         exports << "export ${key}='${value ?: ""}'"
     }
@@ -483,6 +498,38 @@ Map buildPxcTestVariables(Map config) {
     ]
 }
 
+Map buildPgTestVariables(Map config) {
+    return [
+        cluster_name           : config.cluster_name,
+        kubeconfigPath         : config.kubeconfigPath ?: '/tmp',
+        kubeconfig             : config.kubeconfig,
+        skip_kubeconfig        : config.skip_kubeconfig ?: false,
+        debug_tests            : config.debug_tests,
+        cluster_wide           : config.cluster_wide,
+        operator               : 'pg-operator',
+        default_operator_image : config.default_operator_image,
+        test_executor_type     : 'kuttl',
+        images                 : config.images,
+        extra_envs             : config.extra_envs ?: [:]
+    ]
+}
+
+Map buildPsTestVariables(Map config) {
+    return [
+        cluster_name           : config.cluster_name,
+        kubeconfigPath         : config.kubeconfigPath ?: '/tmp',
+        kubeconfig             : config.kubeconfig,
+        skip_kubeconfig        : config.skip_kubeconfig ?: false,
+        debug_tests            : config.debug_tests ?: 'NO',
+        cluster_wide           : config.cluster_wide,
+        operator               : 'ps-operator',
+        default_operator_image : config.default_operator_image,
+        test_executor_type     : 'kuttl',
+        images                 : config.images,
+        extra_envs             : config.extra_envs ?: [:]
+    ]
+}
+
 String defineTestCommand(Map testVariables, String testName) {
     switch (testVariables.test_executor_type) {
         case "kuttl":
@@ -693,6 +740,7 @@ void clusterRunner(String clusterSuffix, Map testVariables) {
         workerCountMax  : testVariables.worker_max_count ?: 6,
         region          : testVariables.region ?: "",
         zone            : testVariables.zone ?: "",
+        hugepages       : testVariables.hugepages ?: false,
         kubeconfig      : "${testVariables.kubeconfigPath}/${getClusterFullName(testVariables.cluster_name, clusterSuffix)}",
         debug           : testVariables.debug
     ]
@@ -729,18 +777,21 @@ void clusterRunner(String clusterSuffix, Map testVariables) {
                 createCluster.call()
             }
 
-            runTest(
-                testId: testId,
-                clusterSuffix: clusterSuffix,
-                testVariables: testVariables,
-                retries: testVariables.retries ?: 1
-            )
+            withKubeCredentials(testVariables) {
+                runTest(
+                    testId: testId,
+                    clusterSuffix: clusterSuffix,
+                    testVariables: testVariables,
+                    retries: testVariables.retries ?: 1
+                )
+            }
         }
     } finally {
-        // Each cluster contains only suffix
         createdClusters.each { cluster ->
             try {
-                clusterCleanup.call()
+                withKubeCredentials(testVariables) {
+                    clusterCleanup.call()
+                }
                 shutdownCluster.call()
                 removeCluster(testVariables.clusters, cluster)
             } catch (Exception e) {
@@ -750,6 +801,23 @@ void clusterRunner(String clusterSuffix, Map testVariables) {
     }
 }
 
+void withKubeCredentials(Map testVariables, Closure body) {
+    switch (testVariables.platform_provider) {
+        case 'doks':
+            withCredentials([string(credentialsId: 'DOKS_TOKEN', variable: 'DIGITALOCEAN_ACCESS_TOKEN')]) {
+                body()
+            }
+            return
+        case 'eks':
+            withCredentials([aws(credentialsId: 'eks-cicd', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) {
+                body()
+            }
+            return
+        default:
+            body()
+    }
+}
+
 Map buildParallelClusterStages(Map testVariables) {
     def parallelStages = [:]
 
@@ -766,12 +834,13 @@ Map buildParallelClusterStages(Map testVariables) {
             stage(clusterSuffix) {
                 if (testVariables.jenkins_agent_label) {
                     node(testVariables.jenkins_agent_label) {
-                        libraries.tools.unstashClonedGitFiles()
-                        libraries.dependencies.prepareNode(
+                        testVariables.libraries.tools.unstashClonedGitFiles()
+                        testVariables.libraries.dependencies.prepareNode(
                             testVariables.libraries,
                             testVariables.test_executor_type,
                             testVariables.operator,
-                            testVariables.platform_provider
+                            testVariables.platform_provider,
+                            testVariables.platform_version
                         )
                         clusterRunner(clusterSuffix, testVariables)
                     }

cloud/common/vars/tools.groovy

@@ -17,7 +17,7 @@ void gitClone(Map cfg) {
         '''
     }
 
-    stash name: 'sourceFILES', includes: 'source/**'
+    stash name: 'sourceFILES', includes: 'source/**', useDefaultExcludes: false
 }
 
 void unstashClonedGitFiles() {
@@ -114,11 +114,8 @@ void dockerBuildAndPush(Map cfg) {
                     docker buildx use multiarch 2>/dev/null || docker buildx create --name multiarch --use
                     echo "\$PASS" | docker login -u "\$USER" --password-stdin
                     export IMAGE=${cfg.operatorImage}:${cfg.branch}
-                    if [[ "$cfg.operator" == "pg-operator" ]]; then
-                        DOCKER_DEFAULT_PLATFORM=linux/amd64,linux/arm64 make build
-                    else
-                        DOCKER_DEFAULT_PLATFORM=linux/amd64,linux/arm64 e2e-tests/build
-                    fi
+                    export DOCKER_DEFAULT_PLATFORM=${cfg.platform ?: 'linux/amd64,linux/arm64'}
+                    ${cfg.buildCommand ?: "e2e-tests/build"}
                     docker logout
                 '