TLS config with RDS tested with a healthy check (#1798)

This tests successful rhdh plugin initialization

readding main deployment

ignore the the TLS check in other tests

consider rds values rename

make all secrets base64 before applying

add rhdh-dyanmic-plugins-root

add path for postgres-crt

Co-authored-by: Joseph Kim <[email protected]>

Author: Omar-AlJaljuli

.ibm/pipelines/env_variables.sh

@@ -19,6 +19,7 @@ RELEASE_NAME_RBAC=rhdh-rbac
 NAME_SPACE="${NAME_SPACE:-showcase}"
 NAME_SPACE_RBAC="${NAME_SPACE_RBAC:-showcase-rbac}"
 NAME_SPACE_POSTGRES_DB="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}"
+NAME_SPACE_RDS="showcase-rds-nightly"
 CHART_VERSION="2.15.2"
 GITHUB_APP_APP_ID=$(cat /tmp/secrets/GITHUB_APP_APP_ID)
 GITHUB_APP_CLIENT_ID=$(cat /tmp/secrets/GITHUB_APP_CLIENT_ID)
@@ -66,6 +67,11 @@ GOOGLE_ACC_COOKIE=$(cat /tmp/secrets/GOOGLE_ACC_COOKIE)
 GOOGLE_USER_ID=$(cat /tmp/secrets/GOOGLE_USER_ID)
 GOOGLE_USER_PASS=$(cat /tmp/secrets/GOOGLE_USER_PASS)
 GOOGLE_2FA_SECRET=$(cat /tmp/secrets/GOOGLE_2FA_SECRET)
+RDS_USER='cmhkaHFl'
+RDS_PASSWORD=$(cat /tmp/secrets/RDS_PASSWORD)
+RDS_1_HOST=$(cat /tmp/secrets/RDS_1_HOST)
+RDS_2_HOST=$(cat /tmp/secrets/RDS_2_HOST)
+RDS_3_HOST=$(cat /tmp/secrets/RDS_3_HOST)
 
 JUNIT_RESULTS="junit-results.xml"
 DATA_ROUTER_URL=$(cat /tmp/secrets/DATA_ROUTER_URL)

.ibm/pipelines/openshift-ci-tests.sh

@@ -359,6 +359,20 @@ initiate_rbac_aks_deployment() {
   helm upgrade -i "${RELEASE_NAME_RBAC}" -n "${NAME_SPACE_RBAC_AKS}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" -f "/tmp/${HELM_CHART_RBAC_AKS_MERGED_VALUE_FILE_NAME}" --set global.host="${K8S_CLUSTER_ROUTER_BASE}" --set upstream.backstage.image.repository="${QUAY_REPO}" --set upstream.backstage.image.tag="${TAG_NAME}"
 }
 
+initiate_rds_deployment() {
+  local release_name=$1
+  local namespace=$2
+  configure_namespace "${namespace}"
+  uninstall_helmchart "${namespace}" "${release_name}"
+  sed -i "s|POSTGRES_USER:.*|POSTGRES_USER: $RDS_USER|g" "${DIR}/resources/postgres-db/postgres-cred.yaml"
+  sed -i "s|POSTGRES_PASSWORD:.*|POSTGRES_PASSWORD: $(echo -n $RDS_PASSWORD | base64 -w 0)|g" "${DIR}/resources/postgres-db/postgres-cred.yaml"
+  sed -i "s|POSTGRES_HOST:.*|POSTGRES_HOST: $(echo -n $RDS_1_HOST | base64 -w 0)|g" "${DIR}/resources/postgres-db/postgres-cred.yaml"
+  oc apply -f "$DIR/resources/postgres-db/postgres-crt-rds.yaml" -n "${namespace}" 
+  oc apply -f "$DIR/resources/postgres-db/postgres-cred.yaml" -n "${namespace}"
+  oc apply -f "$DIR/resources/postgres-db/dynamic-plugins-root-PVC.yaml" -n "${namespace}"
+  helm upgrade -i "${release_name}" -n "${namespace}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" -f "$DIR/resources/postgres-db/values-showcase-postgres.yaml" --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" --set upstream.backstage.image.repository="${QUAY_REPO}" --set upstream.backstage.image.tag="${TAG_NAME}"
+}
+
 check_and_test() {
   local release_name=$1
   local namespace=$2
@@ -437,6 +451,7 @@ main() {
   ENCODED_API_SERVER_URL=$(echo "${API_SERVER_URL}" | base64)
   ENCODED_CLUSTER_NAME=$(echo "my-cluster" | base64)
 
+
   if [[ "$JOB_NAME" == *aks* ]]; then
     initiate_aks_deployment
     check_and_test "${RELEASE_NAME}" "${NAME_SPACE_AKS}"
@@ -448,7 +463,13 @@ main() {
     initiate_deployments
     check_and_test "${RELEASE_NAME}" "${NAME_SPACE}"
     check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}"
+    # Only test TLS config with RDS in nightly jobs
+    if [[ "$JOB_NAME" == *periodic* ]]; then
+      initiate_rds_deployment "${RELEASE_NAME}" "${NAME_SPACE_RDS}"
+      check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RDS}"
+    fi
   fi
+  
   exit "${OVERALL_RESULT}"
 }
 

.ibm/pipelines/resources/postgres-db/dynamic-plugins-root-PVC.yaml

@@ -0,0 +1,10 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: rhdh-dynamic-plugins-root
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

.ibm/pipelines/resources/postgres-db/postgres-cred.yaml

@@ -8,3 +8,4 @@ data:
   POSTGRES_USER: amFudXMtaWRw
   POSTGRES_HOST: dG1w
   PGSSLMODE: cmVxdWlyZQ==
+  NODE_EXTRA_CA_CERTS: L29wdC9hcHAtcm9vdC9zcmMvcG9zdGdyZXMtY3J0LnBlbQ==

.ibm/pipelines/resources/postgres-db/postgres-crt-rds.yaml

@@ -0,0 +1,109 @@
+global:
+  dynamic:
+    includes:
+      - dynamic-plugins.default.yaml
+    plugins: []
+upstream:
+  nameOverride: backstage
+  commonLabels:
+    backstage.io/kubernetes-id: developer-hub
+  backstage:
+    image:
+      pullPolicy: Always
+      registry: quay.io
+      repository: janus-idp/backstage-showcase
+      tag: next
+    appConfig:
+      app:
+        baseUrl: 'https://{{- include "janus-idp.hostname" . }}'
+      backend:
+        auth:
+          externalAccess:
+            - options:
+                secret: '${BACKEND_SECRET}'
+                subject: legacy-default-config
+              type: legacy
+        baseUrl: 'https://{{- include "janus-idp.hostname" . }}'
+        cors:
+          origin: 'https://{{- include "janus-idp.hostname" . }}'
+        database:
+          connection:  # configure Backstage DB connection parameters
+            host: ${POSTGRES_HOST}
+            port: ${POSTGRES_PORT}
+            user: ${POSTGRES_USER}
+            password: ${POSTGRES_PASSWORD}
+      auth:
+        environment: development
+        providers:
+          guest:
+            dangerouslyAllowOutsideDevelopment: true
+    extraEnvVars:
+      - name: BACKEND_SECRET
+        valueFrom:
+          secretKeyRef:
+            key: backend-secret
+            name: '{{ include "janus-idp.backend-secret-name" $ }}'
+    extraVolumeMounts:
+      - mountPath: /opt/app-root/src/dynamic-plugins-root
+        name: dynamic-plugins-root
+      - mountPath: /opt/app-root/src/postgres-crt.pem
+        name: postgres-crt # inject certificate secret to Backstage cont.
+        subPath: postgres-crt.pem
+    extraVolumes:
+      - name: dynamic-plugins-root
+        persistentVolumeClaim:
+          claimName: '{{ printf "%s-dynamic-plugins-root" .Release.Name }}'
+      - configMap:
+          defaultMode: 420
+          name: '{{ printf "%s-dynamic-plugins" .Release.Name }}'
+          optional: true
+        name: dynamic-plugins
+      - name: dynamic-plugins-npmrc
+        secret:
+          defaultMode: 420
+          optional: true
+          secretName: '{{ printf "%s-dynamic-plugins-npmrc" .Release.Name }}'
+      - name: dynamic-plugins-registry-auth
+        secret:
+          defaultMode: 416
+          optional: true
+          secretName: '{{ printf "%s-dynamic-plugins-registry-auth" .Release.Name }}'
+      - name: postgres-crt
+        secret:
+          secretName: postgres-crt
+      - emptyDir: {}
+        name: npmcacache
+    initContainers:
+      - name: install-dynamic-plugins
+        image: '{{ include "backstage.image" . }}'
+        command:
+          - ./install-dynamic-plugins.sh
+          - /dynamic-plugins-root
+        env:
+          - name: NPM_CONFIG_USERCONFIG
+            value: /opt/app-root/src/.npmrc.dynamic-plugins
+        imagePullPolicy: Always
+        volumeMounts:
+          - mountPath: /dynamic-plugins-root
+            name: dynamic-plugins-root
+          - mountPath: /opt/app-root/src/dynamic-plugins.yaml
+            name: dynamic-plugins
+            readOnly: true
+            subPath: dynamic-plugins.yaml
+          - mountPath: /opt/app-root/src/.npmrc.dynamic-plugins
+            name: dynamic-plugins-npmrc
+            readOnly: true
+            subPath: .npmrc
+          - mountPath: /opt/app-root/src/.config/containers
+            name: dynamic-plugins-registry-auth
+            readOnly: true
+          - mountPath: /opt/app-root/src/.npm/_cacache
+            name: npmcacache
+        workingDir: /opt/app-root/src
+    installDir: /opt/app-root/src
+    extraEnvVarsSecrets:
+      - postgres-cred
+  postgresql:
+    enabled: false
+    auth:
+      existingSecret: postgres-cred

.ibm/pipelines/resources/postgres-db/values-showcase-postgres.yaml

@@ -16,6 +16,7 @@
     "showcase-rbac-nightly": "playwright test --project=showcase-rbac",
     "showcase-aks-ci-nightly": "playwright test --project=showcase-aks",
     "showcase-rbac-aks-ci-nightly": "playwright test --project=showcase-rbac-aks",
+    "showcase-rds-nightly": "playwright test --project=postgres-health-check",
     "lint:check": "eslint . --ext .js,.ts",
     "lint:fix": "eslint . --ext .js,.ts --fix",
     "postinstall": "playwright install",

e2e-tests/package.json

@@ -55,6 +55,7 @@ export default defineConfig({
         "**/playwright/e2e/verify-tls-config-with-external-postgres-db.spec.ts",
         "**/playwright/e2e/authProviders/**/*.spec.ts",
         "**/playwright/e2e/plugins/bulk-import.spec.ts",
+        "**/playwright/e2e/verify-tls-config-health-check.spec.ts",
       ],
     },
     {
@@ -77,6 +78,7 @@ export default defineConfig({
       testIgnore: [
         "**/playwright/e2e/authProviders/setup-environment.spec.ts",
         "**/playwright/e2e/authProviders/clear-environment.spec.ts",
+        "**/playwright/e2e/verify-tls-config-health-check.spec.ts",
       ],
       dependencies: ["showcase-auth-providers-setup-environment"],
       teardown: "showcase-auth-providers-clear-environment",
@@ -105,6 +107,7 @@ export default defineConfig({
         "**/playwright/e2e/audit-log/**/*.spec.ts",
         "**/playwright/e2e/verify-redis-cache.spec.ts",
         "**/playwright/e2e/plugins/topology/topology.spec.ts",
+        "**/playwright/e2e/verify-tls-config-health-check.spec.ts",
       ],
     },
     {
@@ -116,6 +119,11 @@ export default defineConfig({
         "**/playwright/e2e/plugins/bulk-import.spec.ts",
       ],
     },
+    {
+      name: "postgres-health-check",
+      ...useCommonDeviceAndViewportConfig,
+      testMatch: ["**/playwright/e2e/verify-tls-config-health-check.spec.ts"],
+    },
 
     // {
     //   name: 'firefox',

e2e-tests/playwright.config.ts

@@ -0,0 +1,59 @@
+import { test } from "@playwright/test";
+import { Common } from "../utils/Common";
+import { kubeCLient } from "../utils/k8sHelper";
+
+export const k8sClient = new kubeCLient();
+
+test.describe
+  .serial("Verify TLS configuration with Postgres DB health check", () => {
+  const namespace = process.env.NAME_SPACE_RDS;
+  const deploymentName = "rhdh-developer-hub";
+  const secretName = "postgres-cred";
+  const hostLatest2 = Buffer.from(process.env.RDS_2_HOST).toString("base64");
+  const hostLatest3 = Buffer.from(process.env.RDS_3_HOST).toString("base64");
+
+  test("Verify successful DB connection and successful initialization of plugins with latest-1 postgres version", async ({
+    page,
+  }) => {
+    const common = new Common(page);
+    await common.loginAsGuest();
+  });
+
+  test("Change the config to use the latest-2 postgres version", async () => {
+    test.setTimeout(120000);
+    const secretData = {
+      POSTGRES_HOST: hostLatest2,
+    };
+    const patch = {
+      data: secretData,
+    };
+    await k8sClient.updateSecret(secretName, namespace, patch);
+    await k8sClient.restartDeployment(deploymentName, namespace);
+  });
+
+  test("Verify successful DB connection and successful initialization of plugins with latest-2 postgres version", async ({
+    page,
+  }) => {
+    const common = new Common(page);
+    await common.loginAsGuest();
+  });
+
+  test("Change the config to use the latest-3 postgres version", async () => {
+    test.setTimeout(120000);
+    const secretData = {
+      POSTGRES_HOST: hostLatest3,
+    };
+    const patch = {
+      data: secretData,
+    };
+    await k8sClient.updateSecret(secretName, namespace, patch);
+    await k8sClient.restartDeployment(deploymentName, namespace);
+  });
+
+  test("Verify successful DB connection and successful initialization of plugins with latest-3 postgres version", async ({
+    page,
+  }) => {
+    const common = new Common(page);
+    await common.loginAsGuest();
+  });
+});

e2e-tests/playwright/e2e/verify-tls-config-health-check.spec.ts

@@ -34,6 +34,32 @@ export class kubeCLient {
     }
   }
 
+  async scaleDeployment(
+    deploymentName: string,
+    namespace: string,
+    replicas: number,
+  ) {
+    const patch = { spec: { replicas: replicas } };
+    try {
+      await this.appsApi.patchNamespacedDeploymentScale(
+        deploymentName,
+        namespace,
+        patch,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        {
+          headers: { "Content-Type": "application/strategic-merge-patch+json" },
+        },
+      );
+      console.log(`Deployment scaled to ${replicas} replicas.`);
+    } catch (error) {
+      console.error("Error scaling deployment:", error);
+    }
+  }
+
   async getSecret(secretName: string, namespace: string) {
     try {
       logger.info(`Getting secret ${secretName} from namespace ${namespace}`);
@@ -181,4 +207,51 @@ export class kubeCLient {
       throw err;
     }
   }
+
+  async waitForDeploymentReady(
+    deploymentName: string,
+    namespace: string,
+    expectedReplicas: number,
+    timeout: number = 100000,
+    checkInterval: number = 10000,
+  ) {
+    const start = Date.now();
+
+    while (Date.now() - start < timeout) {
+      try {
+        const response = await this.appsApi.readNamespacedDeployment(
+          deploymentName,
+          namespace,
+        );
+        const availableReplicas = response.body.status?.availableReplicas || 0;
+
+        if (availableReplicas === expectedReplicas) {
+          console.log(
+            `Deployment ${deploymentName} is ready with ${availableReplicas} replicas.`,
+          );
+          return;
+        }
+
+        console.log(
+          `Waiting for ${deploymentName} to reach ${expectedReplicas} replicas, currently has ${availableReplicas}.`,
+        );
+        await new Promise((resolve) => setTimeout(resolve, checkInterval));
+      } catch (error) {
+        console.error(`Error checking deployment status: ${error}`);
+        throw error;
+      }
+    }
+
+    throw new Error(
+      `Deployment ${deploymentName} did not become ready in time.`,
+    );
+  }
+
+  async restartDeployment(deploymentName: string, namespace: string) {
+    await this.scaleDeployment(deploymentName, namespace, 0);
+    await this.waitForDeploymentReady(deploymentName, namespace, 0);
+
+    await this.scaleDeployment(deploymentName, namespace, 1);
+    await this.waitForDeploymentReady(deploymentName, namespace, 1);
+  }
 }