chore: add Local Test Runner support for E2E Tests for RHDH core. (#4023)

* local test runner

* fix(pipelines): dynamically select latest nodejs UBI9 image tag

Instead of hardcoding nodejs:18-ubi8, query the cluster for available
nodejs imagestream tags and select the latest UBI9 version. Falls back
to 18-ubi8 if no UBI9 tags are found.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* Update .ibm/pipelines/utils.sh

Co-authored-by: Zbyněk Drápela <61500440+zdrapela@users.noreply.github.com>

* feat(e2e-tests): add CLI flags for non-interactive local test runner

Add CLI flags (-j, -r, -t, -p, -s) to local-run.sh for automation and
quick runs without interactive prompts. Also improve container-init.sh
to pre-compute and save config before deployment so URLs are available
even if deployment fails.

Additional fixes:
- Fix secrets parsing in local-test-setup.sh to handle special chars
- Minor formatting fix in utils.sh for piped command

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* feat(e2e-tests): add multi-platform support for AKS, EKS, and GKE

Extend local test runner to support non-OpenShift Kubernetes platforms:
- Derive CONTAINER_PLATFORM from job name (*aks*, *eks*, *gke*, *osd*)
- Add platform-specific cluster login (oc for OCP/OSD, kubectl for others)
- Handle service account token creation for non-OpenShift clusters
- Update container-init.sh to configure kubectl context for AKS/EKS/GKE
- Add interactive menu options for AKS, EKS, GKE job selection
- Update help text and examples for multi-platform usage

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: Zbyněk Drápela <61500440+zdrapela@users.noreply.github.com>

Author: 3 people

.gitignore

@@ -76,6 +76,13 @@ dynamic-plugins-root/*
 .ibm/pipelines/artifact_dir/*
 .ibm/pipelines/env_override.local.sh
 
+# E2E local test runner work directory
+e2e-tests/.local-test/
+
+# E2E test artifacts (generated by Playwright)
+e2e-tests/playwright-report/
+e2e-tests/test-results/
+
 # Python Caches
 **/__pycache__/
 **/.pytest_cache/

.ibm/pipelines/openshift-ci-tests.sh

@@ -15,9 +15,11 @@ source "${DIR}/lib/log.sh"
 export OPENSHIFT_CI="${OPENSHIFT_CI:-false}"
 if [[ -z "${OPENSHIFT_CI}" || "${OPENSHIFT_CI}" == "false" ]]; then
   # NOTE: Use this file to override the environment variables for the local testing.
-  log::debug "Sourcing env_override.local.sh"
-  # shellcheck source=.ibm/pipelines/env_override.local.sh
-  source "${DIR}/env_override.local.sh"
+  if [[ -f "${DIR}/env_override.local.sh" ]]; then
+    log::debug "Sourcing env_override.local.sh"
+    # shellcheck source=.ibm/pipelines/env_override.local.sh
+    source "${DIR}/env_override.local.sh"
+  fi
 fi
 
 log::debug "Sourcing env_variables.sh"

.ibm/pipelines/utils.sh

@@ -9,7 +9,7 @@ retrieve_pod_logs() {
   local pod_name=$1
   local container=$2
   local namespace=$3
-  local log_timeout=${4:-30} # Default timeout: 30 seconds
+  local log_timeout=${4:-5} # Default timeout: 5 seconds (reduced from 30s to speed up failure cases)
   log::debug "Retrieving logs for container: $container"
   # Save logs for the current and previous container with timeout to prevent hanging
   timeout "${log_timeout}" kubectl logs "$pod_name" -c "$container" -n "$namespace" > "pod_logs/${pod_name}_${container}.log" 2> /dev/null || { log::warn "logs for container $container not found or timed out"; }
@@ -680,9 +680,13 @@ deploy_test_backstage_customization_provider() {
 
   # Check if the buildconfig already exists
   if ! oc get buildconfig test-backstage-customization-provider -n "${project}" > /dev/null 2>&1; then
-    log::info "Creating new app for test-backstage-customization-provider"
-    oc new-app -S openshift/nodejs:18-minimal-ubi8
-    oc new-app https://github.com/janus-qe/test-backstage-customization-provider --image-stream="openshift/nodejs:18-ubi8" --namespace="${project}"
+    # Get latest nodejs UBI9 tag from cluster, fallback to 18-ubi8
+    local nodejs_tag
+    nodejs_tag=$(oc get imagestream nodejs -n openshift -o jsonpath='{.spec.tags[*].name}' 2> /dev/null \
+      | tr ' ' '\n' | grep -E '^[0-9]+-ubi9$' | sort -t'-' -k1 -n | tail -1)
+    nodejs_tag="${nodejs_tag:-18-ubi8}"
+    log::info "Creating new app for test-backstage-customization-provider using nodejs:${nodejs_tag}"
+    oc new-app "openshift/nodejs:${nodejs_tag}~https://github.com/janus-qe/test-backstage-customization-provider" --namespace="${project}"
   else
     log::warn "BuildConfig for test-backstage-customization-provider already exists in ${project}. Skipping new-app creation."
   fi
@@ -841,6 +845,33 @@ check_backstage_running() {
     else
       log::warn "Attempt ${i} of ${max_attempts}: Backstage not yet available (HTTP Status: ${http_status})"
       oc get pods -n "${namespace}"
+
+      # Early crash detection: fail fast if RHDH pods are in CrashLoopBackOff
+      # Check both the main deployment and postgresql pods
+      local crash_pods
+      crash_pods=$(oc get pods -n "${namespace}" -l "app.kubernetes.io/instance in (${release_name},redhat-developer-hub,developer-hub,${release_name}-postgresql)" \
+        -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{range .status.containerStatuses[*]}{.state.waiting.reason}{end}{range .status.initContainerStatuses[*]}{.state.waiting.reason}{end}{"\n"}{end}' 2> /dev/null | grep -E "CrashLoopBackOff" || true)
+      # Also check by name pattern for postgresql pods that may have different labels
+      if [ -z "${crash_pods}" ]; then
+        crash_pods=$(oc get pods -n "${namespace}" --no-headers 2> /dev/null | grep -E "(${release_name}|developer-hub|postgresql)" | grep -E "CrashLoopBackOff|Init:CrashLoopBackOff" || true)
+      fi
+
+      if [ -n "${crash_pods}" ]; then
+        log::error "Detected pods in CrashLoopBackOff state - failing fast instead of waiting:"
+        echo "${crash_pods}"
+        log::error "Deployment status:"
+        oc get deployment -l "app.kubernetes.io/instance in (${release_name},redhat-developer-hub,developer-hub)" -n "${namespace}" -o wide 2> /dev/null || true
+        log::error "Recent logs from deployment:"
+        oc logs deployment/${release_name}-developer-hub -n "${namespace}" --tail=100 --all-containers=true 2> /dev/null \
+          || oc logs deployment/${release_name} -n "${namespace}" --tail=100 --all-containers=true 2> /dev/null || true
+        log::error "Recent events:"
+        oc get events -n "${namespace}" --sort-by='.lastTimestamp' | tail -20
+        mkdir -p "${ARTIFACT_DIR}/${namespace}"
+        cp -a "/tmp/${LOGFILE}" "${ARTIFACT_DIR}/${namespace}/" || true
+        save_all_pod_logs "${namespace}"
+        return 1
+      fi
+
       sleep "${wait_seconds}"
     fi
   done
@@ -1320,7 +1351,11 @@ check_and_test() {
   if check_backstage_running "${release_name}" "${namespace}" "${url}" "${max_attempts}" "${wait_seconds}"; then
     echo "Display pods for verification..."
     oc get pods -n "${namespace}"
-    run_tests "${release_name}" "${namespace}" "${playwright_project}" "${url}"
+    if [[ "${SKIP_TESTS:-false}" == "true" ]]; then
+      log::info "SKIP_TESTS=true, skipping test execution for namespace: ${namespace}"
+    else
+      run_tests "${release_name}" "${namespace}" "${playwright_project}" "${url}"
+    fi
   else
     echo "Backstage is not running. Marking deployment as failed and continuing..."
     CURRENT_DEPLOYMENT=$((CURRENT_DEPLOYMENT + 1))