fix(deploy): harden security scan, right-sizing failure tracking, and pod log checks

- Fix Lighthouse security scan exiting with code 1 in warning-only mode
  (set -e kills shell on npm_audit_scan return 1 before handler runs)
- Apply same if/else set -e fix to npm_security_scan for robustness
- Track right-sizing failures across pipe subshell via temp file;
  exit 1 if any deployment fails to stabilize (prevents broken deploys)
- Add --tail=100 and proper quoting to check_pod_logs oc logs call
- Update example.secrets MONITOR_IMAGE to internal OpenShift CLI image

Author: warrenchristian1telus

.github/workflows/build.yml

@@ -776,7 +776,9 @@ jobs:
           source ../../openshift/scripts/utils/npm.sh
 
           # Lighthouse security scan (warnings only - dependencies now installed)
-          lighthouse_security_scan "." "moderate"
+          if ! lighthouse_security_scan "." "moderate"; then
+            echo "⚠️ Lighthouse security scan reported warnings (non-blocking)"
+          fi
 
           cd ../../
           echo ""

example.secrets

@@ -9,7 +9,9 @@ AUTH_TOKEN_DEV=secret.auth.token
 ARTIFACTORY_PULL_SECRET_NAME=artifactory-pull-secret-name--with-credentials-must-exist-in-openshift
 
 # Pod health monitor deployment controls (required)
-MONITOR_IMAGE=artifacts.developer.gov.bc.ca/m950-learning/cron:950003-dev
+# Internal OpenShift CLI image:
+$ Every OCP 4 Silver cluster has the cli image stream pre-installed
+MONITOR_IMAGE=image-registry.openshift-image-registry.svc:5000/openshift/cli:latest
 SERVICE_ACCOUNT=github-actions-sa
 ARTIFACTORY_PULL_SECRET=artifactory-m950-learning
 

openshift/scripts/right-sizing.sh

@@ -51,6 +51,9 @@ source ./openshift/scripts/_utils.sh
 # Initialize utility file arrays for any containerized operations
 initialize_utility_arrays
 
+# Track failures across the pipe subshell
+rm -f /tmp/right-sizing-failures.txt
+
 # Read the CSV file line by line to set deployment resources
 # based on those values
 tail -n +2 ./openshift/${DEPLOY_NAMESPACE}-sizing.csv | while IFS=, read -r Deployment Type PodCount MaxPods PVCCount PVCCapacity CPURequest CPULimit MemRequest MemLimit CPUScaleValue
@@ -65,7 +68,11 @@ do
   if [[ $PodCount -eq 0 ]]; then
     echo "Skipping optional / temporary resource... no pods required to be running."
   else
-    scale_deployment "$Type" "$Deployment" "$PodCount" "$MaxPods"
+    if ! scale_deployment "$Type" "$Deployment" "$PodCount" "$MaxPods"; then
+      echo "❌ $Type/$Deployment failed to stabilize after scaling"
+      # Signal failure to parent shell via temp file (pipe subshell can't set vars)
+      echo "FAILED:$Type/$Deployment" >> /tmp/right-sizing-failures.txt
+    fi
 
     # Check if MaxPods is greater than PodCount before creating the HPA
     if [[ $MaxPods -gt $PodCount ]]; then
@@ -75,3 +82,18 @@ do
     fi
   fi
 done
+
+# Check for any failures that occurred in the pipe subshell
+if [[ -f /tmp/right-sizing-failures.txt ]]; then
+  echo ""
+  echo "❌ RIGHT-SIZING FAILURES DETECTED:"
+  cat /tmp/right-sizing-failures.txt
+  echo ""
+  echo "⚠️  Some resources failed to stabilize after scaling."
+  echo "   The site should NOT exit maintenance mode until these are resolved."
+  rm -f /tmp/right-sizing-failures.txt
+  exit 1
+fi
+
+rm -f /tmp/right-sizing-failures.txt
+echo "✅ Right-sizing completed successfully."

openshift/scripts/utils/npm.sh

@@ -32,8 +32,14 @@ lighthouse_security_scan() {
   local overall_status="PASS"
 
   # Run NPM audit (primary security check)
-  npm_audit_scan "$project_dir" "$audit_level" "audit_result"
-  local audit_exit=$?
+  # Use if/else as it's the only reliable set -e suppression pattern in bash 5.x.
+  # The cmd || var=$? pattern has edge cases with nested function returns.
+  local audit_exit=0
+  if npm_audit_scan "$project_dir" "$audit_level" "audit_result"; then
+    audit_exit=0
+  else
+    audit_exit=$?
+  fi
 
   # Determine status but always continue (warning-only)
   if [ "$audit_result" = "CRITICAL" ]; then
@@ -163,8 +169,13 @@ npm_security_scan() {
   local overall_status="PASS"
 
   # Run NPM audit (now our primary security check)
-  npm_audit_scan "$project_dir" "$audit_level" "audit_result"
-  local audit_exit=$?
+  # Use if/else — the only reliable set -e suppression pattern in bash 5.x.
+  local audit_exit=0
+  if npm_audit_scan "$project_dir" "$audit_level" "audit_result"; then
+    audit_exit=0
+  else
+    audit_exit=$?
+  fi
 
   # Determine overall status based on audit results
   if [ "$audit_result" = "CRITICAL" ]; then

openshift/scripts/utils/openshift.sh

@@ -1326,6 +1326,7 @@ check_pod_logs() {
   local error_search_strings=${3:-"error"}
   local error_handler=${4:-delete_pod}
   local log_file="/tmp/logs/check-pod-logs.log"
+  local tail_lines=${5:-100}  # Only check recent logs to avoid startup noise
 
   # Split the error_search_strings into an array
   IFS=',' read -r -a error_strings <<< "$error_search_strings"
@@ -1341,7 +1342,7 @@ check_pod_logs() {
   IFS=' ' read -r -a container_array <<< "$CONTAINERS"
 
   for container in "${container_array[@]}"; do
-    LOGS=$(oc logs $pod -n $namespace -c $container)
+    LOGS=$(oc logs "$pod" -n "$namespace" -c "$container" --tail="$tail_lines")
 
     for error_search_string in "${error_strings[@]}"; do
       if echo "$LOGS" | grep -q "$error_search_string"; then