Destroy CI clusters when there is no active investigation

To prevent too many resources from being consumed, keeping the cluster
is not the default action.
If a developer wants to investigate, they need to connect to a pod
and remove the 'destroy_cluster.txt' file located at the root of
the source directory.

Signed-off-by: Romain Arnaud <[email protected]>

Author: Roming22

.tekton/pipeline/acceptance-tests.yaml

@@ -115,12 +115,6 @@ spec:
         - input: "$(tasks.deploy-cluster.status)"
           operator: notin
           values: ["None"]
-        - input: "$(tasks.plnsvc-setup.status)"
-          operator: notin
-          values: ["Failed"]
-        - input: "$(tasks.pipeline-service-tests.status)"
-          operator: notin
-          values: ["Failed"]
       params:
         - name: cluster-name
           value: "$(tasks.generate-cluster-name.results.cluster-name)"

.tekton/pipeline/upgrade-tests.yaml

@@ -188,24 +188,6 @@ spec:
         - input: "$(tasks.deploy-cluster.status)"
           operator: notin
           values: ["None"]
-        - input: "$(tasks.plnsvc-setup.status)"
-          operator: notin
-          values: ["Failed"]
-        - input: "$(tasks.tests-before-upgrade.status)"
-          operator: notin
-          values: ["Failed"]
-        - input: "$(tasks.plnsvc-upgrade-setup.status)"
-          operator: notin
-          values: ["Failed"]
-        - input: "$(tasks.tests-post-upgrade.status)"
-          operator: notin
-          values: ["Failed"]
-        - input: "$(tasks.plnsvc-downgrade-setup.status)"
-          operator: notin
-          values: ["Failed"]
-        - input: "$(tasks.tests-post-downgrade.status)"
-          operator: notin
-          values: ["Failed"]
       params:
         - name: cluster-name
           value: "$(tasks.generate-cluster-name.results.cluster-name)"

.tekton/tasks/destroy-cluster.yaml

@@ -33,5 +33,6 @@ spec:
             secretKeyRef:
               name: hypershift-bitwarden
               key: "BW_PASSWORD"
+      workingDir: "$(workspaces.source.path)"
       command:
         - $(workspaces.source.path)/ci/images/ci-runner/hack/bin/destroy-cluster.sh

ci/docs/continuous_integation.md

@@ -22,3 +22,10 @@ After that, you need to add a Configmap for storing the kubeconfig of ROSA clust
 ## Setup GitHub app
 
 - Need to configure GitHub app for Pipelines as Code configuration into Pipeline Service repository.
+
+## Debugging an issue during the CI execution
+The CI will destroy the test cluster at the end of the pipeline by default.
+This is an issue when troubleshooting is required.
+
+To bypass deletion of the test cluster, delete the `destroy-cluster.txt` file in created in the root of the cloned repository.
+There is a 15 minutes window to log onto the container running the `destroy-cluster` step and delete the file.

ci/images/ci-runner/hack/bin/destroy-cluster.sh

@@ -9,6 +9,18 @@ SCRIPT_DIR="$(
   pwd
 )"
 
+# Give developers 15mins to connect to a pod and remove the file
+# if they want to investigate the failure
+if [ -e "$PWD/destroy-cluster.txt" ]; then
+    sleep 900
+    if [ -e "$PWD/destroy-cluster.txt" ]; then
+      echo "Failure is not being investigated, cluster will be destroyed."
+    else
+      echo "Failure under investigation, cluster will not be destroyed."
+      exit 1
+    fi
+fi
+
 # shellcheck source=ci/images/ci-runner/hack/bin/utils.sh
 source "$SCRIPT_DIR/utils.sh"
 

ci/images/ci-runner/hack/bin/run-plnsvc-setup.sh

@@ -4,6 +4,9 @@ set -o nounset
 set -o pipefail
 set -x
 
+# Create a file that will prevent the cluster deletion in case tests are failing
+touch "$PWD/destroy-cluster.txt"
+
 echo "Execute dev_setup.sh script to set up pipeline-service ..."
 kubectl -n default exec pod/ci-runner -- \
     sh -c "/source/ci/images/ci-runner/hack/sidecar/bin/plnsvc_setup.sh $REPO_URL $REPO_REVISION"

ci/images/ci-runner/hack/bin/run-tests.sh

@@ -7,3 +7,6 @@ set -x
 echo "Execute test.sh script to verify pipeline-service ..."
 kubectl -n default exec pod/ci-runner -- \
     sh -c "/source/ci/images/ci-runner/hack/sidecar/bin/plnsvc_test.sh"
+
+# If the tests are successful, the cluster can be destroyed right away
+rm "$PWD/destroy-cluster.txt"