test: Fix kubernetes cleanup in the test environment to avoid hanging tests.

Author: hessjcg

Makefile

@@ -372,17 +372,15 @@ e2e_test_run_gotest:  # Run the golang e2e tests
 
 .PHONY: e2e_cleanup_test_namespaces
 e2e_cleanup_test_namespaces: e2e_project kustomize kubectl # remove e2e test namespaces named "test*"
-	( $(E2E_KUBECTL) get ns -o=name | \
-		grep namespace/test | \
-		$(E2E_KUBECTL_ENV) xargs $(KUBECTL) delete ) || true
-	( $(E2E_PRIVATE_KUBECTL) get ns -o=name | \
-		grep namespace/test | \
-		$(E2E_PRIVATE_KUBECTL_ENV) xargs $(KUBECTL) delete ) || true
+	$(E2E_PRIVATE_KUBECTL_ENV) KUBECTL=$(KUBECTL) $(PWD)/tools/delete-test-namespaces.sh
+	$(E2E_KUBECTL_ENV) KUBECTL=$(KUBECTL) $(PWD)/tools/delete-test-namespaces.sh
 
 .PHONY: e2e_undeploy
 e2e_undeploy: e2e_project kustomize kubectl $(E2E_WORK_DIR) # Remove the operator from the GKE cluster
-	$(E2E_KUBECTL) delete -f $(E2E_WORK_DIR)/operator.yaml
-	$(E2E_PRIVATE_KUBECTL) delete -f $(E2E_WORK_DIR)/operator.yaml
+	$(E2E_KUBECTL) delete -f $(E2E_WORK_DIR)/operator.yaml --timeout=30s || true
+	$(E2E_KUBECTL_ENV) KUBECTL=$(KUBECTL) $(PWD)/tools/delete-test-namespaces.sh namespace/cloud-sql-proxy-operator-system
+	$(E2E_PRIVATE_KUBECTL) delete -f $(E2E_WORK_DIR)/operator.yaml --timeout=30s || true
+	$(E2E_PRIVATE_KUBECTL_ENV) KUBECTL=$(KUBECTL) $(PWD)/tools/delete-test-namespaces.sh namespace/cloud-sql-proxy-operator-system
 
 ###
 # Build the operator docker image and push it to the

infra/permissions/main.tf

@@ -76,6 +76,12 @@ resource "google_project_iam_member" "allow_image_pull" {
   role       = "roles/artifactregistry.reader"
   member     = "serviceAccount:${google_service_account.node_pool.email}"
 }
+resource "google_project_iam_member" "default_node_service_acct" {
+  depends_on = [google_project_service.project["iam.googleapis.com"]]
+  project    = var.project_id
+  role       = "roles/container.defaultNodeServiceAccount"
+  member     = "serviceAccount:${google_service_account.node_pool.email}"
+}
 
 resource "google_project_iam_binding" "cloud_sql_client" {
   depends_on = [google_project_service.project["iam.googleapis.com"]]

tools/delete-test-namespaces.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+KUBECTL=${KUBECTL:-bin/kubectl}
+export USE_GKE_E2E_AUTH_PLUGIN=True
+export KUBECONFIG=${KUBECONFIG:-bin/e2e-kubeconfig.yaml}
+
+mkdir -p bin/ns
+function remove_ns(){
+  # Check that the namespace exists, return if not.
+  if ! $KUBECTL get namespace "$1" ; then
+    return
+  fi
+
+  # Tell kubernetes to delete the namespace, If it times out, force delete.
+  if ! $KUBECTL delete namespace "$1" --timeout=10s ; then
+
+    # Get the namespace, remove finalizers from the namespace spec.
+    $KUBECTL get namespace "$1" -o json | \
+      jq '.spec.finalizers = []' > "bin/ns/$1.json"
+
+    # Force update the namespace resource, removing finalizers.
+    # This will allow Kubernetes to continue the deletion of the resource.
+    $KUBECTL replace --raw "/api/v1/namespaces/$1/finalize" -f "bin/ns/$1.json"
+  fi
+
+}
+
+
+if [[ ${#@} -gt 0 ]] ; then
+  remove_ns "$1"
+else
+  namespaces=( $( $KUBECTL get ns -o=name | grep namespace/test ) )
+  for ns in ${namespaces[*]} ; do
+    ns="${ns#*/}" # remove "namespace/" from the beginning of the string
+    echo "Deleting $ns"
+    remove_ns "$ns"
+  done
+fi

tools/e2e_test_job.sh

@@ -58,6 +58,11 @@ set -x
 echo "TIME: $(date) Run Tests"
 echo "Running tests on environment ${ENVIRONMENT_NAME:-undefined}"
 
+# Force cleanup of the existing resources before running tests.
+echo "Cleaning up the test resources from past runs"
+make e2e_test_clean >> bin/e2e_test.log || true
+
+echo "Starting the tests"
 # Run e2e test, filtering the stdout so that it only logs go test results.
 if make e2e_test_job > bin/e2e_test.log 2>&1 ; then
   echo "STATUS: E2E Test Passed"