Review buildkite scripts cleanup process (#1487)

Review buildkite scripts and hooks to ensure that the cleanup
process performs all the needed operations

Author: mrodm

.buildkite/hooks/pre-command

@@ -1,27 +1,8 @@
 #!/bin/bash
 
+source .buildkite/scripts/tooling.sh
 set -euo pipefail
 
-function retry {
-    local retries=$1
-    shift
-
-    local count=0
-    until "$@"; do
-        exit=$?
-        wait=$((2 ** count))
-        count=$((count + 1))
-        if [ $count -lt "$retries" ]; then
-            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-            sleep $wait
-        else
-            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
-            return $exit
-        fi
-    done
-    return 0
-}
-
 export GO_VERSION=$(cat .go-version)
 
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account

.buildkite/hooks/pre-exit

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+source .buildkite/scripts/tooling.sh
+
+set -euo pipefail
+
+unset_secrets
+
+# integrations-parallel-gcp
+unset GOOGLE_CREDENTIALS
+unset GCP_PROJECT_ID
+
+# integrations-parallel-aws and intregrations-parallel-aws_logs
+unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+unset AWS_ACCESS_KEY_ID
+unset AWS_SECRET_ACCESS_KEY

.buildkite/scripts/integration_tests.sh

@@ -5,10 +5,21 @@ WORKSPACE="$(pwd)"
 TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
 
 cleanup() {
+    local error_code=$?
+
+    if [ $error_code != 0 ] ; then
+        # if variable is defined run the logout
+        if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+             google_cloud_logout_active_account
+        fi
+    fi
+
     echo "Deleting temporal files..."
     cd ${WORKSPACE}
     rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
     echo "Done."
+
+    exit $error_code
 }
 trap cleanup EXIT
 
@@ -67,14 +78,12 @@ if [[ "${TARGET}" == "" ]]; then
 fi
 
 google_cloud_auth_safe_logs() {
-    local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE})
+    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
 
     echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
 
     google_cloud_auth "${secretFileLocation}"
-
-    echo "${gsUtilLocation}"
 }
 
 upload_safe_logs() {
@@ -87,12 +96,11 @@ upload_safe_logs() {
         return
     fi
 
-    local gsUtilLocation=$(google_cloud_auth_safe_logs)
+    google_cloud_auth_safe_logs
 
     gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
 
-    rm -rf "${gsUtilLocation}"
-    unset GOOGLE_APPLICATION_CREDENTIALS
+    google_cloud_logout_active_account
 }
 
 add_bin_path

.buildkite/scripts/signAndPublishPackage.sh

@@ -8,10 +8,21 @@ source .buildkite/scripts/install_deps.sh
 source .buildkite/scripts/tooling.sh
 
 cleanup() {
+    local error_code=$?
+
+    if [ $error_code != 0 ] ; then
+        # if variable is defined, run the logout
+        if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+             google_cloud_logout_active_account
+        fi
+    fi
+
     echo "Deleting temporal files..."
     cd ${WORKSPACE}
     rm -rf ${TMP_FOLDER_TEMPLATE_BASE}.*
     echo "Done."
+
+    exit $error_code
 }
 
 trap cleanup EXIT
@@ -55,32 +66,28 @@ PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH="gs://elastic-bekitzur-pac
 
 
 google_cloud_auth_signing() {
-    local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE})
+    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
 
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
     echo "${SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
 
     google_cloud_auth "${secretFileLocation}"
-
-    echo "${gsUtilLocation}"
 }
 
 google_cloud_auth_publishing() {
-    local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE})
+    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
 
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
     echo "${PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
 
     google_cloud_auth "${secretFileLocation}"
-
-    echo "${gsUtilLocation}"
 }
 
 sign_package() {
     local package=${1}
     local packageZip=$(basename ${package})
 
-    local gsUtilLocation=$(google_cloud_auth_signing)
+    google_cloud_auth_signing
 
     # upload zip package (trailing forward slashes are required)
     echo "Upload package .zip file for signing ${package} to ${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}"
@@ -106,16 +113,15 @@ sign_package() {
 
     ls -l "${BUILD_PACKAGES_PATH}"
 
-    echo "Removing temporal location ${gsUtilLocation}"
-    rm -r "${gsUtilLocation}"
+    google_cloud_logout_active_account
 }
 
 publish_package() {
     local package=$1
     local packageZip=$(basename ${package})
 
     # create file with credentials
-    local gsUtilLocation=$(google_cloud_auth_publishing)
+    google_cloud_auth_publishing
 
     # upload files (trailing forward slashes are required)
     echo "Upload package .zip file ${package} to ${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}"
@@ -135,8 +141,7 @@ publish_package() {
 
     popd > /dev/null
 
-    echo "Removing temporal location ${gsUtilLocation}"
-    rm -r "${gsUtilLocation}"
+    google_cloud_logout_active_account
 }
 
 add_bin_path

.buildkite/scripts/tooling.sh

@@ -1,6 +1,14 @@
 #!/bin/bash
 set -euo pipefail
 
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+        unset "$var"
+    fi
+  done
+}
+
 repo_name() {
     # Example of URL: [email protected]:acme-inc/my-project.git
     local repoUrl=$1
@@ -44,3 +52,17 @@ retry() {
     done
     return 0
 }
+
+google_cloud_logout_active_account() {
+  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
+  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
+    echo "Logging out from GCP for active account"
+    gcloud auth revoke $active_account > /dev/null 2>&1
+  else
+    echo "No active GCP accounts found."
+  fi
+  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+    rm -rf ${GOOGLE_APPLICATION_CREDENTIALS}
+    unset GOOGLE_APPLICATION_CREDENTIALS
+  fi
+}