Upload safe logs (#1184)

Update safe logs to an internal Google storage bucket

Author: mrodm

.buildkite/hooks/pre-command

@@ -28,10 +28,15 @@ GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
 SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials
 PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader
+PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials
 
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+fi
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})

.buildkite/pipeline.trigger.integration.tests.sh

@@ -56,6 +56,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "      - label: \":go: Running integration test: ${package_name}\""
     echo "        key: \"integration-parallel-${package_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+    echo "        env:"
+    echo "          UPLOAD_SAFE_LOGS: 1"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"

.buildkite/scripts/install_deps.sh

@@ -4,6 +4,10 @@ set -euo pipefail
 
 source .buildkite/scripts/tooling.sh
 
+add_bin_path(){
+    export PATH="${WORKSPACE}/bin:${PATH}"
+}
+
 with_kubernetes() {
     mkdir -p ${WORKSPACE}/bin
     retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
@@ -25,6 +29,7 @@ with_go() {
     eval "$(gvm $(cat .go-version))"
     go version
     which go
+    export PATH="$(go env GOPATH)/bin:${PATH}"
 }
 
 with_docker_compose() {

.buildkite/scripts/integration_tests.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
-
 set -euo pipefail
 
-PARALLEL_TARGET="test-check-packages-parallel"
-KIND_TARGET="test-check-packages-with-kind"
+WORKSPACE="$(pwd)"
+TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
+
+cleanup() {
+    echo "Deleting temporal files..."
+    cd ${WORKSPACE}
+    rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
+    echo "Done."
+}
+trap cleanup EXIT
 
 usage() {
     echo "$0 [-t <target>] [-h]"
@@ -14,6 +21,17 @@ usage() {
 }
 
 source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
+PARALLEL_TARGET="test-check-packages-parallel"
+KIND_TARGET="test-check-packages-with-kind"
+TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
+
+JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
+
+REPO_NAME=$(repoName "${BUILDKITE_REPO}")
+REPO_BUILD_TAG="${REPO_NAME}/${BUILDKITE_BUILD_NUMBER}"
 
 TARGET=""
 PACKAGE=""
@@ -48,14 +66,39 @@ if [[ "${TARGET}" == "" ]]; then
     exit 1
 fi
 
-echo "Current path: $(pwd)"
-WORKSPACE="$(pwd)"
-export PATH="${WORKSPACE}/bin:${PATH}"
-echo "Path: $PATH"
+google_cloud_auth_safe_logs() {
+    local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE})
+    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
+
+    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+
+    google_cloud_auth "${secretFileLocation}"
+
+    echo "${gsUtilLocation}"
+}
+
+upload_safe_logs() {
+    local bucket="$1"
+    local source="$2"
+    local target="$3"
+
+    if ! ls ${source} 2>&1 > /dev/null ; then
+        echo "upload_safe_logs: artifacts files not found, nothing will be archived"
+        return
+    fi
+
+    local gsUtilLocation=$(google_cloud_auth_safe_logs)
+
+    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+
+    rm -rf "${gsUtilLocation}"
+    unset GOOGLE_APPLICATIONS_CREDENTIALS
+}
+
+add_bin_path
 
 echo "--- install go"
 with_go
-export PATH="$(go env GOPATH)/bin:${PATH}"
 
 echo "--- install docker-compose"
 with_docker_compose
@@ -69,6 +112,19 @@ echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
     make install
     make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
+
+    if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then
+        upload_safe_logs \
+            "${JOB_GCS_BUCKET_INTERNAL}" \
+            "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*" \
+            "insecure-logs/${PACKAGE}/"
+
+        upload_safe_logs \
+            "${JOB_GCS_BUCKET_INTERNAL}" \
+            "build/container-logs/*.log" \
+            "insecure-logs/${PACKAGE}/container-logs/"
+    fi
+    make check-git-clean
     exit 0
 fi
 

.buildkite/scripts/signAndPublishPackage.sh

@@ -4,6 +4,9 @@ set -euo pipefail
 WORKSPACE="$(pwd)"
 TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
 
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
 cleanup() {
     echo "Deleting temporal files..."
     cd ${WORKSPACE}
@@ -13,19 +16,6 @@ cleanup() {
 
 trap cleanup EXIT
 
-export PATH="${WORKSPACE}/bin:${PATH}"
-
-echo "Checking gsutil command..."
-if ! command -v gsutil &> /dev/null ; then
-    echo "⚠️  gsutil is not installed"
-    exit 1
-else
-    echo "✅ gsutil is installed"
-fi
-
-source .buildkite/scripts/install_deps.sh
-source .buildkite/scripts/tooling.sh
-
 isAlreadyPublished() {
     local packageZip=$1
 
@@ -37,6 +27,13 @@ isAlreadyPublished() {
     return 1
 }
 
+echo "Checking gsutil command..."
+if ! command -v gsutil &> /dev/null ; then
+    echo "⚠️  gsutil is not installed"
+    exit 1
+fi
+
+
 REPO_NAME=$(repoName "${BUILDKITE_REPO}")
 BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}"
 
@@ -142,6 +139,8 @@ publishPackage() {
     rm -r "${gsUtilLocation}"
 }
 
+add_bin_path
+
 # Required to trigger Jenkins job
 with_go