bk: use monorepo-diff (#612)

v1v · web-flow · commit 0ac2526d49b1 · 2025-05-20T09:25:10.000+02:00
diff --git a/.buildkite/fpm-pipeline.yml b/.buildkite/fpm-pipeline.yml
@@ -6,16 +6,14 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
   MAKEFILE: "fpm"
-  CHANGESET_FILE: ".buildkite/scripts/changeset/fpm"
-  DOCKER_FILTER_REF: "docker.elastic.co/beats-dev"
   BUILDX: "0"
 
 steps:
   - label: ":linux: multiarch Linux x86_64/arm64 FPM docker image"
-    key: "build-and-publish-ubuntu-x86"
+    key: "build-and-publish-ubuntu-x86-fpm"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
     command:
-      - ".buildkite/scripts/llvm-fpm/build_and_publish.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
+      - ".buildkite/scripts/llvm-fpm/build_and_publish.sh ${MAKEFILE}"
     notify:
       - github_commit_status:
           context: "Build FPM / Ubuntu X86_64"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -10,7 +10,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && ( "$BUILDKITE_STEP_KEY" == build* || "$BUILDKITE_STEP_KEY" == release* ) ]]; then
+if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple") && ( "$BUILDKITE_STEP_KEY" == build* || "$BUILDKITE_STEP_KEY" == release* ) ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field=data -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
 fi
 
@@ -21,8 +21,7 @@ if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == "release-post" ]]; then
-    GITHUB_USERNAME_SECRET="elasticmachine"
-    export GITHUB_USERNAME_SECRET=$GITHUB_USERNAME_SECRET
-    export GITHUB_EMAIL_SECRET="elasticmachine@elastic.co"
+    export GITHUB_USERNAME="elasticmachine"
+    export GITHUB_EMAIL="elasticmachine@elastic.co"
     export GITHUB_TOKEN_SECRET=$VAULT_GITHUB_TOKEN
 fi
diff --git a/.buildkite/llvm-apple-pipeline.yml b/.buildkite/llvm-apple-pipeline.yml
@@ -7,16 +7,14 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
   MAKEFILE: "go/llvm-apple"
-  CHANGESET_FILE: ".buildkite/scripts/changeset/llvm-apple"
-  DOCKER_FILTER_REF: "*/*/golang-crossbuild:llvm-apple*"
   BUILDX: "0"
 steps:
   - label: ":linux: Build LLVM Apple / Ubuntu X86_64 - {{matrix.debianVersion}}"
-    key: "build-ubuntu-x86"
+    key: "build-ubuntu-x86-llvm-apple"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
     command:
-      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
-      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
+      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE}"
+      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE}"
     notify:
       - github_commit_status:
           context: "Build LLVM Apple / Ubuntu X86_64"
@@ -35,11 +33,11 @@ steps:
           - "12"
 
   - label: ":linux: Build LLVM Apple / Ubuntu ARM - {{matrix.debianVersion}}"
-    key: "build-ubuntu-arm"
+    key: "build-ubuntu-arm-llvm-apple"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
     command:
-      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
-      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
+      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE}"
+      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE}"
     notify:
       - github_commit_status:
           context: "Build LLVM Apple / Ubuntu ARM"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+              interpolation: false
+              watch:
+                - path:
+                    - .buildkite/pipeline.yml
+                    - .buildkite/fpm-pipeline.yml
+                    - .buildkite/scripts/llvm-fpm
+                    - fpm/**
+                  config:
+                    label: ":pipeline: Upload FPM Pipeline"
+                    command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+                    env:
+                      - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                      - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                      - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+      # Trigger for the main branch only
+      - label: ":pipeline: Upload FPM Pipeline"
+        if: build.branch == 'main'
+        command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+        env:
+          - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+          - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+          - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - group: "llvm-apple"
+    key: "llvm-apple"
+
+    steps:
+      - label: "Trigger llvm-apple-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+              interpolation: false
+              watch:
+                - path:
+                    - .buildkite/pipeline.yml
+                    - .buildkite/llvm-apple-pipeline.yml
+                    - .buildkite/scripts/llvm-apple
+                    - go/llvm-apple/**
+                  config:
+                    label: ":pipeline: Upload FPM Pipeline"
+                    command: "buildkite-agent pipeline upload .buildkite/llvm-apple-pipeline.yml"
+                    env:
+                      - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                      - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                      - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+      # Trigger for the main branch only
+      - label: ":pipeline: Upload llvm-apple Pipeline"
+        if: build.branch == 'main'
+        command: "buildkite-agent pipeline upload .buildkite/llvm-apple-pipeline.yml"
+        env:
+          - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+          - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+          - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
   - group: "Staging"
     key: "staging"
 
diff --git a/.buildkite/scripts/build.sh b/.buildkite/scripts/build.sh
@@ -14,10 +14,11 @@ with_mage
 google_cloud_auth
 
 make -C go -f "${MAKEFILE}" build"${is_arm}" GS_BUCKET_PATH=ingest-buildkite-ci
-echo ":: List Docker images staging ::"
+
+echo "--- List Docker images staging"
 docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${STAGING_IMAGE}/golang-crossbuild"
 
-echo ":: List Docker images production ::"
+echo "--- List Docker images production"
 docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${DOCKER_REGISTRY}/beats-dev/golang-crossbuild"
 
 
diff --git a/.buildkite/scripts/changeset/fpm b/.buildkite/scripts/changeset/fpm
diff --git a/.buildkite/scripts/changeset/llvm-apple b/.buildkite/scripts/changeset/llvm-apple
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
@@ -8,7 +8,6 @@ BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 TMP_FOLDER="tmp.${REPO}"
-GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 
 if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
@@ -92,7 +91,9 @@ retry() {
 }
 
 google_cloud_auth() {
+    echo "running google_cloud_auth"
     local gsUtilLocation=$(mktemp -d -p ${BIN} -t "${TMP_FOLDER}.XXXXXXXXX")
+    GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
     echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
     gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
@@ -146,15 +147,3 @@ check_is_arm() {
     is_arm=""
   fi
 }
-
-are_files_changed() {
-  changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    return 0;
-  else
-    echo "WARN! No files changed in $changeset"
-    return 1;
-  fi
-}
-
diff --git a/.buildkite/scripts/llvm-apple/build.sh b/.buildkite/scripts/llvm-apple/build.sh
@@ -5,16 +5,13 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 makefile=${1}
-patterns=${2}
-docker_filter_ref=${3}
-
-if ! are_files_changed "$patterns" ; then
-    exit 0
-fi
 
 add_bin_path
 with_go "${GOLANG_VERSION}"
 with_mage
+google_cloud_auth
 
 retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
-docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${docker_filter_ref}"
+
+echo "--- List Docker images"
+docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
diff --git a/.buildkite/scripts/llvm-apple/publish.sh b/.buildkite/scripts/llvm-apple/publish.sh
@@ -5,11 +5,6 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 makefile=${1}
-patterns=${2}
-
-if ! are_files_changed "$patterns" ; then
-    exit 0
-fi
 
 add_bin_path
 retry 3 make -C ${makefile} push
diff --git a/.buildkite/scripts/llvm-fpm/build_and_publish.sh b/.buildkite/scripts/llvm-fpm/build_and_publish.sh
@@ -5,16 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 makefile=${1}
-patterns=${2}
-docker_filter_ref=${3}
-
-if ! are_files_changed "$patterns" ; then
-    exit 0
-fi
 
 add_bin_path
 with_go "${GOLANG_VERSION}"
 with_mage
 
 retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
-docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${docker_filter_ref}"
+
+echo "--- List Docker images"
+docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
diff --git a/.buildkite/scripts/post-release.sh b/.buildkite/scripts/post-release.sh
@@ -8,8 +8,8 @@ TAG="v$1"
 TAG_EXISTS=$(tag_Exists ${TAG})
 
 set_git_config() {
-    git config user.name "${GITHUB_USERNAME_SECRET}"
-    git config user.email "${GITHUB_EMAIL_SECRET}"
+    git config user.name "${GITHUB_USERNAME}"
+    git config user.email "${GITHUB_EMAIL}"
 }
 
 tag_commit() {
@@ -19,7 +19,7 @@ tag_commit() {
 
 git_push_with_auth() {
   echo "Pushing tag ${TAG}"
-  retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN_SECRET}@github.com/elastic/golang-crossbuild.git ${TAG}
+  retry 3 git push https://${GITHUB_USERNAME}:${GITHUB_TOKEN_SECRET}@github.com/elastic/golang-crossbuild.git ${TAG}
 }
 
 if [[ "${TAG_EXISTS}" == true ]]; then
diff --git a/catalog-info.yaml b/catalog-info.yaml
@@ -137,7 +137,7 @@ spec:
     spec:
       branch_configuration: "main 1.*" # temporarily disable to build PRs from forks
       pipeline_file: ".buildkite/fpm-pipeline.yml"
-      maximum_timeout_in_minutes: 120
+      maximum_timeout_in_minutes: 360 # cmake is taking at least 4h to run
       provider_settings:
         build_tags: true
         publish_commit_status: true
