ci: reproducible builds using pinned VM images (#8211)

(cherry picked from commit 9259f35)

Author: v1v

.buildkite/bk.integration.pipeline.yml

@@ -5,12 +5,14 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   ASDF_MAGE_VERSION: 1.14.0
 
-  IMAGE_UBUNTU_2404_X86_64: "family/platform-ingest-elastic-agent-ubuntu-2404"
-  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64"
-  IMAGE_RHEL_8: "family/platform-ingest-elastic-agent-rhel-8"
-  IMAGE_DEBIAN_12: "family/platform-ingest-elastic-agent-debian-12"
-  IMAGE_WIN_2022: "family/platform-ingest-elastic-agent-windows-2022"
-  IMAGE_WIN_2025: "family/platform-ingest-elastic-agent-windows-2025"
+  # The following images are defined here and their values will be updated by updatecli
+  # Please do not change them manually.
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1748347205"
+  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1748347205"
+  IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1748347205"
+  IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1748347205"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1748347205"
+  IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1748347205"
 
 steps:
   - label: Start ESS stack for integration tests
@@ -249,7 +251,7 @@ steps:
           - build/diagnostics/**
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_2404_ARM_64}"
+          image: "${IMAGE_UBUNTU_2404_ARM_64}"
           instanceType: "m6g.2xlarge"
         retry:
           automatic:
@@ -286,7 +288,7 @@ steps:
             limit: 1
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_2404_ARM_64}"
+          image: "${IMAGE_UBUNTU_2404_ARM_64}"
           instanceType: "m6g.xlarge"
         matrix:
           - default

.buildkite/pipeline.yml

@@ -4,12 +4,14 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 
-  IMAGE_UBUNTU_2204_X86_64: "family/platform-ingest-elastic-agent-ubuntu-2204"
-  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64"
-  IMAGE_WIN_2016: "family/platform-ingest-elastic-agent-windows-2016"
-  IMAGE_WIN_2022: "family/platform-ingest-elastic-agent-windows-2022"
-  IMAGE_WIN_10: "family/platform-ingest-elastic-agent-windows-10"
-  IMAGE_WIN_11: "family/platform-ingest-elastic-agent-windows-11"
+  # The following images are defined here and their values will be updated by updatecli
+  # Please do not change them manually.
+  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1748347205"
+  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1748347205"
+  IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1748347205"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1748347205"
+  IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1748347205"
+  IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1748347205"
 
 steps:
   - label: "check-ci"
@@ -52,7 +54,7 @@ steps:
           - "coverage-*.out"
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_2204_ARM_64}"
+          image: "${IMAGE_UBUNTU_2204_ARM_64}"
           diskSizeGb: 200
           instanceType: "m6g.xlarge"
         retry:

.ci/updatecli/updatecli-bump-vm-images.yml

@@ -0,0 +1,65 @@
+# update-cli configuration for automated VM image version bumping
+---
+name: Bump vm-images to latest version
+
+scms:
+  githubConfig:
+    kind: github
+    spec:
+      user: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      owner: '{{ .scm.owner }}'
+      repository: '{{ .scm.repository }}'
+      token: '{{ requiredEnv "GITHUB_TOKEN" }}'
+      commitusingapi: true
+      branch: '{{ .scm.branch }}'
+      force: false
+
+actions:
+  elastic-agent:
+    kind: github/pullrequest
+    scmid: githubConfig
+    sourceid: latestGoVersion
+    spec:
+      automerge: false
+      labels:
+        - dependencies
+        - backport-active-all
+      title: '[Automation] Bump VM Image version to {{ source "latestVersion" }}'
+
+sources:
+  latestVersion:
+    name: Get latest available build
+    kind: json
+    spec:
+      file: https://storage.googleapis.com/artifacts-api/vm-images/elastic-agent/latest.json
+      key: .date
+
+conditions:
+  latestVersion-check:
+    name: Check if defined latest version differs
+    kind: shell
+    sourceid: latestVersion
+    spec:
+      command: 'grep -q -v {{ source "latestVersion" }} .buildkite/pipeline.yml #'
+
+targets:
+  update-buildkite-pipeline:
+    name: "Update .buildkite/pipeline.yml"
+    sourceid: latestVersion
+    scmid: githubConfig
+    kind: file
+    spec:
+      file: .buildkite/pipeline.yml
+      matchpattern: '(IMAGE_.+): "platform-ingest-elastic-agent-(.+)-(.+)"'
+      replacepattern: '$1: "platform-ingest-elastic-agent-$2-{{ source "latestVersion" }}"'
+
+  update-buildkite-bk.integration.pipeline:
+    name: "Update .buildkite/bk.integration.pipeline.yml"
+    sourceid: latestVersion
+    scmid: githubConfig
+    kind: file
+    spec:
+      file: .buildkite/bk.integration.pipeline.yml
+      matchpattern: '(IMAGE_.+): "platform-ingest-elastic-agent-(.+)-(.+)"'
+      replacepattern: '$1: "platform-ingest-elastic-agent-$2-{{ source "latestVersion" }}"'

.github/workflows/bump-vm-images.yml

@@ -0,0 +1,51 @@
+---
+name: bump-vm-images
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 10 * * 0"
+
+permissions:
+  contents: read
+
+env:
+  JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+jobs:
+  bump:
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: elastic/oblt-actions/updatecli/run@v1
+        with:
+          command: apply --config .ci/updatecli/updatecli-bump-vm-images.yml --values .ci/updatecli/values.d/scm.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - if: ${{ failure()  }}
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
+        with:
+          method: chat.postMessage
+          token: ${{ secrets.SLACK_BOT_TOKEN }}
+          payload: |
+            {
+              "channel": "#ingest-notifications",
+              "text": "${{ env.SLACK_MESSAGE }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "${{ env.SLACK_MESSAGE }}"
+                  }
+                }
+              ]
+            }
+        env:
+          #SLACK_MESSAGE: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@agent-team` please look what's going on <${{ env.JOB_URL }}|here>"
+          SLACK_MESSAGE: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`"