ci(serverless): use OIDC for running the serverless ITs (#8263)

v1v · web-flow · commit 1b7a4fafb618 · 2025-06-16T09:42:00.000+02:00
diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml
@@ -13,6 +13,31 @@ env:
   IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749862860"
   IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749862860"
 
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - google_oidc_plugin: &google_oidc_plugin
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      elastic/oblt-google-auth#v1.3.0:
+        lifetime: 10800 # seconds
+        project-id: "elastic-observability-ci"
+        project-number: "911195782929"
+# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/10
+# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/11
+#  - gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
+      #avaly/gcp-secret-manager#v1.2.0:
+  - gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
+      elastic/gcp-secret-manager#v1.3.0-elastic:
+        env:
+          # These secrets are created in .github/workflows/serverless-project.yml
+          ELASTICSEARCH_HOST: ea-serverless-it-elasticsearch-hostname
+          ELASTICSEARCH_PASSWORD: ea-serverless-it-elasticsearch-password
+          ELASTICSEARCH_USERNAME: ea-serverless-it-elasticsearch-username
+          KIBANA_HOST: ea-serverless-it-kibana-hostname
+          KIBANA_USERNAME: ea-serverless-it-kibana-username
+          KIBANA_PASSWORD: ea-serverless-it-kibana-password
+
 steps:
   - label: Start ESS stack for integration tests
     key: integration-ess
@@ -416,6 +441,76 @@ steps:
               - v1.32.0
               - v1.33.0
 
+  - group: "Serverless integration test"
+    key: integration-tests-serverless
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
+    steps:
+      - label: "Windows:2022:amd64:sudo"
+        depends_on:
+          - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/buildkite-integration-tests.ps1 fleet true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          image: "${IMAGE_WIN_2022}"
+        plugins:
+          - *google_oidc_plugin
+          - *gcp_serverless_secrets_plugin
+
+      - label: "Windows:2025:amd64:sudo"
+        depends_on:
+          - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/buildkite-integration-tests.ps1 fleet true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          image: "${IMAGE_WIN_2025}"
+        plugins:
+          - *google_oidc_plugin
+          - *gcp_serverless_secrets_plugin
+      - label: "Ubuntu:2404:amd64:sudo"
+        depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
+          sudo -E .buildkite/scripts/buildkite-integration-tests.sh fleet true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *google_oidc_plugin
+          - *gcp_serverless_secrets_plugin
+
   - label: ESS stack cleanup
     depends_on:
       - integration-tests-ubuntu
@@ -440,6 +535,7 @@ steps:
       - integration-tests-win
       - integration-tests-rhel8
       - integration-tests-kubernetes
+      - integration-tests-serverless
     allow_dependency_failure: true
     command: |
       buildkite-agent artifact download "build/*.xml" .
diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -147,31 +147,6 @@ steps:
           imagePrefix: "core-ubuntu-2204-aarch64"
           diskSizeGb: 200
 
-  - label: "Serverless integration test"
-    key: "serverless-integration-tests"
-    depends_on:
-      - int-packaging
-    concurrency_group: elastic-agent-extended-testing/serverless-integration
-    concurrency: 8
-    env:
-      # we run each step in a different data center to spread the load
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: |
-      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
-      .buildkite/scripts/steps/integration_tests.sh serverless integration:testServerless
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n2-standard-8"
-    retry:
-      automatic:
-        limit: 1
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
-
   - label: "Triggering Integration tests"
     depends_on:
       - int-packaging
diff --git a/.github/workflows/serverless-project.yml b/.github/workflows/serverless-project.yml
@@ -0,0 +1,104 @@
+---
+name: serverless-project
+
+on:
+  workflow_dispatch:
+  schedule:
+    # To run more often if needed, for now daily at 4:00 UTC
+    - cron: "0 4 * * *"
+
+permissions:
+  contents: read
+
+jobs:
+  create-serverless:
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    env:
+      PREFIX: "ea-serverless-it"
+    steps:
+      ####################################
+      # 1. Create the serverless project
+      ####################################
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "checks": "read",
+              "contents": "write",
+              "pull_requests": "write"
+            }
+          repositories: >-
+            ["observability-test-environments"]
+
+      - uses: elastic/oblt-actions/git/setup@v1
+        with:
+          github-token: ${{ steps.get_token.outputs.token }}
+
+      - name: Get day of the week
+        id: get_day
+        run: echo "day=$(date +'%a' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
+
+      - uses: elastic/oblt-actions/oblt-cli/cluster-create-custom@v1
+        id: create_serverless
+        with:
+          template: 'serverless-ea-it'
+          parameters: '{"Target":"production","ProjectType":"observability"}'
+          cluster-name-prefix: "${{ env.PREFIX }}-${{ steps.get_day.outputs.day }}"
+          github-token: ${{ steps.get_token.outputs.token }}
+          gitops: true
+          wait: '15'
+
+      # Authenticate to the elastic-observability to get the cluster credentials
+      - uses: elastic/oblt-actions/google/auth@v1
+
+      - uses: elastic/oblt-actions/oblt-cli/cluster-credentials@v1
+        with:
+          cluster-name: ${{ steps.create_serverless.outputs.cluster-name }}
+          github-token: ${{ steps.get_token.outputs.token }}
+
+      - name: Smoke test
+        run: curl -X GET ${ELASTICSEARCH_HOST}/_cat/indices?v -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}
+
+      ####################################
+      # 2. Copy the serverless secrets
+      ####################################
+      # Authenticate to the elastic-observability-ci to rotate the cluster credentials
+      - uses: elastic/oblt-actions/google/auth@v1
+        with:
+          project-number: "911195782929"
+          project-id: "elastic-observability-ci"
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a
+
+      # TODO: as soon as the oblt-framework supports elastic-observability-ci we can avoid this step.
+      # NOTE:
+      #   * While runnning this workflow, it might cause some hiccups if a PR runs when rotating the secrets
+      #   * Secrets need to be created firstly. gcloud secrets create otherwise gcloud secrets versions add will fail.
+      #     That's not an issue now, as we use the same secret name.
+      - name: Rotate GCSM secrets
+        env:
+          GCP_PROJECT: "elastic-observability-ci"
+        run: |
+          echo -n "${ELASTICSEARCH_HOST}" | gcloud secrets versions add "${PREFIX}-elasticsearch-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
+          echo -n "${ELASTICSEARCH_PASSWORD}" | gcloud secrets versions add "${PREFIX}-elasticsearch-password" --data-file=- --quiet --project "${GCP_PROJECT}"
+          echo -n "${ELASTICSEARCH_USERNAME}" | gcloud secrets versions add "${PREFIX}-elasticsearch-username" --data-file=- --quiet --project "${GCP_PROJECT}"
+          echo -n "${KIBANA_HOST}" | gcloud secrets versions add "${PREFIX}-kibana-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
+          echo -n "${KIBANA_USERNAME}" | gcloud secrets versions add "${PREFIX}-kibana-username" --data-file=- --quiet --project "${GCP_PROJECT}"
+          echo -n "${KIBANA_PASSWORD}" | gcloud secrets versions add "${PREFIX}-kibana-password" --data-file=- --quiet --project "${GCP_PROJECT}"
+
+      # TODO: if rotation fails then rollback to the previous cluster.
+      - if: ${{ failure()  }}
+        uses: elastic/oblt-actions/slack/send@v1
+        env:
+          JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: "#ingest-notifications"
+          message: ":traffic_cone: serverless project creation failed for `${{ github.repository }}@${{ github.ref_name }}`, `@robots-ci` please look what's going on <${{ env.JOB_URL }}|here>"
diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md
@@ -28,6 +28,17 @@ The Elastic Agent package that is used for integration tests packages Beats buil
 ESS (production) API Key to create on <https://cloud.elastic.co/account/keys>
 Warning: if you never created a deployment on it, you won't have permission to get this key, so you will need to create one first.
 
+
+#### Setup Serverless deployment
+
+This process is now automated and runs daily, utilizing the existing `oblt-cli` framework. Serverless deployments are created each day and automatically destroyed every three days.
+
+The automation is configured in the `serverless-project.yml` file located in the `.github/workflows` directory.
+
+If necessary, you can create a new serverless deployment manually; the previous deployments will be destroyed automatically, but not immediately. To do so, you need to run the GitHub action called [serverless-project.yml](https://github.com/elastic/elastic-agent/actions/workflows/serverless-project.yml).
+
+Credentials for these deployments are securely stored in Google and can only be accessed by Buildkite pipelines. The access control is set using [OpenID Connect in Google Cloud Platform](https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-google-cloud-platform). And that's managed by the Robots team.
+
 ## Running tests
 
 Some integration and E2E tests are safe to run locally. These tests set
