Skip to content

Commit 285d177

Browse files
mergify[bot]mergify[bot]
authored
[8.17] (backport #8263) ci(serverless): use OIDC for running the serverless ITs (#8515)
1 parent 92582bc commit 285d177

File tree

4 files changed

+211
-25
lines changed

4 files changed

+211
-25
lines changed

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,31 @@ env:
1313
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749258065"
1414
IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749258065"
1515

16+
# This section is used to define the plugins that will be used in the pipeline.
17+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
18+
common:
19+
- google_oidc_plugin: &google_oidc_plugin
20+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
21+
# This plugin authenticates to Google Cloud using the OIDC token.
22+
elastic/oblt-google-auth#v1.3.0:
23+
lifetime: 10800 # seconds
24+
project-id: "elastic-observability-ci"
25+
project-number: "911195782929"
26+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/10
27+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/11
28+
# - gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
29+
#avaly/gcp-secret-manager#v1.2.0:
30+
- gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
31+
elastic/gcp-secret-manager#v1.3.0-elastic:
32+
env:
33+
# These secrets are created in .github/workflows/serverless-project.yml
34+
ELASTICSEARCH_HOST: ea-serverless-it-elasticsearch-hostname
35+
ELASTICSEARCH_PASSWORD: ea-serverless-it-elasticsearch-password
36+
ELASTICSEARCH_USERNAME: ea-serverless-it-elasticsearch-username
37+
KIBANA_HOST: ea-serverless-it-kibana-hostname
38+
KIBANA_USERNAME: ea-serverless-it-kibana-username
39+
KIBANA_PASSWORD: ea-serverless-it-kibana-password
40+
1641
steps:
1742
- label: Start ESS stack for integration tests
1843
key: integration-ess
@@ -427,6 +452,76 @@ steps:
427452
- v1.30.8
428453
- v1.31.0
429454

455+
- group: "Serverless integration test"
456+
key: integration-tests-serverless
457+
notify:
458+
- github_commit_status:
459+
context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
460+
steps:
461+
- label: "Windows:2022:amd64:sudo"
462+
depends_on:
463+
- packaging-windows
464+
env:
465+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
466+
command: |
467+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
468+
.buildkite/scripts/buildkite-integration-tests.ps1 fleet true
469+
artifact_paths:
470+
- build/**
471+
- build/diagnostics/**
472+
retry:
473+
automatic:
474+
limit: 1
475+
agents:
476+
provider: "gcp"
477+
machineType: "n2-standard-8"
478+
image: "${IMAGE_WIN_2022}"
479+
plugins:
480+
- *google_oidc_plugin
481+
- *gcp_serverless_secrets_plugin
482+
483+
- label: "Windows:2025:amd64:sudo"
484+
depends_on:
485+
- packaging-windows
486+
env:
487+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
488+
command: |
489+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
490+
.buildkite/scripts/buildkite-integration-tests.ps1 fleet true
491+
artifact_paths:
492+
- build/**
493+
- build/diagnostics/**
494+
retry:
495+
automatic:
496+
limit: 1
497+
agents:
498+
provider: "gcp"
499+
machineType: "n2-standard-8"
500+
image: "${IMAGE_WIN_2025}"
501+
plugins:
502+
- *google_oidc_plugin
503+
- *gcp_serverless_secrets_plugin
504+
- label: "Ubuntu:2404:amd64:sudo"
505+
depends_on: packaging-ubuntu-x86-64
506+
env:
507+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
508+
command: |
509+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
510+
sudo -E .buildkite/scripts/buildkite-integration-tests.sh fleet true
511+
artifact_paths:
512+
- build/**
513+
- build/diagnostics/**
514+
retry:
515+
automatic:
516+
limit: 1
517+
agents:
518+
provider: "gcp"
519+
machineType: "n2-standard-8"
520+
image: "${IMAGE_UBUNTU_2404_X86_64}"
521+
plugins:
522+
- *google_oidc_plugin
523+
- *gcp_serverless_secrets_plugin
524+
430525
- label: ESS stack cleanup
431526
depends_on:
432527
- integration-tests-ubuntu
@@ -451,6 +546,7 @@ steps:
451546
- integration-tests-win
452547
- integration-tests-rhel8
453548
- integration-tests-kubernetes
549+
- integration-tests-serverless
454550
allow_dependency_failure: true
455551
command: |
456552
buildkite-agent artifact download "build/*.xml" .

‎.buildkite/integration.pipeline.yml‎

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -87,31 +87,6 @@ steps:
8787
imagePrefix: "core-ubuntu-2204-aarch64"
8888
diskSizeGb: 200
8989

90-
- label: "Serverless integration test"
91-
key: "serverless-integration-tests"
92-
depends_on:
93-
- int-packaging
94-
concurrency_group: elastic-agent-extended-testing/serverless-integration
95-
concurrency: 8
96-
env:
97-
# we run each step in a different data center to spread the load
98-
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
99-
command: |
100-
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
101-
.buildkite/scripts/steps/integration_tests.sh serverless integration:testServerless
102-
artifact_paths:
103-
- "build/TEST-**"
104-
- "build/diagnostics/*"
105-
agents:
106-
provider: "gcp"
107-
machineType: "n2-standard-8"
108-
retry:
109-
automatic:
110-
limit: 1
111-
notify:
112-
- github_commit_status:
113-
context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
114-
11590
- label: "Triggering Integration tests"
11691
depends_on:
11792
- int-packaging

‎.github/workflows/serverless-project.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
---
2+
name: serverless-project
3+
4+
on:
5+
workflow_dispatch:
6+
schedule:
7+
# To run more often if needed, for now daily at 4:00 UTC
8+
- cron: "0 4 * * *"
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
create-serverless:
15+
permissions:
16+
id-token: write
17+
runs-on: ubuntu-latest
18+
env:
19+
PREFIX: "ea-serverless-it"
20+
steps:
21+
####################################
22+
# 1. Create the serverless project
23+
####################################
24+
- name: Get token
25+
id: get_token
26+
uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
27+
with:
28+
app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
29+
private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
30+
permissions: >-
31+
{
32+
"checks": "read",
33+
"contents": "write",
34+
"pull_requests": "write"
35+
}
36+
repositories: >-
37+
["observability-test-environments"]
38+
39+
- uses: elastic/oblt-actions/git/setup@v1
40+
with:
41+
github-token: ${{ steps.get_token.outputs.token }}
42+
43+
- name: Get day of the week
44+
id: get_day
45+
run: echo "day=$(date +'%a' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
46+
47+
- uses: elastic/oblt-actions/oblt-cli/cluster-create-custom@v1
48+
id: create_serverless
49+
with:
50+
template: 'serverless-ea-it'
51+
parameters: '{"Target":"production","ProjectType":"observability"}'
52+
cluster-name-prefix: "${{ env.PREFIX }}-${{ steps.get_day.outputs.day }}"
53+
github-token: ${{ steps.get_token.outputs.token }}
54+
gitops: true
55+
wait: '15'
56+
57+
# Authenticate to the elastic-observability to get the cluster credentials
58+
- uses: elastic/oblt-actions/google/auth@v1
59+
60+
- uses: elastic/oblt-actions/oblt-cli/cluster-credentials@v1
61+
with:
62+
cluster-name: ${{ steps.create_serverless.outputs.cluster-name }}
63+
github-token: ${{ steps.get_token.outputs.token }}
64+
65+
- name: Smoke test
66+
run: curl -X GET ${ELASTICSEARCH_HOST}/_cat/indices?v -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}
67+
68+
####################################
69+
# 2. Copy the serverless secrets
70+
####################################
71+
# Authenticate to the elastic-observability-ci to rotate the cluster credentials
72+
- uses: elastic/oblt-actions/google/auth@v1
73+
with:
74+
project-number: "911195782929"
75+
project-id: "elastic-observability-ci"
76+
77+
- name: Set up Cloud SDK
78+
uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a
79+
80+
# TODO: as soon as the oblt-framework supports elastic-observability-ci we can avoid this step.
81+
# NOTE:
82+
# * While runnning this workflow, it might cause some hiccups if a PR runs when rotating the secrets
83+
# * Secrets need to be created firstly. gcloud secrets create otherwise gcloud secrets versions add will fail.
84+
# That's not an issue now, as we use the same secret name.
85+
- name: Rotate GCSM secrets
86+
env:
87+
GCP_PROJECT: "elastic-observability-ci"
88+
run: |
89+
echo -n "${ELASTICSEARCH_HOST}" | gcloud secrets versions add "${PREFIX}-elasticsearch-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
90+
echo -n "${ELASTICSEARCH_PASSWORD}" | gcloud secrets versions add "${PREFIX}-elasticsearch-password" --data-file=- --quiet --project "${GCP_PROJECT}"
91+
echo -n "${ELASTICSEARCH_USERNAME}" | gcloud secrets versions add "${PREFIX}-elasticsearch-username" --data-file=- --quiet --project "${GCP_PROJECT}"
92+
echo -n "${KIBANA_HOST}" | gcloud secrets versions add "${PREFIX}-kibana-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
93+
echo -n "${KIBANA_USERNAME}" | gcloud secrets versions add "${PREFIX}-kibana-username" --data-file=- --quiet --project "${GCP_PROJECT}"
94+
echo -n "${KIBANA_PASSWORD}" | gcloud secrets versions add "${PREFIX}-kibana-password" --data-file=- --quiet --project "${GCP_PROJECT}"
95+
96+
# TODO: if rotation fails then rollback to the previous cluster.
97+
- if: ${{ failure() }}
98+
uses: elastic/oblt-actions/slack/send@v1
99+
env:
100+
JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
101+
with:
102+
bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
103+
channel-id: "#ingest-notifications"
104+
message: ":traffic_cone: serverless project creation failed for `${{ github.repository }}@${{ github.ref_name }}`, `@robots-ci` please look what's going on <${{ env.JOB_URL }}|here>"

‎docs/test-framework-dev-guide.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,17 @@ The Elastic Agent package that is used for integration tests packages Beats buil
2828
ESS (production) API Key to create on <https://cloud.elastic.co/account/keys>
2929
Warning: if you never created a deployment on it, you won't have permission to get this key, so you will need to create one first.
3030

31+
32+
#### Setup Serverless deployment
33+
34+
This process is now automated and runs daily, utilizing the existing `oblt-cli` framework. Serverless deployments are created each day and automatically destroyed every three days.
35+
36+
The automation is configured in the `serverless-project.yml` file located in the `.github/workflows` directory.
37+
38+
If necessary, you can create a new serverless deployment manually; the previous deployments will be destroyed automatically, but not immediately. To do so, you need to run the GitHub action called [serverless-project.yml](https://github.com/elastic/elastic-agent/actions/workflows/serverless-project.yml).
39+
40+
Credentials for these deployments are securely stored in Google and can only be accessed by Buildkite pipelines. The access control is set using [OpenID Connect in Google Cloud Platform](https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-google-cloud-platform). And that's managed by the Robots team.
41+
3142
## Running tests
3243

3344
Some integration and E2E tests are safe to run locally. These tests set

0 commit comments

Comments
 (0)