Skip to content

Commit 5c50f8a

Browse files
dliappisdliappis
authored
Merge branch '8.17' into mergify/bp/8.17/pr-8359
2 parents 0f926a8 + 99734da commit 5c50f8a

40 files changed

+1435
-1071
lines changed

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,37 @@ env:
1313
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749258065"
1414
IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749258065"
1515

16+
# This section is used to define the plugins that will be used in the pipeline.
17+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
18+
common:
19+
- google_oidc_plugin: &google_oidc_plugin
20+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
21+
# This plugin authenticates to Google Cloud using the OIDC token.
22+
elastic/oblt-google-auth#v1.3.0:
23+
lifetime: 10800 # seconds
24+
project-id: "elastic-observability-ci"
25+
project-number: "911195782929"
26+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/10
27+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/11
28+
# - gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
29+
#avaly/gcp-secret-manager#v1.2.0:
30+
- gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
31+
elastic/gcp-secret-manager#v1.3.0-elastic:
32+
env:
33+
# These secrets are created in .github/workflows/serverless-project.yml
34+
ELASTICSEARCH_HOST: ea-serverless-it-elasticsearch-hostname
35+
ELASTICSEARCH_PASSWORD: ea-serverless-it-elasticsearch-password
36+
ELASTICSEARCH_USERNAME: ea-serverless-it-elasticsearch-username
37+
KIBANA_HOST: ea-serverless-it-kibana-hostname
38+
KIBANA_USERNAME: ea-serverless-it-kibana-username
39+
KIBANA_PASSWORD: ea-serverless-it-kibana-password
40+
1641
steps:
1742
- label: Start ESS stack for integration tests
1843
key: integration-ess
44+
notify:
45+
- github_commit_status:
46+
context: "buildkite/elastic-agent-extended-testing - ESS stack provision"
1947
env:
2048
ASDF_TERRAFORM_VERSION: 1.9.2
2149
command: |
@@ -31,6 +59,9 @@ steps:
3159

3260
- group: "Extended runtime leak tests"
3361
key: extended-integration-tests
62+
notify:
63+
- github_commit_status:
64+
context: "buildkite/elastic-agent-extended-testing - Runtime leak tests"
3465
depends_on:
3566
- integration-ess
3667
steps:
@@ -90,6 +121,9 @@ steps:
90121

91122
- group: "Stateful: Windows"
92123
key: integration-tests-win
124+
notify:
125+
- github_commit_status:
126+
context: "buildkite/elastic-agent-extended-testing - Windows"
93127
depends_on:
94128
- integration-ess
95129
steps:
@@ -184,6 +218,9 @@ steps:
184218

185219
- group: "Stateful:Ubuntu"
186220
key: integration-tests-ubuntu
221+
notify:
222+
- github_commit_status:
223+
context: "buildkite/elastic-agent-extended-testing - Ubuntu"
187224
depends_on:
188225
- integration-ess
189226
steps:
@@ -291,6 +328,9 @@ steps:
291328

292329
- group: "Stateful:Debian"
293330
key: integration-tests-debian
331+
notify:
332+
- github_commit_status:
333+
context: "buildkite/elastic-agent-extended-testing - Debian"
294334
depends_on:
295335
- integration-ess
296336
steps:
@@ -348,6 +388,9 @@ steps:
348388

349389
- group: "Stateful(Sudo):RHEL8"
350390
key: integration-tests-rhel8
391+
notify:
392+
- github_commit_status:
393+
context: "buildkite/elastic-agent-extended-testing - RHEL8"
351394
depends_on:
352395
- integration-ess
353396
steps:
@@ -370,6 +413,9 @@ steps:
370413

371414
- group: "Kubernetes"
372415
key: integration-tests-kubernetes
416+
notify:
417+
- github_commit_status:
418+
context: "buildkite/elastic-agent-extended-testing - Kubernetes"
373419
depends_on:
374420
- integration-ess
375421
- packaging-containers-x86-64
@@ -406,6 +452,76 @@ steps:
406452
- v1.30.8
407453
- v1.31.0
408454

455+
- group: "Serverless integration test"
456+
key: integration-tests-serverless
457+
notify:
458+
- github_commit_status:
459+
context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
460+
steps:
461+
- label: "Windows:2022:amd64:sudo"
462+
depends_on:
463+
- packaging-windows
464+
env:
465+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
466+
command: |
467+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
468+
.buildkite/scripts/buildkite-integration-tests.ps1 fleet true
469+
artifact_paths:
470+
- build/**
471+
- build/diagnostics/**
472+
retry:
473+
automatic:
474+
limit: 1
475+
agents:
476+
provider: "gcp"
477+
machineType: "n2-standard-8"
478+
image: "${IMAGE_WIN_2022}"
479+
plugins:
480+
- *google_oidc_plugin
481+
- *gcp_serverless_secrets_plugin
482+
483+
- label: "Windows:2025:amd64:sudo"
484+
depends_on:
485+
- packaging-windows
486+
env:
487+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
488+
command: |
489+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
490+
.buildkite/scripts/buildkite-integration-tests.ps1 fleet true
491+
artifact_paths:
492+
- build/**
493+
- build/diagnostics/**
494+
retry:
495+
automatic:
496+
limit: 1
497+
agents:
498+
provider: "gcp"
499+
machineType: "n2-standard-8"
500+
image: "${IMAGE_WIN_2025}"
501+
plugins:
502+
- *google_oidc_plugin
503+
- *gcp_serverless_secrets_plugin
504+
- label: "Ubuntu:2404:amd64:sudo"
505+
depends_on: packaging-ubuntu-x86-64
506+
env:
507+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
508+
command: |
509+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
510+
sudo -E .buildkite/scripts/buildkite-integration-tests.sh fleet true
511+
artifact_paths:
512+
- build/**
513+
- build/diagnostics/**
514+
retry:
515+
automatic:
516+
limit: 1
517+
agents:
518+
provider: "gcp"
519+
machineType: "n2-standard-8"
520+
image: "${IMAGE_UBUNTU_2404_X86_64}"
521+
plugins:
522+
- *google_oidc_plugin
523+
- *gcp_serverless_secrets_plugin
524+
409525
- label: ESS stack cleanup
410526
depends_on:
411527
- integration-tests-ubuntu
@@ -430,6 +546,7 @@ steps:
430546
- integration-tests-win
431547
- integration-tests-rhel8
432548
- integration-tests-kubernetes
549+
- integration-tests-serverless
433550
allow_dependency_failure: true
434551
command: |
435552
buildkite-agent artifact download "build/*.xml" .

‎.buildkite/integration.pipeline.yml‎

Lines changed: 3 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ env:
66
steps:
77
- group: "Integration tests: packaging"
88
key: "int-packaging"
9+
notify:
10+
- github_commit_status:
11+
context: "buildkite/elastic-agent - Packaging"
912
steps:
1013
# Build matrix is not used for packaging in favor to unique step keys
1114
# Packaging linux/amd64
@@ -84,31 +87,6 @@ steps:
8487
imagePrefix: "core-ubuntu-2204-aarch64"
8588
diskSizeGb: 200
8689

87-
- label: "Serverless integration test"
88-
key: "serverless-integration-tests"
89-
depends_on:
90-
- int-packaging
91-
concurrency_group: elastic-agent-extended-testing/serverless-integration
92-
concurrency: 8
93-
env:
94-
# we run each step in a different data center to spread the load
95-
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
96-
command: |
97-
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
98-
.buildkite/scripts/steps/integration_tests.sh serverless integration:testServerless
99-
artifact_paths:
100-
- "build/TEST-**"
101-
- "build/diagnostics/*"
102-
agents:
103-
provider: "gcp"
104-
machineType: "n2-standard-8"
105-
retry:
106-
automatic:
107-
limit: 1
108-
notify:
109-
- github_commit_status:
110-
context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
111-
11290
- label: "Triggering Integration tests"
11391
depends_on:
11492
- int-packaging

‎.buildkite/scripts/buildkite-k8s-integration-tests.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,14 @@ EOF
9595
pod_logs_base="${PWD}/build/${fully_qualified_group_name}.pod_logs_dump"
9696

9797
set +e
98-
K8S_TESTS_POD_LOGS_BASE="${pod_logs_base}" AGENT_IMAGE="${image}" DOCKER_VARIANT="${variant}" gotestsum --hide-summary=skipped --format testname --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags kubernetes,integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${group_name}" -integration.sudo="false"
98+
K8S_TESTS_POD_LOGS_BASE="${pod_logs_base}" AGENT_IMAGE="${image}" DOCKER_VARIANT="${variant}" gotestsum --hide-summary=skipped --format testname --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags kubernetes,integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration/k8s -v -args -integration.groups="${group_name}" -integration.sudo="false"
9999
exit_status=$?
100100
set -e
101101

102+
if [[ $exit_status -ne 0 ]]; then
103+
echo "^^^ +++"
104+
fi
105+
102106
if [[ $TESTS_EXIT_STATUS -eq 0 && $exit_status -ne 0 ]]; then
103107
TESTS_EXIT_STATUS=$exit_status
104108
fi

‎.buildkite/scripts/steps/k8s-extended-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ else
2626
fi
2727

2828
SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package
29-
TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix
29+
TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:testKubernetesMatrix
3030
TESTS_EXIT_STATUS=$?
3131
set -e
3232

‎.github/workflows/serverless-project.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
---
2+
name: serverless-project
3+
4+
on:
5+
workflow_dispatch:
6+
schedule:
7+
# To run more often if needed, for now daily at 4:00 UTC
8+
- cron: "0 4 * * *"
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
create-serverless:
15+
permissions:
16+
id-token: write
17+
runs-on: ubuntu-latest
18+
env:
19+
PREFIX: "ea-serverless-it"
20+
steps:
21+
####################################
22+
# 1. Create the serverless project
23+
####################################
24+
- name: Get token
25+
id: get_token
26+
uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
27+
with:
28+
app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
29+
private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
30+
permissions: >-
31+
{
32+
"checks": "read",
33+
"contents": "write",
34+
"pull_requests": "write"
35+
}
36+
repositories: >-
37+
["observability-test-environments"]
38+
39+
- uses: elastic/oblt-actions/git/setup@v1
40+
with:
41+
github-token: ${{ steps.get_token.outputs.token }}
42+
43+
- name: Get day of the week
44+
id: get_day
45+
run: echo "day=$(date +'%a' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
46+
47+
- uses: elastic/oblt-actions/oblt-cli/cluster-create-custom@v1
48+
id: create_serverless
49+
with:
50+
template: 'serverless-ea-it'
51+
parameters: '{"Target":"production","ProjectType":"observability"}'
52+
cluster-name-prefix: "${{ env.PREFIX }}-${{ steps.get_day.outputs.day }}"
53+
github-token: ${{ steps.get_token.outputs.token }}
54+
gitops: true
55+
wait: '15'
56+
57+
# Authenticate to the elastic-observability to get the cluster credentials
58+
- uses: elastic/oblt-actions/google/auth@v1
59+
60+
- uses: elastic/oblt-actions/oblt-cli/cluster-credentials@v1
61+
with:
62+
cluster-name: ${{ steps.create_serverless.outputs.cluster-name }}
63+
github-token: ${{ steps.get_token.outputs.token }}
64+
65+
- name: Smoke test
66+
run: curl -X GET ${ELASTICSEARCH_HOST}/_cat/indices?v -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}
67+
68+
####################################
69+
# 2. Copy the serverless secrets
70+
####################################
71+
# Authenticate to the elastic-observability-ci to rotate the cluster credentials
72+
- uses: elastic/oblt-actions/google/auth@v1
73+
with:
74+
project-number: "911195782929"
75+
project-id: "elastic-observability-ci"
76+
77+
- name: Set up Cloud SDK
78+
uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a
79+
80+
# TODO: as soon as the oblt-framework supports elastic-observability-ci we can avoid this step.
81+
# NOTE:
82+
# * While runnning this workflow, it might cause some hiccups if a PR runs when rotating the secrets
83+
# * Secrets need to be created firstly. gcloud secrets create otherwise gcloud secrets versions add will fail.
84+
# That's not an issue now, as we use the same secret name.
85+
- name: Rotate GCSM secrets
86+
env:
87+
GCP_PROJECT: "elastic-observability-ci"
88+
run: |
89+
echo -n "${ELASTICSEARCH_HOST}" | gcloud secrets versions add "${PREFIX}-elasticsearch-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
90+
echo -n "${ELASTICSEARCH_PASSWORD}" | gcloud secrets versions add "${PREFIX}-elasticsearch-password" --data-file=- --quiet --project "${GCP_PROJECT}"
91+
echo -n "${ELASTICSEARCH_USERNAME}" | gcloud secrets versions add "${PREFIX}-elasticsearch-username" --data-file=- --quiet --project "${GCP_PROJECT}"
92+
echo -n "${KIBANA_HOST}" | gcloud secrets versions add "${PREFIX}-kibana-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
93+
echo -n "${KIBANA_USERNAME}" | gcloud secrets versions add "${PREFIX}-kibana-username" --data-file=- --quiet --project "${GCP_PROJECT}"
94+
echo -n "${KIBANA_PASSWORD}" | gcloud secrets versions add "${PREFIX}-kibana-password" --data-file=- --quiet --project "${GCP_PROJECT}"
95+
96+
# TODO: if rotation fails then rollback to the previous cluster.
97+
- if: ${{ failure() }}
98+
uses: elastic/oblt-actions/slack/send@v1
99+
env:
100+
JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
101+
with:
102+
bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
103+
channel-id: "#ingest-notifications"
104+
message: ":traffic_cone: serverless project creation failed for `${{ github.repository }}@${{ github.ref_name }}`, `@robots-ci` please look what's going on <${{ env.JOB_URL }}|here>"

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ repos:
88
exclude: (.*/.*.(tmpl|tftpl)$)
99
- id: check-merge-conflict
1010
args: ['--assume-in-merge']
11+
exclude: (^NOTICE.*\.txt$)

0 commit comments

Comments
 (0)