From 3621adbee6041fc06d4ef1b8d90787c09786d755 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 16 Jun 2025 17:22:08 +0200 Subject: [PATCH 1/5] [CI] Stable stack version override (#8352) * [CI] Stable stack version override * Update stable version * Added UpdateCLI config * Added UpdateCLI config * Applied proposed changes * Update branch filter * Update branch filter * Added th action * Improved PR description * improve PR message * Pin version in create_deployment_csp_configuration * Added control plane team label * Remove redundant : * Applied proposed changes * Update test_infra/ess/deployment.tf Co-authored-by: Victor Martinez * Added fallback to stack_version for docker images * build to .build * Added pipelineid and force * force: false * Enabled automerge --------- Co-authored-by: Victor Martinez (cherry picked from commit e728b39b5021e98afc543b3f9c85d8dd5c118a8f) # Conflicts: # pkg/testing/ess/create_deployment_csp_configuration.yaml --- .../updatecli-bump-stack-version.yml | 51 +++++++++++++++++ .github/workflows/bump-stable-snapshot.yml | 55 +++++++++++++++++++ .../create_deployment_csp_configuration.yaml | 8 +++ test_infra/ess/deployment.tf | 22 ++++++-- test_infra/ess/readme.md | 28 ++++++++++ 5 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 .ci/updatecli/updatecli-bump-stack-version.yml create mode 100644 .github/workflows/bump-stable-snapshot.yml create mode 100644 test_infra/ess/readme.md diff --git a/.ci/updatecli/updatecli-bump-stack-version.yml b/.ci/updatecli/updatecli-bump-stack-version.yml new file mode 100644 index 00000000000..2c8900c12a0 --- /dev/null +++ b/.ci/updatecli/updatecli-bump-stack-version.yml @@ -0,0 +1,51 @@ +# update-cli configuration for automated go updates +--- +name: Bump ESS snapshots + +pipelineid: 'bump-ess-snapshot-{{ requiredEnv "BRANCH" }}' + +scms: + githubConfig: + kind: github + spec: + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + owner: "{{ .scm.owner }}" + repository: "{{ .scm.repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + branch: '{{ requiredEnv "BRANCH_NAME" }}' + commitusingapi: true + force: false + +sources: + lastStableStackSnapshot: + name: "Latest Stable ESS snapshot for current branch" + kind: json + spec: + file: 'https://storage.googleapis.com/artifacts-api/channels/{{ requiredEnv "BRANCH_NAME" }}.json' + key: .build +targets: + updateStackImages: + name: "Update create_deployment_csp_configuration.yaml" + sourceid: lastStableStackSnapshot + scmid: githubConfig + kind: file + spec: + file: "pkg/testing/ess/create_deployment_csp_configuration.yaml" + matchpattern: '(\S*_image: ".*docker.*):(.+)"' + replacepattern: '$1:{{ source "lastStableStackSnapshot" }}-SNAPSHOT"' +actions: + elastic-agent: + kind: github/pullrequest + scmid: githubConfig + sourceid: lastStableStackSnapshot + spec: + automerge: true + labels: + - dependencies + - backport-skip + - skip-changelog + - "Team:Elastic-Agent-Control-Plane" + title: '[{{ requiredEnv "BRANCH_NAME" }}][Automation] Bump stack images versions to {{ source "lastStableStackSnapshot" }}' + description: | + Updates `create_deployment_csp_configuration.yml`. It is used to pin stable images versions for integration tests. diff --git a/.github/workflows/bump-stable-snapshot.yml b/.github/workflows/bump-stable-snapshot.yml new file mode 100644 index 00000000000..bd463babf26 --- /dev/null +++ b/.github/workflows/bump-stable-snapshot.yml @@ -0,0 +1,55 @@ +--- +name: update-stable-snapshot + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +permissions: + contents: read + +env: + JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + +jobs: + filter: + runs-on: ubuntu-latest + timeout-minutes: 1 + outputs: + matrix: ${{ steps.generator.outputs.matrix }} + permissions: + contents: read + steps: + - id: generator + uses: elastic/oblt-actions/elastic/active-branches@v1 + with: + filter-branches: true + + update-stable-snapshot: + runs-on: ubuntu-latest + needs: [filter] + permissions: + contents: write + pull-requests: write + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.filter.outputs.matrix) }} + steps: + # We have same behavior for all branches. + - uses: actions/checkout@v4 + + - name: Run UpdateCLI in Apply mode + uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: apply --config .ci/updatecli/updatecli-bump-stack-version.yml --values .ci/updatecli/values.d/scm.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH_NAME: ${{ matrix.branch }} + + - if: ${{ failure() }} + uses: elastic/oblt-actions/slack/send@v1 + with: + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: "#ingest-notifications" + message: ":traffic_cone: Update Stable snapshot: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@agent-team` and @robots-ci please look what's going on <${{ env.JOB_URL }}|here>" diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml index c0a12f8d67a..9a13713273c 100644 --- a/pkg/testing/ess/create_deployment_csp_configuration.yaml +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -13,3 +13,11 @@ aws: elasticsearch_conf_id: "aws.es.datahot.i3" elasticsearch_deployment_template_id: "aws-storage-optimized" kibana_instance_configuration_id: "aws.kibana.c5d" +<<<<<<< HEAD +======= +# Automatically updated by updatecli-bump-stack-version.yml. Don't change the structure and key names +docker: + integration_server_image: "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" + elasticsearch_image: "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" + kibana_image: "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" +>>>>>>> e728b39b5 ([CI] Stable stack version override (#8352)) diff --git a/test_infra/ess/deployment.tf b/test_infra/ess/deployment.tf index c3945275e2e..8e4ed26e9ef 100644 --- a/test_infra/ess/deployment.tf +++ b/test_infra/ess/deployment.tf @@ -61,7 +61,6 @@ locals { ess_region = coalesce(var.ess_region, "gcp-us-east1") deployment_template_id = coalesce(var.deployment_template_id, "gcp-storage-optimized") - ess_properties = merge( { docker = { @@ -73,10 +72,23 @@ locals { }, yamldecode(file("${path.module}/../../pkg/testing/ess/create_deployment_csp_configuration.yaml"))) - - integration_server_docker_image = coalesce(var.integration_server_docker_image, local.ess_properties.docker.integration_server_image, "docker.elastic.co/cloud-release/elastic-agent-cloud:${var.stack_version}") - elasticsearch_docker_image = coalesce(var.elasticsearch_docker_image, local.ess_properties.docker.elasticsearch_image, "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${var.stack_version}") - kibana_docker_image = coalesce(var.kibana_docker_image, local.ess_properties.docker.kibana_image, "docker.elastic.co/cloud-release/kibana-cloud:${var.stack_version}") + integration_server_docker_image = coalesce( + var.integration_server_docker_image, + local.ess_properties.docker.integration_server_image, + "docker.elastic.co/cloud-release/elastic-agent-cloud:${var.stack_version}" + ) + + elasticsearch_docker_image = coalesce( + var.elasticsearch_docker_image, + local.ess_properties.docker.elasticsearch_image, + "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${var.stack_version}" + ) + + kibana_docker_image = coalesce( + var.kibana_docker_image, + local.ess_properties.docker.kibana_image, + "docker.elastic.co/cloud-release/kibana-cloud:${var.stack_version}" + ) } # If we have defined a stack version, validate that this version exists on that region and return it. diff --git a/test_infra/ess/readme.md b/test_infra/ess/readme.md new file mode 100644 index 00000000000..2f5f83a24a3 --- /dev/null +++ b/test_infra/ess/readme.md @@ -0,0 +1,28 @@ +## Terraform-managed Elastic Stack deployment for integration tests + +### Required variables +`stack_version` - The version of the Elastic Stack to deploy, e.g., `9.1.0-SNAPSHOT`. + +### Stable snapshots and images override +This deployment uses the latest tested snapshots of the Elastic Stack components by default. However, you can override the Docker images used for the integration server, Elasticsearch, and Kibana. + +`create_deployment_csp_configuration.yaml` contains the default configuration for the deployment, including the Docker images for the integration server, Elasticsearch, and Kibana. These images are automatically updated to the latest tested snapshots of the Elastic Stack components. +* `integration_server_image` - The Docker image for the integration server +* `elasticsearch_docker_image` - The Docker image for Elasticsearch +* `kibana_docker_image` - The Docker image for Kibana + +Alternatively, you can override these images by setting terraform variables by setting the following variables in your local `terraform.tfvars` file: +```hcl +integration_server_docker_image = "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" +elasticsearch_docker_image = "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" +kibana_docker_image = "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" +``` + +or by setting the environment variables: +```bash +export TF_VAR_integration_server_docker_image="docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" +export TF_VAR_elasticsearch_docker_image="docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" +export TF_VAR_kibana_docker_image="docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" +``` + +Note: terraform variables take precedence over `create_deployment_csp_configuration.yaml` docker images configuration. Use `terraform.tfvars` and `TF_VAR_` environment variables to override the images if you don't want to modify the `create_deployment_csp_configuration.yaml` file. \ No newline at end of file From d39009bfe94d8ef3832737e6d379d3623aabff83 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 16 Jun 2025 17:29:53 +0200 Subject: [PATCH 2/5] Resolved conflicts --- pkg/testing/ess/create_deployment_csp_configuration.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml index 9a13713273c..8784f22dfa1 100644 --- a/pkg/testing/ess/create_deployment_csp_configuration.yaml +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -13,11 +13,8 @@ aws: elasticsearch_conf_id: "aws.es.datahot.i3" elasticsearch_deployment_template_id: "aws-storage-optimized" kibana_instance_configuration_id: "aws.kibana.c5d" -<<<<<<< HEAD -======= # Automatically updated by updatecli-bump-stack-version.yml. Don't change the structure and key names docker: integration_server_image: "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" elasticsearch_image: "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" kibana_image: "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" ->>>>>>> e728b39b5 ([CI] Stable stack version override (#8352)) From 574e31fbba75799bc39e9146215ceafdd0e14800 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 16 Jun 2025 17:32:17 +0200 Subject: [PATCH 3/5] Updated snapshot version to 9.0 --- pkg/testing/ess/create_deployment_csp_configuration.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml index 8784f22dfa1..14d930e2251 100644 --- a/pkg/testing/ess/create_deployment_csp_configuration.yaml +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -15,6 +15,6 @@ aws: kibana_instance_configuration_id: "aws.kibana.c5d" # Automatically updated by updatecli-bump-stack-version.yml. Don't change the structure and key names docker: - integration_server_image: "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" - elasticsearch_image: "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" - kibana_image: "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" + integration_server_image: "docker.elastic.co/cloud-release/elastic-agent-cloud:9.0.3-b76cb169-SNAPSHOT" + elasticsearch_image: "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.0.3-b76cb169-SNAPSHOT" + kibana_image: "docker.elastic.co/cloud-release/kibana-cloud:9.0.3-b76cb169-SNAPSHOT" From 45fbdf4c4c600a3a1327e12a59e30b20147577dc Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 16 Jun 2025 19:30:26 +0200 Subject: [PATCH 4/5] Fix typo --- .ci/updatecli/updatecli-bump-stack-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/updatecli/updatecli-bump-stack-version.yml b/.ci/updatecli/updatecli-bump-stack-version.yml index 2c8900c12a0..3da9ffd8bd9 100644 --- a/.ci/updatecli/updatecli-bump-stack-version.yml +++ b/.ci/updatecli/updatecli-bump-stack-version.yml @@ -2,7 +2,7 @@ --- name: Bump ESS snapshots -pipelineid: 'bump-ess-snapshot-{{ requiredEnv "BRANCH" }}' +pipelineid: 'bump-ess-snapshot-{{ requiredEnv "BRANCH_NAME" }}' scms: githubConfig: From a5cbc7bfcfc03ef25cff955862f7201d073569b2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 18 Jun 2025 10:00:35 +0200 Subject: [PATCH 5/5] Apply suggestions from code review --- .ci/updatecli/updatecli-bump-stack-version.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/updatecli/updatecli-bump-stack-version.yml b/.ci/updatecli/updatecli-bump-stack-version.yml index 3da9ffd8bd9..d00938cd229 100644 --- a/.ci/updatecli/updatecli-bump-stack-version.yml +++ b/.ci/updatecli/updatecli-bump-stack-version.yml @@ -1,4 +1,3 @@ -# update-cli configuration for automated go updates --- name: Bump ESS snapshots @@ -40,7 +39,7 @@ actions: scmid: githubConfig sourceid: lastStableStackSnapshot spec: - automerge: true + automerge: false labels: - dependencies - backport-skip