From 07c355000fba1b9730123c76e2cfc121f262e52d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 9 Jul 2025 12:51:56 +0200 Subject: [PATCH 1/2] Revert [CI] Stable stack version override #8352 and #8528 (#8889) (cherry picked from commit fb8b8838198abdd37ed83bbb941dda981c9266c1) # Conflicts: # pkg/testing/ess/create_deployment_csp_configuration.yaml --- .../updatecli-bump-stack-version.yml | 51 ----------------- .github/workflows/bump-stable-snapshot.yml | 57 ------------------- .mergify.yml | 7 +-- .../create_deployment_csp_configuration.yaml | 3 + test_infra/ess/deployment.tf | 22 ++----- test_infra/ess/readme.md | 28 --------- 6 files changed, 10 insertions(+), 158 deletions(-) delete mode 100644 .ci/updatecli/updatecli-bump-stack-version.yml delete mode 100644 .github/workflows/bump-stable-snapshot.yml delete mode 100644 test_infra/ess/readme.md diff --git a/.ci/updatecli/updatecli-bump-stack-version.yml b/.ci/updatecli/updatecli-bump-stack-version.yml deleted file mode 100644 index 3da9ffd8bd9..00000000000 --- a/.ci/updatecli/updatecli-bump-stack-version.yml +++ /dev/null @@ -1,51 +0,0 @@ -# update-cli configuration for automated go updates ---- -name: Bump ESS snapshots - -pipelineid: 'bump-ess-snapshot-{{ requiredEnv "BRANCH_NAME" }}' - -scms: - githubConfig: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .scm.owner }}" - repository: "{{ .scm.repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - branch: '{{ requiredEnv "BRANCH_NAME" }}' - commitusingapi: true - force: false - -sources: - lastStableStackSnapshot: - name: "Latest Stable ESS snapshot for current branch" - kind: json - spec: - file: 'https://storage.googleapis.com/artifacts-api/channels/{{ requiredEnv "BRANCH_NAME" }}.json' - key: .build -targets: - updateStackImages: - name: "Update create_deployment_csp_configuration.yaml" - sourceid: lastStableStackSnapshot - scmid: githubConfig - kind: file - spec: - file: "pkg/testing/ess/create_deployment_csp_configuration.yaml" - matchpattern: '(\S*_image: ".*docker.*):(.+)"' - replacepattern: '$1:{{ source "lastStableStackSnapshot" }}-SNAPSHOT"' -actions: - elastic-agent: - kind: github/pullrequest - scmid: githubConfig - sourceid: lastStableStackSnapshot - spec: - automerge: true - labels: - - dependencies - - backport-skip - - skip-changelog - - "Team:Elastic-Agent-Control-Plane" - title: '[{{ requiredEnv "BRANCH_NAME" }}][Automation] Bump stack images versions to {{ source "lastStableStackSnapshot" }}' - description: | - Updates `create_deployment_csp_configuration.yml`. It is used to pin stable images versions for integration tests. diff --git a/.github/workflows/bump-stable-snapshot.yml b/.github/workflows/bump-stable-snapshot.yml deleted file mode 100644 index fb17968d3aa..00000000000 --- a/.github/workflows/bump-stable-snapshot.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -name: update-stable-snapshot - -on: - workflow_dispatch: - schedule: - - cron: "0 0 * * *" - -permissions: - contents: read - -env: - JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - -jobs: - filter: - runs-on: ubuntu-latest - timeout-minutes: 1 - outputs: - matrix: ${{ steps.generator.outputs.matrix }} - permissions: - contents: read - steps: - - id: generator - uses: elastic/oblt-actions/elastic/active-branches@v1 - with: - # Remove exclusions when backports are merged - exclude-branches: "8.17,8.18,8.19,9.0" - filter-branches: true - - update-stable-snapshot: - runs-on: ubuntu-latest - needs: [filter] - permissions: - contents: write - pull-requests: write - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.filter.outputs.matrix) }} - steps: - # We have same behavior for all branches. - - uses: actions/checkout@v4 - - - name: Run UpdateCLI in Apply mode - uses: elastic/oblt-actions/updatecli/run@v1 - with: - command: apply --config .ci/updatecli/updatecli-bump-stack-version.yml --values .ci/updatecli/values.d/scm.yml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH_NAME: ${{ matrix.branch }} - - - if: ${{ failure() }} - uses: elastic/oblt-actions/slack/send@v1 - with: - bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - channel-id: "#ingest-notifications" - message: ":traffic_cone: Update Stable snapshot: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@agent-team` and @robots-ci please look what's going on <${{ env.JOB_URL }}|here>" diff --git a/.mergify.yml b/.mergify.yml index 68d9c62b55b..deefe5c8884 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -19,24 +19,22 @@ defaults: labels: - "backport" pull_request_rules: - - name: automatic approval for updatecli pull requests with changes in .buildkite or create_deployment_csp_configuration.yaml + - name: automatic approval for updatecli pull requests with changes in .buildkite conditions: - author=github-actions[bot] - check-success=buildkite/elastic-agent - or: - - files~=^pkg/testing/ess/create_deployment_csp_configuration.yaml$ - files~=^.buildkite/(pipeline.yml|bk.integration.pipeline.yml)$ - head~=^updatecli_.* actions: review: type: APPROVE message: Automatically approving mergify - - name: automatic squash and merge with success checks and the files matching the regex .buildkite or create_deployment_csp_configuration.yaml is modified. + - name: automatic squash and merge with success checks and the files matching the regex .buildkite is modified. conditions: - author=github-actions[bot] - check-success=buildkite/elastic-agent - or: - - files~=^pkg/testing/ess/create_deployment_csp_configuration.yaml$ - files~=^.buildkite/(pipeline.yml|bk.integration.pipeline.yml)$ - head~=^updatecli_.* - "#approved-reviews-by>=1" @@ -47,7 +45,6 @@ pull_request_rules: conditions: - author=github-actions[bot] - or: - - files~=^pkg/testing/ess/create_deployment_csp_configuration.yaml$ - files~=^.buildkite/(pipeline.yml|bk.integration.pipeline.yml)$ - head~=^updatecli_.* - "#check-failure>0" diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml index 56033e79311..286def2d60e 100644 --- a/pkg/testing/ess/create_deployment_csp_configuration.yaml +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -13,8 +13,11 @@ aws: elasticsearch_conf_id: "aws.es.datahot.i3" elasticsearch_deployment_template_id: "aws-storage-optimized" kibana_instance_configuration_id: "aws.kibana.c5d" +<<<<<<< HEAD # Automatically updated by updatecli-bump-stack-version.yml. Don't change the structure and key names docker: integration_server_image: "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-dd974feb-SNAPSHOT" elasticsearch_image: "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-dd974feb-SNAPSHOT" kibana_image: "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-dd974feb-SNAPSHOT" +======= +>>>>>>> fb8b88381 (Revert [CI] Stable stack version override #8352 and #8528 (#8889)) diff --git a/test_infra/ess/deployment.tf b/test_infra/ess/deployment.tf index 8e4ed26e9ef..c3945275e2e 100644 --- a/test_infra/ess/deployment.tf +++ b/test_infra/ess/deployment.tf @@ -61,6 +61,7 @@ locals { ess_region = coalesce(var.ess_region, "gcp-us-east1") deployment_template_id = coalesce(var.deployment_template_id, "gcp-storage-optimized") + ess_properties = merge( { docker = { @@ -72,23 +73,10 @@ locals { }, yamldecode(file("${path.module}/../../pkg/testing/ess/create_deployment_csp_configuration.yaml"))) - integration_server_docker_image = coalesce( - var.integration_server_docker_image, - local.ess_properties.docker.integration_server_image, - "docker.elastic.co/cloud-release/elastic-agent-cloud:${var.stack_version}" - ) - - elasticsearch_docker_image = coalesce( - var.elasticsearch_docker_image, - local.ess_properties.docker.elasticsearch_image, - "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${var.stack_version}" - ) - - kibana_docker_image = coalesce( - var.kibana_docker_image, - local.ess_properties.docker.kibana_image, - "docker.elastic.co/cloud-release/kibana-cloud:${var.stack_version}" - ) + + integration_server_docker_image = coalesce(var.integration_server_docker_image, local.ess_properties.docker.integration_server_image, "docker.elastic.co/cloud-release/elastic-agent-cloud:${var.stack_version}") + elasticsearch_docker_image = coalesce(var.elasticsearch_docker_image, local.ess_properties.docker.elasticsearch_image, "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${var.stack_version}") + kibana_docker_image = coalesce(var.kibana_docker_image, local.ess_properties.docker.kibana_image, "docker.elastic.co/cloud-release/kibana-cloud:${var.stack_version}") } # If we have defined a stack version, validate that this version exists on that region and return it. diff --git a/test_infra/ess/readme.md b/test_infra/ess/readme.md deleted file mode 100644 index f625549fed4..00000000000 --- a/test_infra/ess/readme.md +++ /dev/null @@ -1,28 +0,0 @@ -## Terraform-managed Elastic Stack deployment for integration tests - -### Required variables -`stack_version` - The version of the Elastic Stack to deploy, e.g., `9.1.0-SNAPSHOT`. - -### Stable snapshots and images override -This deployment uses the latest tested snapshots of the Elastic Stack components by default. However, you can override the Docker images used for the integration server, Elasticsearch, and Kibana. - -`create_deployment_csp_configuration.yaml` contains the default configuration for the deployment, including the Docker images for the integration server, Elasticsearch, and Kibana. These images are automatically updated to the latest tested snapshots of the Elastic Stack components. -* `integration_server_image` - The Docker image for the integration server -* `elasticsearch_docker_image` - The Docker image for Elasticsearch -* `kibana_docker_image` - The Docker image for Kibana - -Alternatively, you can override these images by setting terraform variables by setting the following variables in your local `terraform.tfvars` file: -```hcl -integration_server_docker_image = "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" -elasticsearch_docker_image = "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" -kibana_docker_image = "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" -``` - -or by setting the environment variables: -```bash -export TF_VAR_integration_server_docker_image="docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" -export TF_VAR_elasticsearch_docker_image="docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" -export TF_VAR_kibana_docker_image="docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" -``` - -Note: terraform variables take precedence over `create_deployment_csp_configuration.yaml` docker images configuration. Use `terraform.tfvars` and `TF_VAR_` environment variables to override the images if you don't want to modify the `create_deployment_csp_configuration.yaml` file. From 08eec3b600131c7fa86fe19c63ab923e441f6149 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 9 Jul 2025 12:57:43 +0200 Subject: [PATCH 2/2] Update pkg/testing/ess/create_deployment_csp_configuration.yaml --- pkg/testing/ess/create_deployment_csp_configuration.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml index 286def2d60e..c0a12f8d67a 100644 --- a/pkg/testing/ess/create_deployment_csp_configuration.yaml +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -13,11 +13,3 @@ aws: elasticsearch_conf_id: "aws.es.datahot.i3" elasticsearch_deployment_template_id: "aws-storage-optimized" kibana_instance_configuration_id: "aws.kibana.c5d" -<<<<<<< HEAD -# Automatically updated by updatecli-bump-stack-version.yml. Don't change the structure and key names -docker: - integration_server_image: "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-dd974feb-SNAPSHOT" - elasticsearch_image: "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-dd974feb-SNAPSHOT" - kibana_image: "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-dd974feb-SNAPSHOT" -======= ->>>>>>> fb8b88381 (Revert [CI] Stable stack version override #8352 and #8528 (#8889))