diff --git a/.ci/updatecli/updatecli-bump-stack-version.yml b/.ci/updatecli/updatecli-bump-stack-version.yml new file mode 100644 index 00000000000..2c8900c12a0 --- /dev/null +++ b/.ci/updatecli/updatecli-bump-stack-version.yml @@ -0,0 +1,51 @@ +# update-cli configuration for automated go updates +--- +name: Bump ESS snapshots + +pipelineid: 'bump-ess-snapshot-{{ requiredEnv "BRANCH" }}' + +scms: + githubConfig: + kind: github + spec: + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + owner: "{{ .scm.owner }}" + repository: "{{ .scm.repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + branch: '{{ requiredEnv "BRANCH_NAME" }}' + commitusingapi: true + force: false + +sources: + lastStableStackSnapshot: + name: "Latest Stable ESS snapshot for current branch" + kind: json + spec: + file: 'https://storage.googleapis.com/artifacts-api/channels/{{ requiredEnv "BRANCH_NAME" }}.json' + key: .build +targets: + updateStackImages: + name: "Update create_deployment_csp_configuration.yaml" + sourceid: lastStableStackSnapshot + scmid: githubConfig + kind: file + spec: + file: "pkg/testing/ess/create_deployment_csp_configuration.yaml" + matchpattern: '(\S*_image: ".*docker.*):(.+)"' + replacepattern: '$1:{{ source "lastStableStackSnapshot" }}-SNAPSHOT"' +actions: + elastic-agent: + kind: github/pullrequest + scmid: githubConfig + sourceid: lastStableStackSnapshot + spec: + automerge: true + labels: + - dependencies + - backport-skip + - skip-changelog + - "Team:Elastic-Agent-Control-Plane" + title: '[{{ requiredEnv "BRANCH_NAME" }}][Automation] Bump stack images versions to {{ source "lastStableStackSnapshot" }}' + description: | + Updates `create_deployment_csp_configuration.yml`. It is used to pin stable images versions for integration tests. diff --git a/.github/workflows/bump-stable-snapshot.yml b/.github/workflows/bump-stable-snapshot.yml new file mode 100644 index 00000000000..bd463babf26 --- /dev/null +++ b/.github/workflows/bump-stable-snapshot.yml @@ -0,0 +1,55 @@ +--- +name: update-stable-snapshot + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +permissions: + contents: read + +env: + JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + +jobs: + filter: + runs-on: ubuntu-latest + timeout-minutes: 1 + outputs: + matrix: ${{ steps.generator.outputs.matrix }} + permissions: + contents: read + steps: + - id: generator + uses: elastic/oblt-actions/elastic/active-branches@v1 + with: + filter-branches: true + + update-stable-snapshot: + runs-on: ubuntu-latest + needs: [filter] + permissions: + contents: write + pull-requests: write + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.filter.outputs.matrix) }} + steps: + # We have same behavior for all branches. + - uses: actions/checkout@v4 + + - name: Run UpdateCLI in Apply mode + uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: apply --config .ci/updatecli/updatecli-bump-stack-version.yml --values .ci/updatecli/values.d/scm.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH_NAME: ${{ matrix.branch }} + + - if: ${{ failure() }} + uses: elastic/oblt-actions/slack/send@v1 + with: + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: "#ingest-notifications" + message: ":traffic_cone: Update Stable snapshot: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@agent-team` and @robots-ci please look what's going on <${{ env.JOB_URL }}|here>" diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml index c2cc5916d74..8784f22dfa1 100644 --- a/pkg/testing/ess/create_deployment_csp_configuration.yaml +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -12,4 +12,9 @@ aws: integrations_server_conf_id: "aws.integrationsserver.c5d" elasticsearch_conf_id: "aws.es.datahot.i3" elasticsearch_deployment_template_id: "aws-storage-optimized" - kibana_instance_configuration_id: "aws.kibana.c5d" \ No newline at end of file + kibana_instance_configuration_id: "aws.kibana.c5d" +# Automatically updated by updatecli-bump-stack-version.yml. Don't change the structure and key names +docker: + integration_server_image: "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" + elasticsearch_image: "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" + kibana_image: "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" diff --git a/test_infra/ess/deployment.tf b/test_infra/ess/deployment.tf index c3945275e2e..8e4ed26e9ef 100644 --- a/test_infra/ess/deployment.tf +++ b/test_infra/ess/deployment.tf @@ -61,7 +61,6 @@ locals { ess_region = coalesce(var.ess_region, "gcp-us-east1") deployment_template_id = coalesce(var.deployment_template_id, "gcp-storage-optimized") - ess_properties = merge( { docker = { @@ -73,10 +72,23 @@ locals { }, yamldecode(file("${path.module}/../../pkg/testing/ess/create_deployment_csp_configuration.yaml"))) - - integration_server_docker_image = coalesce(var.integration_server_docker_image, local.ess_properties.docker.integration_server_image, "docker.elastic.co/cloud-release/elastic-agent-cloud:${var.stack_version}") - elasticsearch_docker_image = coalesce(var.elasticsearch_docker_image, local.ess_properties.docker.elasticsearch_image, "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${var.stack_version}") - kibana_docker_image = coalesce(var.kibana_docker_image, local.ess_properties.docker.kibana_image, "docker.elastic.co/cloud-release/kibana-cloud:${var.stack_version}") + integration_server_docker_image = coalesce( + var.integration_server_docker_image, + local.ess_properties.docker.integration_server_image, + "docker.elastic.co/cloud-release/elastic-agent-cloud:${var.stack_version}" + ) + + elasticsearch_docker_image = coalesce( + var.elasticsearch_docker_image, + local.ess_properties.docker.elasticsearch_image, + "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${var.stack_version}" + ) + + kibana_docker_image = coalesce( + var.kibana_docker_image, + local.ess_properties.docker.kibana_image, + "docker.elastic.co/cloud-release/kibana-cloud:${var.stack_version}" + ) } # If we have defined a stack version, validate that this version exists on that region and return it. diff --git a/test_infra/ess/readme.md b/test_infra/ess/readme.md new file mode 100644 index 00000000000..2f5f83a24a3 --- /dev/null +++ b/test_infra/ess/readme.md @@ -0,0 +1,28 @@ +## Terraform-managed Elastic Stack deployment for integration tests + +### Required variables +`stack_version` - The version of the Elastic Stack to deploy, e.g., `9.1.0-SNAPSHOT`. + +### Stable snapshots and images override +This deployment uses the latest tested snapshots of the Elastic Stack components by default. However, you can override the Docker images used for the integration server, Elasticsearch, and Kibana. + +`create_deployment_csp_configuration.yaml` contains the default configuration for the deployment, including the Docker images for the integration server, Elasticsearch, and Kibana. These images are automatically updated to the latest tested snapshots of the Elastic Stack components. +* `integration_server_image` - The Docker image for the integration server +* `elasticsearch_docker_image` - The Docker image for Elasticsearch +* `kibana_docker_image` - The Docker image for Kibana + +Alternatively, you can override these images by setting terraform variables by setting the following variables in your local `terraform.tfvars` file: +```hcl +integration_server_docker_image = "docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" +elasticsearch_docker_image = "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" +kibana_docker_image = "docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" +``` + +or by setting the environment variables: +```bash +export TF_VAR_integration_server_docker_image="docker.elastic.co/cloud-release/elastic-agent-cloud:9.1.0-48398db3-SNAPSHOT" +export TF_VAR_elasticsearch_docker_image="docker.elastic.co/cloud-release/elasticsearch-cloud-ess:9.1.0-48398db3-SNAPSHOT" +export TF_VAR_kibana_docker_image="docker.elastic.co/cloud-release/kibana-cloud:9.1.0-48398db3-SNAPSHOT" +``` + +Note: terraform variables take precedence over `create_deployment_csp_configuration.yaml` docker images configuration. Use `terraform.tfvars` and `TF_VAR_` environment variables to override the images if you don't want to modify the `create_deployment_csp_configuration.yaml` file. \ No newline at end of file