Merge branch 'master' of github.com:mongodb/mongodb-kubernetes into multi-arch-pipeline-combined

Author: nammn

.evergreen-functions.yml

@@ -360,6 +360,7 @@ functions:
     - command: shell.exec
       type: setup
       params:
+        continue_on_err: true
         shell: bash
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
@@ -443,6 +444,7 @@ functions:
   upload_e2e_logs:
     - command: s3.put
       params:
+        continue_on_err: true
         aws_key: ${enterprise_aws_access_key_id}
         aws_secret: ${enterprise_aws_secret_access_key}
         local_files_include_filter:
@@ -550,30 +552,14 @@ functions:
           # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror
           docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use
           docker buildx inspect --bootstrap
-    - command: ec2.assume_role
-      display_name: Assume IAM role with permissions to pull Kondukto API token
-      params:
-        role_arn: ${kondukto_role_arn}
-    - command: shell.exec
-      display_name: Pull Kondukto API token from AWS Secrets Manager and write it to file
-      params:
-        silent: true
-        shell: bash
-        include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
-        script: |
-          set -e
-          # use AWS CLI to get the Kondukto API token from AWS Secrets Manager
-          kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)
-          # write the KONDUKTO_TOKEN environment variable to Silkbomb environment file
-          echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/silkbomb.env
     - command: subprocess.exec
       retry_on_failure: true
       type: setup
       params:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name}
+        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents}
 
   # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function
   legacy_pipeline:

.evergreen-periodic-builds.yaml

@@ -19,6 +19,15 @@ variables:
       - func: switch_context
 
 tasks:
+  - name: periodic_teardown_aws
+    commands:
+      - func: cleanup_aws
+
+  - name: periodic_teardown_cloudqa
+    commands:
+      - func: teardown_cloud_qa_all
+
+task_groups:
   - name: periodic_teardown_task_group
     <<: *setup_group
     tasks:

.evergreen.yml

@@ -61,14 +61,14 @@ variables:
         variant: init_test_run
       - name: build_test_image
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
       - name: build_readiness_probe_image
         variant: init_test_run
       - name: build_upgrade_hook_image
         variant: init_test_run
       - name: build_mco_test_image
         variant: init_test_run
+      - name: build_agent_images_ubi
+        variant: init_test_run
 
   - &setup_group
     setup_group_can_fail_task: true
@@ -124,9 +124,9 @@ variables:
       - func: setup_cloud_qa
     teardown_task_can_fail_task: true
     teardown_task:
+      - func: teardown_cloud_qa
       - func: upload_e2e_logs
       - func: teardown_kubernetes_environment
-      - func: teardown_cloud_qa
 
   - &setup_and_teardown_task
     setup_task_can_fail_task: true
@@ -163,6 +163,25 @@ variables:
       - name: build_agent_images_ubi
         variant: init_test_run
 
+  - &base_om7_dependency_with_race
+    depends_on:
+      - name: build_om_images
+        variant: build_om70_images
+      - name: build_operator_race_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image
+        variant: init_test_run
+      - name: build_init_appdb_images_ubi
+        variant: init_test_run
+      - name: build_init_om_images_ubi
+        variant: init_test_run
+      - name: build_agent_images_ubi
+        variant: init_test_run
+
   - &base_om8_dependency
     depends_on:
       - name: build_om_images
@@ -335,19 +354,6 @@ tasks:
           image_name: init-ops-manager
           include_tags: release
 
-  - name: release_agent_operator_release
-    tags: [ "image_release" ]
-    allowed_requesters: [ "patch", "github_tag" ]
-    commands:
-      - func: clone
-      - func: setup_building_host
-      - func: quay_login
-      - func: setup_docker_sbom
-      - func: legacy_pipeline
-        vars:
-          image_name: agent
-          include_tags: release
-
   # pct only triggers this variant once a new agent image is out
   - name: release_agent
     # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr)
@@ -359,8 +365,7 @@ tasks:
       - func: setup_docker_sbom
       - func: legacy_pipeline
         vars:
-          image_name: agent-pct
-          include_tags: release
+          image_name: agent
 
   - name: run_precommit_and_push
     tags: ["patch-run"]
@@ -380,48 +385,17 @@ tasks:
           working_dir: src/github.com/mongodb/mongodb-kubernetes
           binary: scripts/evergreen/precommit_bump.sh
 
-  # Pct only triggers this variant once a new agent image is out
-  # these releases the agent with the operator suffix (not patch id) on ecr to allow for digest pinning to pass.
-  # For this to work, we rely on skip_tags which is used to determine whether
-  # we want to release on quay or not, in this case - ecr instead.
-  # We rely on the init_database from ecr for the agent x operator images.
-  # This runs on agent releases that are not concurrent with operator releases.
-  - name: release_agents_on_ecr_conditional
-    commands:
-      - func: clone
-      - func: run_task_conditionally
-        vars:
-          condition_script: scripts/evergreen/should_release_agents_on_ecr.sh
-          variant: init_release_agents_on_ecr
-          task: release_agents_on_ecr
-
-  - name: release_agents_on_ecr
-    # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr)
-    allowed_requesters: [ "patch", "github_pr" ]
-    priority: 70
-    commands:
-      - func: clone
-      - func: setup_building_host
-      - func: legacy_pipeline
-        vars:
-          image_name: agent-pct
-          skip_tags: release
-
   - name: release_all_agents_on_ecr
-    # this enables us to run this manually (patch) and release all agent versions to ECR
-    # it's needed during operator new version release process - e2e tests (especially olm tests)
-    # will look for agent with new operator version suffix, but during PR checks we only build
-    # agent versions for most recent major OM versions and the tests will fail. Before running the PR
-    # we have to manually release all agents to ECR by triggering this patch
+    # this enables us to run this manually (patch) and release all agent versions to ECR to verify
+    # Dockerfile, script changes etc.
     allowed_requesters: [ "patch" ]
     commands:
       - func: clone
       - func: setup_building_host
-      - func: legacy_pipeline
+      - func: pipeline
         vars:
-          image_name: agent-pct
-          skip_tags: release
-          all_agents: true
+          image_name: agent
+          all_agents: "--all-agents"
 
   - name: build_test_image
     commands:
@@ -461,18 +435,23 @@ tasks:
       - func: setup_building_host
       - func: pipeline
         vars:
-          skip_tags: ubuntu,release
-          distro: ubi
           image_name: operator
 
+  - name: build_operator_race_ubi
+    commands:
+      - func: clone
+      - func: setup_building_host
+      - func: pipeline
+        vars:
+          image_name: operator-race
+
   - name: build_init_om_images_ubi
     commands:
       - func: clone
       - func: setup_building_host
       - func: pipeline
         vars:
           image_name: init-ops-manager
-          skip_tags: ubuntu,release
 
   - name: build_init_appdb_images_ubi
     commands:
@@ -481,7 +460,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: init-appdb
-          skip_tags: ubuntu,release
 
   - name: build_agent_images_ubi
     commands:
@@ -490,7 +468,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: agent
-          skip_tags: ubuntu,release
 
   - name: build_init_database_image_ubi
     commands:
@@ -499,7 +476,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: init-database
-          skip_tags: ubuntu,release
 
   - name: build_database_image_ubi
     commands:
@@ -508,7 +484,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: database
-          skip_tags: ubuntu,release
 
   - name: build_readiness_probe_image
     commands:
@@ -517,7 +492,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: readiness-probe
-          skip_tags: ubuntu,release
 
   - name: build_upgrade_hook_image
     commands:
@@ -526,7 +500,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: upgrade-hook
-          skip_tags: ubuntu,release
 
   - name: prepare_aws
     priority: 59
@@ -1354,8 +1327,7 @@ buildvariants:
         variant: init_test_run
       - name: build_init_database_image_ubi
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
+
     tasks:
       - name: e2e_custom_domain_task_group
 
@@ -1389,8 +1361,7 @@ buildvariants:
         variant: init_test_run
       - name: build_init_database_image_ubi
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
+
     run_on:
       - ubuntu2204-small
     tasks:
@@ -1472,7 +1443,7 @@ buildvariants:
     tags: [ "e2e_test_suite" ]
     run_on:
       - ubuntu1804-xlarge
-    <<: *base_om7_dependency
+    <<: *base_om7_dependency_with_race
     tasks:
       - name: e2e_operator_race_with_telemetry_task_group
 
@@ -1674,6 +1645,8 @@ buildvariants:
         variant: init_test_run
       - name: prepare_and_upload_openshift_bundles_for_e2e
         variant: init_tests_with_olm
+      - name: build_agent_images_ubi
+        variant: init_test_run
     tasks:
       - name: e2e_kind_olm_group
 
@@ -1699,6 +1672,7 @@ buildvariants:
         variant: init_test_run
       - name: build_agent_images_ubi
         variant: init_test_run
+
     tasks:
       - name: e2e_kind_olm_group
 
@@ -1751,6 +1725,7 @@ buildvariants:
       - ubuntu2204-small
     tasks:
       - name: build_operator_ubi
+      - name: build_operator_race_ubi
       - name: build_test_image
       - name: build_mco_test_image
       - name: build_init_appdb_images_ubi
@@ -1780,18 +1755,6 @@ buildvariants:
     tasks:
       - name: build_test_image_ibm
 
-  - name: init_release_agents_on_ecr
-    display_name: init_release_agents_on_ecr
-    # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr)
-    allowed_requesters: [ "patch", "github_pr" ]
-    tags: [ "release_agents_on_ecr" ]
-    # We want that to run first and finish asap. Digest pinning depends on this to succeed.
-    priority: 70
-    run_on:
-      - ubuntu2204-large
-    tasks:
-      - name: release_agents_on_ecr_conditional
-
   - name: run_pre_commit
     priority: 70
     display_name: run_pre_commit
@@ -1819,8 +1782,7 @@ buildvariants:
         variant: init_test_run
       - name: build_init_om_images_ubi
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
+
     run_on:
       - ubuntu2204-small
     tasks:
@@ -1906,13 +1868,6 @@ buildvariants:
       - name: release_init_database
       - name: release_init_ops_manager
       - name: release_database
-      # Once we release the operator, we will also release the init databases, we require them to be out first
-      # such that we can reference them and retrieve those binaries.
-      # Since we immediately run daily rebuild after creating the image, we can ensure that the init_database is out
-      # such that the agent image build can use it.
-      - name: release_agent_operator_release
-        depends_on:
-          - name: release_init_database
 
   - name: preflight_release_images
     display_name: preflight_release_images
@@ -1944,13 +1899,13 @@ buildvariants:
 
     # It will be called by pct while bumping the agent cloud manager image
   - name: release_agent
-    display_name: (Static Containers) Release Agent matrix
+    display_name: release_agent
     tags: [ "release_agent" ]
     run_on:
       - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
     depends_on:
-      - variant: init_release_agents_on_ecr
-        name: '*'
+      - variant: init_test_run
+        name: build_agent_images_ubi # this ensures the agent gets released to ECR as well
       - variant: e2e_multi_cluster_kind
         name: '*'
       - variant: e2e_static_multi_cluster_2_clusters