CLOUDP-341025: Refactor of existing snippets on master

.evergreen-functions.yml

@@ -830,7 +830,7 @@ functions:
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         binary: scripts/code_snippets/sample_commit_output.sh
 
-  # it executes a script by convention: ./scripts/code_snippets/${task_name}_test.sh
+  # it executes a script by convention: ./scripts/code_snippets/tests/${task_name}
   test_code_snippets:
     - *switch_context
     - command: shell.exec
@@ -843,4 +843,4 @@ functions:
           - code_snippets_reset
           - task_name
         script: |
-          ./scripts/code_snippets/${task_name}_test.sh
+          ./scripts/code_snippets/tests/${task_name}

.evergreen-snippets.yml

@@ -0,0 +1,141 @@
+variables:
+  - &setup_and_teardown_group_gke_code_snippets
+    setup_task_can_fail_task: true
+    setup_group:
+      - func: clone
+      - func: setup_gcloud_cli
+      - func: setup_mongosh
+      - func: download_kube_tools
+      - func: build_multi_cluster_binary
+    teardown_task:
+      - func: upload_e2e_logs
+      - func: upload_code_snippets_logs
+
+  - &setup_and_teardown_group_kind_code_snippets
+    setup_task_can_fail_task: true
+    setup_group:
+      - func: clone
+      - func: cleanup_exec_environment
+      - func: download_kube_tools
+      - func: configure_docker_auth
+      - func: setup_kubernetes_environment
+    teardown_task:
+      - func: upload_e2e_logs
+      - func: upload_code_snippets_logs
+
+# This variable is copied over from .evergreen.yml because anchors don't work for included files
+  - &base_om8_dependency
+    depends_on:
+      - name: build_om_images
+        variant: build_om80_images
+      - name: build_operator_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image
+        variant: init_test_run
+      - name: build_init_appdb_images_ubi
+        variant: init_test_run
+      - name: build_init_om_images_ubi
+        variant: init_test_run
+      - name: build_agent_images_ubi
+        variant: init_test_run
+
+tasks:
+  # Code snippets tasks
+  # Each scripts/code_snippets/tests/test_*.sh should have its task defined here.
+  # TODO: it should be autogenerated
+  # Each task executes test_code_snippets functon executes scripts/code_snippets/tests/${task_name} by task name convention
+  - name: test_gke_multi_cluster_snippets.sh
+    tags: [ "code_snippets" ]
+    commands:
+      - func: test_code_snippets
+      - func: sample_commit_output
+
+  - name: test_gke_multi_cluster_no_mesh_snippets.sh
+    tags: [ "code_snippets" ]
+    commands:
+      - func: test_code_snippets
+      - func: sample_commit_output
+
+  - name: test_kind_search_community_snippets.sh
+    tags: [ "code_snippets", "patch-run" ]
+    commands:
+      - func: test_code_snippets
+      - func: sample_commit_output
+
+task_groups:
+  - name: gke_code_snippets_task_group
+    <<: *setup_and_teardown_group_gke_code_snippets
+    max_hosts: -1
+    tasks:
+      - test_gke_multi_cluster_snippets.sh
+      - test_gke_multi_cluster_no_mesh_snippets.sh
+
+  - name: kind_code_snippets_task_group
+    <<: *setup_and_teardown_group_kind_code_snippets
+    max_hosts: -1
+    tasks:
+      - test_kind_search_community_snippets.sh
+
+buildvariants:
+  # These variants are used to test the code snippets and each one can be used in patches
+  # Prerelease is especially used when the repo is tagged
+  # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe
+  - name: public_gke_code_snippets
+    display_name: public_gke_code_snippets
+    allowed_requesters: [ "patch" ]
+    run_on:
+      - ubuntu2204-small
+    tasks:
+      - name: gke_code_snippets_task_group
+
+  - name: prerelease_gke_code_snippets
+    display_name: prerelease_gke_code_snippets
+    tags: [ "release" ]
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - variant: release_images
+        name: '*'
+        patch_optional: true
+    run_on:
+      - ubuntu2204-small
+    tasks:
+      - name: gke_code_snippets_task_group
+
+  - name: private_gke_code_snippets
+    display_name: private_gke_code_snippets
+    allowed_requesters: [ "patch" ]
+    run_on:
+      - ubuntu2204-small
+    <<: *base_om8_dependency
+    tasks:
+      - name: gke_code_snippets_task_group
+
+  - name: private_kind_code_snippets
+    display_name: private_kind_code_snippets
+    tags: [ "e2e_test_suite" ]
+    allowed_requesters: [ "patch", "github_pr" ]
+    run_on:
+      - ubuntu2204-large
+    <<: *base_om8_dependency
+    tasks:
+      - name: kind_code_snippets_task_group
+
+  - name: prerelease_kind_code_snippets
+    display_name: prerelease_kind_code_snippets
+    allowed_requesters: [ "patch" ]
+    run_on:
+      - ubuntu2204-large
+    tasks:
+      - name: kind_code_snippets_task_group
+
+  - name: public_kind_code_snippets
+    display_name: public_kind_code_snippets
+    allowed_requesters: [ "patch" ]
+    run_on:
+      - ubuntu2204-large
+    tasks:
+      - name: kind_code_snippets_task_group

.evergreen.yml

@@ -5,6 +5,7 @@ include:
   - filename: .evergreen-functions.yml
   - filename: .evergreen-tasks.yml
   - filename: .evergreen-mco.yml
+  - filename: .evergreen-snippets.yml
 
 variables:
   - &ops_manager_60_latest 6.0.27 # The order/index is important, since these are anchors. Please do not change
@@ -86,29 +87,6 @@ variables:
       - func: setup_building_host
       - func: build_multi_cluster_binary
 
-  - &setup_and_teardown_group_gke_code_snippets
-    setup_task_can_fail_task: true
-    setup_group:
-      - func: clone
-      - func: setup_gcloud_cli
-      - func: setup_mongosh
-      - func: download_kube_tools
-      - func: build_multi_cluster_binary
-    teardown_group:
-      - func: upload_code_snippets_logs
-
-  - &setup_and_teardown_group_kind_code_snippets
-    setup_task_can_fail_task: true
-    setup_group:
-      - func: clone
-      - func: cleanup_exec_environment
-      - func: download_kube_tools
-      - func: configure_docker_auth
-      - func: setup_kubernetes_environment
-    teardown_task:
-      - func: upload_e2e_logs
-      - func: upload_code_snippets_logs
-
   - &setup_and_teardown_task_cloudqa
     setup_task_can_fail_task: true
     setup_task:
@@ -172,6 +150,7 @@ variables:
       - name: build_init_om_images_ubi
         variant: init_test_run
 
+  # Any change to base_om8_dependency should be reflected to its copy in .evergreen-snippets.yml
   - &base_om8_dependency
     depends_on:
       - name: build_om_images
@@ -667,19 +646,6 @@ task_groups:
       - unit_tests_python
       - sbom_tests
 
-  - name: gke_code_snippets_task_group
-    <<: *setup_and_teardown_group_gke_code_snippets
-    max_hosts: -1
-    tasks:
-      - task_gke_multi_cluster_snippets
-      - task_gke_multi_cluster_no_mesh_snippets
-
-  - name: kind_code_snippets_task_group
-    <<: *setup_and_teardown_group_kind_code_snippets
-    max_hosts: -1
-    tasks:
-      - task_kind_community_search_snippets
-
   # Task group for deploying mongodbcommunity resources and testing the (former) MCO
   - name: e2e_mdb_community_task_group
     max_hosts: -1
@@ -1994,65 +1960,6 @@ buildvariants:
     tasks:
       - name: release_all_currently_used_agents_on_ecr
 
-    # These variants are used to test the code snippets and each one can be used in patches
-    # Prerelease is especially used when the repo is tagged
-    # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe
-  - name: public_gke_code_snippets
-    display_name: public_gke_code_snippets
-    allowed_requesters: ["patch"]
-    run_on:
-      - ubuntu2204-small
-    tasks:
-      - name: gke_code_snippets_task_group
-
-  - name: prerelease_gke_code_snippets
-    display_name: prerelease_gke_code_snippets
-    tags: [ "release" ]
-    allowed_requesters: ["patch", "github_tag"]
-    depends_on:
-      - variant: release_images
-        name: '*'
-        patch_optional: true
-    run_on:
-      - ubuntu2204-small
-    tasks:
-      - name: gke_code_snippets_task_group
-
-  - name: private_gke_code_snippets
-    display_name: private_gke_code_snippets
-    allowed_requesters: ["patch"]
-    run_on:
-      - ubuntu2204-small
-    <<: *base_om8_dependency
-    tasks:
-      - name: gke_code_snippets_task_group
-
-  - name: private_kind_code_snippets
-    display_name: private_kind_code_snippets
-    tags: [ "e2e_test_suite" ]
-    allowed_requesters: ["patch", "github_pr"]
-    run_on:
-      - ubuntu2204-large
-    <<: *base_om8_dependency
-    tasks:
-      - name: kind_code_snippets_task_group
-
-  - name: prerelease_kind_code_snippets
-    display_name: prerelease_kind_code_snippets
-    allowed_requesters: ["patch"]
-    run_on:
-      - ubuntu2204-large
-    tasks:
-      - name: kind_code_snippets_task_group
-
-  - name: public_kind_code_snippets
-    display_name: public_kind_code_snippets
-    allowed_requesters: ["patch"]
-    run_on:
-      - ubuntu2204-large
-    tasks:
-      - name: kind_code_snippets_task_group
-
   ### Build variants for manual patch only
 
   - name: backup_csv_images

.githooks/pre-commit

@@ -163,6 +163,10 @@ lint_code() {
   scripts/evergreen/lint_code.sh
 }
 
+function validate_snippets() {
+  scripts/code_snippets/validate_snippets.py
+}
+
 # bg_job_ vars are global; run_job_in_background function is appending to them on each call
 bg_job_pids=()
 bg_job_pids_with_names=()
@@ -217,6 +221,7 @@ pre_commit() {
   run_job_in_background "regenerate_public_rbac_multi_cluster"
   run_job_in_background "python_formatting"
   run_job_in_background "check_erroneous_kubebuilder_annotations"
+  run_job_in_background "validate_snippets"
 
   if wait_for_all_background_jobs; then
     echo -e "${GREEN}pre-commit: All checks passed!${NO_COLOR}"

.gitignore

@@ -91,3 +91,5 @@ docs/**/test.sh.run.log
 
 # goreleaser generated files
 dist
+logs
+*.run.log