Reference Architectures & automated code snippets (#3910)

# Summary

This is the culmination of the code snippets automation & reference
architecture epic.

The main changes are:
- `architectures` - this contains all the reference architectures with
their respective code snippets
- `scripts/code_snippets` - this contains all the scripts required for
automating the code snippet tests (they were moved from public/scripts)
- `scripts/dev/contexts` - 3 additional contexts which represent the 3
possible configurations for running the code snippet tests
- evergreen configurations and scripts

TD:
REDACTED

---------

Co-authored-by: Lucian Tosa <[email protected]>
Co-authored-by: Lucian Tosa <[email protected]>

Author: 2 people

.evergreen-functions.yml

@@ -50,6 +50,9 @@ functions:
       shell: bash
       working_dir: src/github.com/10gen/ops-manager-kubernetes
       <<: *e2e_include_expansions_in_env
+      add_to_path:
+        - ${workdir}/bin
+        - ${workdir}/google-cloud-sdk/bin
       script: |
         echo "Switching context"
         scripts/dev/switch_context.sh "${build_variant}"
@@ -77,6 +80,14 @@ functions:
       type: setup
       params:
         directory: src/github.com/10gen/ops-manager-kubernetes
+    - command: subprocess.exec
+      type: setup
+      params:
+        command: "git config --global user.name 'Evergreen'"
+    - command: subprocess.exec
+      type: setup
+      params:
+        command: "git config --global user.email '[email protected]'"
     - *setup_context
 
   setup_kubectl: &setup_kubectl
@@ -111,6 +122,26 @@ functions:
         - ${workdir}/bin
       binary: scripts/evergreen/setup_aws.sh
 
+  setup_gcloud_cli:
+    command: subprocess.exec
+    type: setup
+    params:
+      working_dir: src/github.com/10gen/ops-manager-kubernetes
+      include_expansions_in_env:
+        - GCP_SERVICE_ACCOUNT_JSON_FOR_SNIPPETS_TESTS
+      add_to_path:
+        - ${workdir}/google-cloud-sdk/bin
+      binary: scripts/evergreen/setup_gcloud_cli.sh
+
+  setup_mongosh:
+    command: subprocess.exec
+    type: setup
+    params:
+      working_dir: src/github.com/10gen/ops-manager-kubernetes
+      add_to_path:
+        - ${workdir}/google-cloud-sdk/bin
+      binary: scripts/evergreen/setup_mongosh.sh
+
   # configures Docker size, installs the Kind binary (if necessary)
   setup_kind: &setup_kind
     command: subprocess.exec
@@ -397,6 +428,19 @@ functions:
       params:
         file: "src/github.com/10gen/ops-manager-kubernetes/logs/myreport.xml"
 
+  upload_code_snippets_logs:
+    - command: s3.put
+      params:
+        aws_key: ${enterprise_aws_access_key_id}
+        aws_secret: ${enterprise_aws_secret_access_key}
+        local_files_include_filter:
+          - src/github.com/10gen/ops-manager-kubernetes/public/architectures/**/*.log
+          - src/github.com/10gen/ops-manager-kubernetes/public/architectures/**/*.out
+        remote_file: logs/${task_id}/${execution}/
+        bucket: operator-e2e-artifacts
+        permissions: public-read
+        content_type: text/plain
+
   preflight_image:
     - *switch_context
     - command: subprocess.exec
@@ -687,3 +731,28 @@ functions:
         files:
           - evergreen_tasks.json
         optional: true
+
+  #
+  # Code snippet test automation
+  #
+
+  gke_multi_cluster_snippets:
+    - *switch_context
+    - command: shell.exec
+      params:
+        shell: bash
+        working_dir: src/github.com/10gen/ops-manager-kubernetes
+        include_expansions_in_env:
+          - version_id
+          - code_snippets_teardown
+        script: |
+          ./scripts/code_snippets/gke_multi_cluster_test.sh
+
+  code_snippets_commit_output:
+    - *switch_context
+    - command: shell.exec
+      params:
+        shell: bash
+        working_dir: src/github.com/10gen/ops-manager-kubernetes
+        script: |
+          ./scripts/code_snippets/sample_commit_output.sh

.evergreen-tasks.yml

@@ -65,6 +65,9 @@ tasks:
         vars:
           image_name: ops-manager
 
+  - name: gke_multi_cluster_snippets
+    commands:
+      - func: gke_multi_cluster_snippets
 ## Below are only e2e runs for .evergreen.yml ##
 
   - name: e2e_multiple_cluster_failures

.evergreen.yml

@@ -69,6 +69,17 @@ variables:
       - func: setup_building_host
       - func: build_multi_cluster_binary
 
+  - &setup_and_teardown_group_gke_code_snippets
+    setup_group:
+      - func: clone
+      - func: setup_gcloud_cli
+      - func: setup_mongosh
+      - func: download_kube_tools
+      - func: build_multi_cluster_binary
+    teardown_group:
+      - func: code_snippets_commit_output
+      - func: upload_code_snippets_logs
+
   - &setup_and_teardown_task_cloudqa
     setup_task_can_fail_task: true
     setup_task:
@@ -149,6 +160,10 @@ parameters:
     value: ""
     description: "Patch id to reuse images from other Evergreen build"
 
+  - key: code_snippets_teardown
+    value: "true"
+    description: set this to false if you would like to keep the clusters created during code snippets tests
+
 # Triggered manually or by PCT.
 patch_aliases:
   - alias: "periodic_builds"
@@ -532,6 +547,11 @@ task_groups:
       - unit_tests_python
       - sbom_tests
 
+  - name: gke_code_snippets
+    <<: *setup_and_teardown_group_gke_code_snippets
+    tasks:
+      - gke_multi_cluster_snippets
+
   # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant
   - name: e2e_mdb_kind_cloudqa_task_group
     max_hosts: -1
@@ -1672,6 +1692,38 @@ buildvariants:
     tasks:
       - name: release_all_agents_on_ecr
 
+    # These variants are used to test the code snippets and each one can be used in patches
+    # Prerelease is especially used when the repo is tagged
+    # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe
+  - name: public_gke_code_snippets
+    display_name: public_gke_code_snippets
+    allowed_requesters: ["patch"]
+    run_on:
+      - ubuntu2204-small
+    tasks:
+      - name: gke_code_snippets
+
+  - name: prerelease_gke_code_snippets
+    display_name: prerelease_gke_code_snippets
+    allowed_requesters: ["patch", "github_tag"]
+    depends_on:
+      - variant: release_images
+        name: '*'
+        patch_optional: true
+    run_on:
+      - ubuntu2204-small
+    tasks:
+      - name: gke_code_snippets
+
+  - name: private_gke_code_snippets
+    display_name: private_gke_code_snippets
+    allowed_requesters: ["patch"]
+    run_on:
+      - ubuntu2204-small
+    <<: *base_om8_dependency
+    tasks:
+      - name: gke_code_snippets
+
   ### Build variants for manual patch only
 
   - name: publish_om60_images

.githooks/pre-commit

@@ -147,17 +147,18 @@ function pre_commit() {
 run_shellcheck() {
   local file="$1"
   echo "Running shellcheck on $file"
-  if ! shellcheck -x "$file" -e SC2154 -e SC1091 -e SC1090 -o require-variable-braces -P "scripts"; then
-    echo "shellcheck failed on $file"
-    exit 1
+  if ! shellcheck -x "$file" -e SC2154 -e SC1091 -e SC1090 -e SC2148 -o require-variable-braces -P "scripts"; then
+   echo "shellcheck failed on $file"
+   exit 1
   fi
 }
 
 start_shellcheck() {
   files_1=$(find scripts -type f -name "*.sh")
   files_2=$(find scripts/dev/contexts -type f)
   files_3=$(find scripts/funcs -type f)
-  files=$(echo -e "$files_1\n$files_2\n$files_3")
+  files_4=$(find public/architectures -type f -name "*.sh")
+  files=$(echo -e "$files_1\n$files_2\n$files_3\n$files_4")
   # Process each file in parallel
   for file in $files; do
     run_shellcheck "$file" &

.gitignore

@@ -35,10 +35,10 @@ scripts/dev/contexts/private-context-*
 
 public/support/*.gz
 public/support/logs*
-public/samples/**/log
-public/samples/**/*.run.log
-public/samples/**/.generated
-public/samples/**/certs/*
+public/architectures/**/log
+public/architectures/**/*.run.log
+public/architectures/**/.generated
+public/architectures/**/certs/*
 
 docker/mongodb-enterprise-appdb/content/readinessprobe
 ops-manager-kubernetes

public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1050_generate_certs.sh

@@ -0,0 +1,18 @@
+kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - <<EOF
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mdb-cert
+spec:
+  dnsNames:
+  - "*.${MDB_NAMESPACE}.svc.cluster.local"
+  duration: 240h0m0s
+  issuerRef:
+    name: my-ca-issuer
+    kind: ClusterIssuer
+  renewBefore: 120h0m0s
+  secretName: cert-prefix-mdb-cert
+  usages:
+  - server auth
+  - client auth
+EOF

public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1100_mongodb_replicaset_multi_cluster.sh

@@ -0,0 +1,30 @@
+kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - <<EOF
+apiVersion: mongodb.com/v1
+kind: MongoDBMultiCluster
+metadata:
+  name: ${RESOURCE_NAME}
+spec:
+  type: ReplicaSet
+  version: 8.0.3
+  opsManager:
+    configMapRef:
+      name: mdb-org-project-config
+  credentials: mdb-org-owner-credentials
+  duplicateServiceObjects: false
+  persistent: true
+  externalAccess: {}
+  security:
+    certsSecretPrefix: cert-prefix
+    tls:
+      ca: ca-issuer
+    authentication:
+      enabled: true
+      modes: ["SCRAM"]
+  clusterSpecList:
+    - clusterName: ${K8S_CLUSTER_0_CONTEXT_NAME}
+      members: 2
+    - clusterName: ${K8S_CLUSTER_1_CONTEXT_NAME}
+      members: 1
+    - clusterName: ${K8S_CLUSTER_2_CONTEXT_NAME}
+      members: 2
+EOF

public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1110_mongodb_replicaset_multi_cluster_wait_for_running_state.sh

@@ -0,0 +1,8 @@
+echo; echo "Waiting for MongoDB to reach Running phase..."
+kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" wait --for=jsonpath='{.status.phase}'=Running "mdbmc/${RESOURCE_NAME}" --timeout=900s
+echo; echo "Pods running in cluster ${K8S_CLUSTER_0_CONTEXT_NAME}"
+kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" get pods
+echo; echo "Pods running in cluster ${K8S_CLUSTER_1_CONTEXT_NAME}"
+kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" get pods
+echo; echo "Pods running in cluster ${K8S_CLUSTER_2_CONTEXT_NAME}"
+kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" get pods

public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1200_create_mongodb_user.sh

@@ -0,0 +1,27 @@
+kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: rs-user-password
+type: Opaque
+stringData:
+  password: password
+---
+apiVersion: mongodb.com/v1
+kind: MongoDBUser
+metadata:
+  name: rs-user
+spec:
+  passwordSecretKeyRef:
+    name: rs-user-password
+    key: password
+  username: "rs-user"
+  db: "admin"
+  mongodbResourceRef:
+    name: ${RESOURCE_NAME}
+  roles:
+  - db: "admin"
+    name: "root"
+EOF
+
+kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" wait --for=jsonpath='{.status.phase}'=Updated -n "${MDB_NAMESPACE}" mdbu/rs-user

public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1210_verify_mongosh_connection.sh

@@ -0,0 +1,12 @@
+# Load Balancers sometimes take longer to get an IP assigned, we need to retry
+while [ -z "$(kubectl get --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" svc "${RESOURCE_NAME}-0-0-svc-external" -o=jsonpath="{.status.loadBalancer.ingress[0].ip}")" ]
+do
+  sleep 5
+done
+
+external_ip="$(kubectl get --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" svc "${RESOURCE_NAME}-0-0-svc-external" -o=jsonpath="{.status.loadBalancer.ingress[0].ip}")"
+
+mkdir -p certs
+kubectl get --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" cm/ca-issuer -o=jsonpath='{.data.ca-pem}' > certs/ca.crt
+
+mongosh --host "${external_ip}" --username rs-user --password password --tls --tlsCAFile certs/ca.crt --tlsAllowInvalidHostnames --eval "db.runCommand({connectionStatus : 1})"