[CDRIVER-5535] Create asset groups in Silk (#1619)

* Add a script and Earthly target to create a Silk asset group
* Define new EVG tasks:

- Add a "Miscellaneous" "variant" to group one-off tasks that don't vary.
- Add a "create-silk-asset-group" task that creates a Silk asset group
  for the current branch if it does not already exist.

* Ignore a `.secret` file in the repo directory
* Run the asset group creation on all builds

Author: vector-of-bool

.evergreen/config_generator/components/earthly.py

@@ -5,7 +5,7 @@
 from typing import Iterable, Literal, Mapping, NamedTuple, TypeVar
 
 from shrub.v3.evg_build_variant import BuildVariant
-from shrub.v3.evg_command import EvgCommandType, subprocess_exec
+from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, subprocess_exec
 from shrub.v3.evg_task import EvgTaskRef
 
 from ..etc.utils import Task, all_possible
@@ -164,6 +164,30 @@ def variants_for(config: Configuration) -> Iterable[EarthlyVariant]:
     return filter(allow_env_for_config, all_envs)
 
 
+def earthly_exec(
+    *,
+    kind: Literal["test", "setup", "system"],
+    target: str,
+    secrets: Mapping[str, str] | None = None,
+    args: Mapping[str, str] | None = None,
+) -> BuiltInCommand:
+    """Create a subprocess_exec command that runs Earthly with the given arguments"""
+    env: dict[str, str] = {}
+    if secrets:
+        env["EARTHLY_SECRETS"] = ",".join(f"{k}={v}" for k, v in secrets.items())
+    return subprocess_exec(
+        "bash",
+        args=[
+            "tools/earthly.sh",
+            f"+{target}",
+            *(f"--{arg}={val}" for arg, val in (args or {}).items()),
+        ],
+        command_type=EvgCommandType(kind),
+        env=env if env else None,
+        working_dir="mongoc",
+    )
+
+
 def earthly_task(
     *,
     name: str,
@@ -184,40 +208,30 @@ def earthly_task(
         return
     # Generate the build-arg arguments based on the configuration options. The
     # NamedTuple field names must match with the ARG keys in the Earthfile!
-    earthly_args = [f"--{key}={val}" for key, val in config._asdict().items()]
-    # Add arguments that come from parameter expansions defined in the build variant
-    earthly_args += [
-        f"--env=${{{_ENV_PARAM_NAME}}}",
-        f"--c_compiler=${{{_CC_PARAM_NAME}}}",
-    ]
+    earthly_args = config._asdict()
+    earthly_args |= {
+        # Add arguments that come from parameter expansions defined in the build variant
+        "env": f"${{{_ENV_PARAM_NAME}}}",
+        "c_compiler": f"${{{_CC_PARAM_NAME}}}",
+    }
     return Task(
         name=name,
         commands=[
             # First, just build the "env-warmup" which will prepare the build environment.
             # This won't generate any output, but allows EVG to track it as a separate build step
             # for timing and logging purposes. The subequent build step will cache-hit the
             # warmed-up build environments.
-            subprocess_exec(
-                "bash",
-                args=[
-                    "tools/earthly.sh",
-                    "+env-warmup",
-                    *earthly_args,
-                ],
-                working_dir="mongoc",
-                command_type=EvgCommandType.SETUP,
+            earthly_exec(
+                kind="setup",
+                target="env-warmup",
+                args=earthly_args,
             ),
             # Now execute the main tasks:
-            subprocess_exec(
-                "bash",
-                args=[
-                    "tools/earthly.sh",
-                    "+run",
-                    f"--targets={' '.join(targets)}",
-                    *earthly_args,
-                ],
-                working_dir="mongoc",
-                command_type=EvgCommandType.TEST,
+            earthly_exec(
+                kind="test",
+                target="run",
+                # The "targets" arg is for +run to specify which targets to run
+                args={"targets": " ".join(targets)} | earthly_args,
             ),
         ],
         tags=[f"earthly", "pr-merge-gate", *env_tags],
@@ -249,5 +263,5 @@ def tasks() -> Iterable[Task]:
             yield task
 
 
-def variants() -> list[BuildVariant]:
-    return [ev.as_evg_variant() for ev in all_possible(EarthlyVariant)]
+def variants() -> Iterable[BuildVariant]:
+    yield from (ev.as_evg_variant() for ev in all_possible(EarthlyVariant))

.evergreen/config_generator/components/misc.py

@@ -0,0 +1,36 @@
+from typing import Iterable
+
+from shrub.v3.evg_build_variant import BuildVariant
+from shrub.v3.evg_task import EvgTaskRef
+
+from ..etc.utils import Task
+from . import earthly
+
+
+def tasks() -> Iterable[Task]:
+    yield Task(
+        name="create-silk-asset-group",
+        commands=[
+            earthly.earthly_exec(
+                kind="setup",
+                target="create-silk-asset-group",
+                args={
+                    "branch": r"${branch_name}",
+                },
+                secrets={
+                    "SILK_CLIENT_ID": r"${silk_client_id}",
+                    "SILK_CLIENT_SECRET": r"${silk_client_secret}",
+                },
+            )
+        ],
+        run_on=earthly.CONTAINER_RUN_DISTROS,
+        tags=["misc", "pr-merge-gate"],
+    )
+
+
+def variants() -> Iterable[BuildVariant]:
+    yield BuildVariant(
+        name="misc",
+        tasks=[EvgTaskRef(name=".misc")],
+        display_name="Miscellaneous",
+    )

.evergreen/generated_configs/tasks.yml

@@ -1169,6 +1169,30 @@ tasks:
     tags: [clang-format]
     commands:
       - func: clang-format
+  - name: create-silk-asset-group
+    run_on:
+      - ubuntu2204-small
+      - ubuntu2204-large
+      - ubuntu2004-small
+      - ubuntu2004
+      - ubuntu1804
+      - ubuntu1804-medium
+      - debian10
+      - debian11
+      - amazon2
+    tags: [misc, pr-merge-gate]
+    commands:
+      - command: subprocess.exec
+        type: setup
+        params:
+          binary: bash
+          working_dir: mongoc
+          env:
+            EARTHLY_SECRETS: SILK_CLIENT_ID=${silk_client_id},SILK_CLIENT_SECRET=${silk_client_secret}
+          args:
+            - tools/earthly.sh
+            - +create-silk-asset-group
+            - --branch=${branch_name}
   - name: cse-sasl-cyrus-darwinssl-macos-1100-clang-compile
     run_on: macos-1100
     tags: [cse-matrix-darwinssl, compile, macos-1100, clang, cse, sasl-cyrus]

.evergreen/generated_configs/variants.yml

@@ -101,6 +101,10 @@ buildvariants:
     display_name: loadbalanced
     tasks:
       - name: .loadbalanced
+  - name: misc
+    display_name: Miscellaneous
+    tasks:
+      - name: .misc
   - name: mock-server-test
     display_name: Mock Server Test
     expansions:

.gitignore

@@ -38,3 +38,6 @@ dist
 # `secrets-export.sh` is created by the drivers-evergreen-tools `setup_secrets.sh` script.
 # Ignore this file to reduce risk of committing credentials during local development.
 secrets-export.sh
+
+# A file that can be used to store secrets for Earthly
+.secret

Earthfile

@@ -191,6 +191,37 @@ sbom-generate:
     # Save the result back to the host:
     SAVE ARTIFACT /s/cyclonedx.sbom.json AS LOCAL etc/cyclonedx.sbom.json
 
+# create-silk-asset-group :
+#   Create an asset group in Silk for the Git branch if one is not already defined.
+#
+# Requires credentials for Silk access.
+#
+# If --branch is not specified, it will be inferred from the current Git branch
+create-silk-asset-group:
+    ARG branch
+    # Get a default value for $branch
+    FROM alpine:3.19
+    IF test "${branch}" = ""
+        LOCALLY
+        LET branch=$(git rev-parse --abbrev-ref HEAD)
+        RUN --no-cache echo "Inferred asset-group name from Git HEAD to be “${branch}”"
+    END
+    # Reset to alpine from the LOCALLY above
+    FROM alpine:3.19
+    RUN apk add python3
+    # Copy in the script
+    COPY tools/create-silk-asset-group.py /opt/
+    # # Run the creation script. Refer to tools/create-silk-asset-group.py for details
+    RUN --no-cache \
+        --secret SILK_CLIENT_ID \
+        --secret SILK_CLIENT_SECRET \
+        python /opt/create-silk-asset-group.py \
+            --branch=${branch} \
+            --project=mongo-c-driver \
+            --code-repo-url=https://github.com/mongodb/mongo-c-driver \
+            --sbom-lite-path=etc/cyclonedx.sbom.json \
+            --exist-ok
+
 # test-vcpkg-classic :
 #   Builds src/libmongoc/examples/cmake/vcpkg by using vcpkg to download and
 #   install a mongo-c-driver build in "classic mode". *Does not* use the local