Fix and update EVG configuration for r1.30 (#2076)

* CDRIVER-5891 address implicit bool-to-null conversion in return statements (#1888)
* CDRIVER-5891 extend C standard coverage up to C23 (#1890)
* CDRIVER-5959 sync change streams unified spec tests with 0aee4aad  (#1971)
* CDRIVER-5304 set cluster time in test sessions (#2008)
* Ignore -Wpre-c11-compat warnings when building public-header-warnings (#2027)
* CDRIVER-6042 Migrate Python scripts from Poetry to Astral UV (#2039)
* Migrate EVG task coverage to latest Debian, Ubuntu, and RHEL distros (#2044)
* CDRIVER-6010 use `ec2.assume_role` for Azure KMS task (#2051)
* CDRIVER-5971 Use Amazon ECR to obtain OCI images in EVG (#2058)
* update Earthly version from 0.8.3 to 0.8.16 (#2072)

---------

Co-authored-by: Kevin Albertson <[email protected]>

Author: eramongodb

.evergreen/config_generator/README.md

@@ -3,36 +3,22 @@
 The scripts in this directory are used to generate the Evergreen configuration
 files stored in `.evergreen/generated_configs/`.
 
-The easiest way to execute these scripts is using the Poetry to install the
-dependencies and then run the scripts.
+The easiest way to execute these scripts is to use [uv](https://docs.astral.sh/uv/) to run the scripts.
 
 **Note**: These scripts require Python 3.10 or newer.
 
 
 ## Setting Up
 
-Before running, use Poetry to install a virtualenv containing the dependencies.
-This can be done by using the `poetry.sh` (or `poetry.ps1`) script contained in
-the `tools/` directory at the root of the `mongo-c-driver` repository:
-
-```sh
-./tools/poetry.sh install --with=dev
-```
-
-Or with PowerShell:
-
-```pwsh
-./tools/poetry.ps1 install --with=dev
-```
+`uv` is required to run Python scripts. See ["Installing uv"](https://docs.astral.sh/uv/getting-started/installation/) for instructions on how to obtain `uv`.
 
 
 ## Running the Generator
 
-The package defines a program `mc-evg-generate`, which can be run within the
-virtualenv. This can be done via Poetry as well, following the setup:
+The package provides the `mc-evg-generate` [entry point](https://packaging.python.org/en/latest/specifications/entry-points):
 
 ```sh
-./tools/poetry.sh run mc-evg-generate
+uv run --frozen mc-evg-generate
 ```
 
 This command will ready the generation files and generate a new set of Evergreen

.evergreen/config_generator/components/c_std_compile.py

@@ -6,7 +6,7 @@
 
 from config_generator.etc.distros import find_large_distro
 from config_generator.etc.distros import make_distro_str
-from config_generator.etc.distros import to_cc
+from config_generator.etc.distros import compiler_to_vars
 from config_generator.etc.function import Function
 from config_generator.etc.utils import bash_exec
 
@@ -17,14 +17,31 @@
 # pylint: disable=line-too-long
 # fmt: off
 MATRIX = [
-    ('debian92',          'clang',     None,   [11,   ]),
-    ('debian10',          'clang',     None,   [11,   ]),
-    ('debian10',          'gcc',       None,   [11, 17]),
-    ('debian11',          'clang',     None,   [11,   ]),
-    ('debian11',          'gcc',       None,   [11, 17]),
-    ('ubuntu2004',        'clang',     None,   [11,   ]),
-    ('ubuntu2004',        'gcc',       None,   [11,   ]),
-    ('windows-vsCurrent', 'vs2017x64', None,   [11, 17]),
+    ('rhel80',     'clang',    None, [99, 11, 17,   ]), # Clang 7.0
+    ('ubuntu2004', 'clang-10', None, [99, 11, 17, 23]), # Clang 10.0 (max: C2x)
+    ('rhel84',     'clang',    None, [99, 11, 17, 23]), # Clang 11.0 (max: C2x)
+    ('ubuntu2204', 'clang-12', None, [99, 11, 17, 23]), # Clang 12.0 (max: C2x)
+    ('rhel90',     'clang',    None, [99, 11, 17, 23]), # Clang 13.0 (max: C2x)
+    ('rhel91',     'clang',    None, [99, 11, 17, 23]), # Clang 14.0 (max: C2x)
+    ('rhel92',     'clang',    None, [99, 11, 17, 23]), # Clang 15.0 (max: C2x)
+    ('rhel93',     'clang',    None, [99, 11, 17, 23]), # Clang 16.0 (max: C2x)
+    ('rhel94',     'clang',    None, [99, 11, 17, 23]), # Clang 17.0 (max: C2x)
+    ('rhel95',     'clang',    None, [99, 11, 17, 23]), # Clang 18.0 (max: C23)
+
+    ('rhel7-latest', 'gcc',    None, [99, 11,       ]), # GCC 4.8 (max: C11)
+    ('rhel80',       'gcc',    None, [99, 11, 17,   ]), # GCC 8.2 (max: C17)
+    ('rhel84',       'gcc',    None, [99, 11, 17,   ]), # GCC 8.4 (max: C17)
+    ('ubuntu2004',   'gcc-9',  None, [99, 11, 17, 23]), # GCC 9.4 (max: C2x)
+    ('debian11',     'gcc-10', None, [99, 11, 17, 23]), # GCC 10.2 (max: C2x)
+    ('rhel90',       'gcc',    None, [99, 11, 17, 23]), # GCC 11.2 (max: C2x)
+    ('rhel92',       'gcc',    None, [99, 11, 17, 23]), # GCC 11.3 (max: C2x)
+    ('rhel94',       'gcc',    None, [99, 11, 17, 23]), # GCC 11.4 (max: C2x)
+    ('rhel95',       'gcc',    None, [99, 11, 17, 23]), # GCC 11.5 (max: C2x)
+    ('ubuntu2404',   'gcc-13', None, [99, 11, 17, 23]), # GCC 13.3 (max: C2x)
+
+    ('windows-vsCurrent', 'vs2017x64', None, [99, 11, 17, 'latest']), # Max: C17, clatest (C2x)
+    ('windows-vsCurrent', 'vs2019x64', None, [99, 11, 17, 'latest']), # Max: C17, clatest (C2x)
+    ('windows-vsCurrent', 'vs2022x64', None, [99, 11, 17, 'latest']), # Max: C17, clatest (C2x)
 ]
 # fmt: on
 # pylint: enable=line-too-long
@@ -54,18 +71,20 @@ def tasks():
     res = []
 
     for distro_name, compiler, arch, stds in MATRIX:
-        tags = [TAG, distro_name, compiler, 'compile']
+        compiler_type = compiler.split('-')[0]
+
+        tags = [TAG, distro_name, compiler_type, 'compile']
 
         distro = find_large_distro(distro_name)
 
         compile_vars = None
-        compile_vars = {'CC': to_cc(compiler)}
+        compile_vars = compiler_to_vars(compiler)
 
         if arch:
             tags.append(arch)
             compile_vars.update({'MARCH': arch})
 
-        distro_str = make_distro_str(distro_name, compiler, arch)
+        distro_str = make_distro_str(distro_name, compiler_type, arch)
 
         for std in stds:
             with_std = {'C_STD_VERSION': std}

.evergreen/config_generator/components/clang_format.py

@@ -3,34 +3,24 @@
 from shrub.v3.evg_task import EvgTask
 from shrub.v3.evg_task import EvgTaskRef
 
-from config_generator.etc.distros import find_large_distro
+from config_generator.etc.distros import find_small_distro
 from config_generator.etc.function import Function
 from config_generator.etc.utils import bash_exec
 
 
 TAG = "clang-format"
 
-DISTROS = [
-    find_large_distro("ubuntu2204").name,
-    find_large_distro("ubuntu2004").name,
-]
-
 
 class ClangFormat(Function):
     name = TAG
     commands = [
-        bash_exec(
-            command_type=EvgCommandType.SETUP,
-            working_dir="mongoc",
-            script="./tools/poetry.sh install --with=dev"
-        ),
         bash_exec(
             command_type=EvgCommandType.TEST,
             working_dir="mongoc",
             env={
                 "DRYRUN": "1",
             },
-            script="./tools/poetry.sh run .evergreen/scripts/clang-format-all.sh",
+            script="uv run --frozen --only-group format .evergreen/scripts/clang-format-all.sh",
         ),
     ]
 
@@ -56,7 +46,7 @@ def variants():
         BuildVariant(
             name=TAG,
             display_name=TAG,
-            run_on=DISTROS,
+            run_on=[find_small_distro("ubuntu2204").name],
             tasks=[EvgTaskRef(name=f'.{TAG}')],
         ),
     ]

.evergreen/config_generator/components/cse/openssl.py

@@ -15,35 +15,40 @@
 # pylint: disable=line-too-long
 # fmt: off
 COMPILE_MATRIX = [
-    ('debian92', 'clang', None, ['cyrus']),
-    ('debian92', 'gcc',   None, ['cyrus']),
-    ('debian10', 'clang', None, ['cyrus']),
-    ('debian10', 'gcc',   None, ['cyrus']),
-    ('debian11', 'clang', None, ['cyrus']),
-    ('debian11', 'gcc',   None, ['cyrus']),
-    ('rhel80',            'gcc',       None, ['cyrus']),
+    # For test matrix.
+    ('rhel8-latest',      'gcc',       None, ['cyrus']),
     ('rhel8-zseries',     'gcc',       None, ['cyrus']),
-    ('ubuntu2004',        'clang',     None, ['cyrus']),
-    ('ubuntu2004',        'gcc',       None, ['cyrus']),
     ('ubuntu2004-arm64',  'gcc',       None, ['cyrus']),
     ('windows-vsCurrent', 'vs2017x64', None, ['cyrus']),
+
+    # For compile only.
+    ('debian11',   'clang',    None, ['cyrus']),
+    ('debian11',   'gcc',      None, ['cyrus']),
+    ('debian12',   'clang',    None, ['cyrus']),
+    ('debian12',   'gcc',      None, ['cyrus']),
+    ('rhel80',     'gcc',      None, ['cyrus']),
+    ('ubuntu2004', 'gcc',      None, ['cyrus']),
+    ('ubuntu2004', 'clang',    None, ['cyrus']),
+    ('ubuntu2204', 'gcc',      None, ['cyrus']),
+    ('ubuntu2204', 'clang-12', None, ['cyrus']),
+    ('ubuntu2404', 'gcc',      None, ['cyrus']),
+    ('ubuntu2404', 'clang-14', None, ['cyrus']),
 ]
 
 # TODO (CDRIVER-3789): test cse with the 'sharded' topology.
+# CSFLE requires 4.2+. QE requires 7.0+ and are skipped on "server" tasks.
 TEST_MATRIX = [
-    # 4.2 and 4.4 not available on rhel8-zseries.
-    ('rhel8-zseries', 'gcc', None, 'cyrus', ['auth'], ['server'], ['5.0']),
+    # rhel8-latest provides 4.2+.
+    ('rhel8-latest', 'gcc', None, 'cyrus', ['auth'], ['server', 'replica'], ['4.2', '4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
 
-    ('windows-vsCurrent', 'vs2017x64', None, 'cyrus', ['auth'], ['server'], ['4.2', '4.4', '5.0', '6.0' ]),
+    # windows-vsCurrent provides 4.2+.
+    ('windows-vsCurrent', 'vs2017x64', None, 'cyrus', ['auth'], ['server', 'replica'], ['4.2', '4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
 
-    # Test 7.0+ with a replica set since Queryable Encryption does not support the 'server' topology. Queryable Encryption tests require 7.0+.
-    ('ubuntu2004',        'gcc',       None, 'cyrus', ['auth'], ['server', 'replica'], ['4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
-    ('rhel8-zseries',     'gcc',       None, 'cyrus', ['auth'], ['server', 'replica'], [ '7.0', '8.0', 'latest']),
-    ('ubuntu2004-arm64',  'gcc',       None, 'cyrus', ['auth'], ['server', 'replica'], ['4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
-    ('windows-vsCurrent', 'vs2017x64', None, 'cyrus', ['auth'], ['server', 'replica'], [ '7.0', '8.0', 'latest']),
+    # ubuntu2004-arm64 provides 4.4+.
+    ('ubuntu2004-arm64', 'gcc', None, 'cyrus', ['auth'], ['server', 'replica'], ['4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
 
-    # Test 4.2 with Debian 10 since 4.2 does not ship on Ubuntu 20.04+.
-    ('debian10',          'gcc',       None, 'cyrus', ['auth'], ['server', 'replica'], ['4.2']),
+    # rhel8-zseries provides 5.0+.
+    ('rhel8-zseries', 'gcc', None, 'cyrus', ['auth'], ['server', 'replica'], ['5.0', '6.0', '7.0', '8.0', 'latest']),
 ]
 # fmt: on
 # pylint: enable=line-too-long
@@ -106,6 +111,8 @@ def variants():
         else:
             tasks.append(task.get_task_ref())
 
+    tasks.sort(key=lambda t: t.name)
+
     return [
         BuildVariant(
             name=TAG,

.evergreen/config_generator/components/earthly.py

@@ -5,9 +5,18 @@
 from typing import Iterable, Literal, Mapping, NamedTuple, TypeVar
 
 from shrub.v3.evg_build_variant import BuildVariant
-from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, subprocess_exec
+from shrub.v3.evg_command import (
+    BuiltInCommand,
+    EvgCommandType,
+    KeyValueParam,
+    ec2_assume_role,
+    expansions_update,
+    subprocess_exec,
+)
 from shrub.v3.evg_task import EvgTask, EvgTaskRef
 
+from config_generator.etc.function import Function
+
 from ..etc.utils import all_possible
 
 T = TypeVar("T")
@@ -138,6 +147,34 @@ def suffix(self) -> str:
         return f"{_SEPARATOR}".join(f"{k}={v}" for k, v in self._asdict().items())
 
 
+# Authenticate with DevProd-provided Amazon ECR instance to use as pull-through cache for DockerHub.
+class DockerLoginAmazonECR(Function):
+    name = 'docker-login-amazon-ecr'
+    commands = [
+        # Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
+        expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
+        ec2_assume_role(role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"),
+        subprocess_exec(
+            binary="bash",
+            command_type=EvgCommandType.SETUP,
+            include_expansions_in_env=[
+                "AWS_ACCESS_KEY_ID",
+                "AWS_SECRET_ACCESS_KEY",
+                "AWS_SESSION_TOKEN",
+                "DOCKER_CONFIG",
+            ],
+            args=[
+                "-c",
+                'aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
+            ],
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
 def task_filter(env: EarthlyVariant, conf: Configuration) -> bool:
     """
     Control which tasks are actually defined by matching on the platform and
@@ -189,11 +226,16 @@ def earthly_exec(
     return subprocess_exec(
         "./tools/earthly.sh",
         args=[
+            # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits.
+            "--buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3",
             *(f"--secret={k}" for k in (secrets or ())),
             f"+{target}",
+            # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits.
+            "--default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub",
             *(f"--{arg}={val}" for arg, val in (args or {}).items()),
         ],
         command_type=EvgCommandType(kind),
+        include_expansions_in_env=["DOCKER_CONFIG"],
         env=env if env else None,
         working_dir="mongoc",
     )
@@ -228,15 +270,7 @@ def earthly_task(
     return EvgTask(
         name=name,
         commands=[
-            # Ensure subsequent Docker commands are authenticated.
-            subprocess_exec(
-                binary='bash',
-                command_type="setup",
-                args=[
-                    "-c",
-                    'docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"'
-                ],
-            ),
+            DockerLoginAmazonECR.call(),
             # First, just build the "env-warmup" which will prepare the build environment.
             # This won't generate any output, but allows EVG to track it as a separate build step
             # for timing and logging purposes. The subequent build step will cache-hit the
@@ -260,13 +294,18 @@ def earthly_task(
 
 
 CONTAINER_RUN_DISTROS = [
-    "ubuntu2204-large",
-    "debian10-large",
-    "debian11-large",
     "amazon2",
+    "debian11-large",
+    "debian12-large",
+    "ubuntu2204-large",
+    "ubuntu2404-large",
 ]
 
 
+def functions():
+    return DockerLoginAmazonECR.defn()
+
+
 def tasks() -> Iterable[EvgTask]:
     for conf in all_possible(Configuration):
         # test-example is a target in all configurations

.evergreen/config_generator/components/loadbalanced.py

@@ -12,8 +12,8 @@
 from config_generator.etc.distros import make_distro_str, find_small_distro, find_large_distro
 from config_generator.etc.utils import bash_exec
 
-# Use `rhel8.9` distro. `rhel8.9` distro includes necessary dependency: `haproxy`.
-_DISTRO_NAME = "rhel8.9"
+# Use `rhel8-latest` distro. `rhel8-latest` distro includes necessary dependency: `haproxy`.
+_DISTRO_NAME = "rhel8-latest"
 _COMPILER = "gcc"
 
 

.evergreen/config_generator/components/make_docs.py

@@ -17,11 +17,9 @@ class MakeDocs(Function):
             working_dir="mongoc",
             include_expansions_in_env=["distro_id"],
             script="""\
-                set -o errexit
-                ./tools/poetry.sh install --with=docs
                 # See SphinxBuild.cmake for EVG_DOCS_BUILD reasoning
-                ./tools/poetry.sh run env EVG_DOCS_BUILD=1 .evergreen/scripts/build-docs.sh
-                """,
+                uv run --frozen --only-group docs env EVG_DOCS_BUILD=1 .evergreen/scripts/build-docs.sh
+            """,
         ),
     ]