CDRIVER-6071 replace build-and-test-with-toolchain with openssl-compat (#2082)

* Remove build-and-test-with-toolchain task
* Add openssl-downloader.sh
* CDRIVER-5995: unskip relevant auth tests
* Fix and update run-auth-tests.sh
* Fix and update run-aws-tests.sh
* Fix and update run-ocsp-test.sh
* Migrate OpenSSL compatibility checks into openssl-compat matrix
* Disable failing OCSP darwinssl tasks
* Add verification of shared vs. static linkage of OpenSSL
* Remove obsolete BYPASS_FIND_CMAKE branches
* Support forwarding extra CMake flags to compile_libmongocrypt
* Add reference to CDRIVER-3759

Author: eramongodb

.evergreen/config_generator/components/openssl_compat.py

@@ -0,0 +1,109 @@
+from config_generator.etc.distros import find_large_distro, make_distro_str
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+from config_generator.components.funcs.fetch_source import FetchSource
+from config_generator.components.funcs.find_cmake_latest import FindCMakeLatest
+
+from shrub.v3.evg_build_variant import BuildVariant
+from shrub.v3.evg_command import EvgCommandType, FunctionCall
+from shrub.v3.evg_task import EvgTask, EvgTaskRef
+
+from itertools import product
+
+TAG = 'openssl-compat'
+
+# pylint: disable=line-too-long
+# fmt: off
+OPENSSL_MATRIX = [
+    ('ubuntu2404', 'gcc', ['shared', 'static'], ['1.0.2', '1.1.1', '3.0.9', '3.1.2', '3.2.5', '3.3.4', '3.4.2', '3.5.1']),
+]
+# fmt: on
+
+# pylint: disable=line-too-long
+# fmt: off
+OPENSSL_FIPS_MATRIX = [
+    # https://openssl-library.org/source/
+    # > The following OpenSSL version(s) are FIPS validated:
+    # > - 3.1.2: FIPS 140-3
+    # > - 3.0.9: FIPS 140-2
+    # > - ...
+    ('ubuntu2404', 'gcc', ['shared', 'static'], ['3.0.9', '3.1.2']),
+]
+# fmt: on
+
+
+class OpenSSLSetup(Function):
+    name = 'openssl-compat'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            working_dir='mongoc',
+            include_expansions_in_env=['OPENSSL_VERSION', 'OPENSSL_ENABLE_FIPS', 'OPENSSL_USE_STATIC_LIBS'],
+            script='.evergreen/scripts/openssl-compat-setup.sh',
+        ),
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            working_dir='mongoc',
+            include_expansions_in_env=['OPENSSL_VERSION', 'OPENSSL_USE_STATIC_LIBS'],
+            script='.evergreen/scripts/openssl-compat-check.sh',
+        ),
+    ]
+
+
+def functions():
+    return OpenSSLSetup.defn()
+
+
+def tasks():
+    for distro_name, compiler, link_types, versions in OPENSSL_MATRIX:
+        distro_str = make_distro_str(distro_name, compiler, None)
+
+        for link_type, version in product(link_types, versions):
+            vars = {'OPENSSL_VERSION': version}
+
+            if link_type == 'static':
+                vars |= {'OPENSSL_USE_STATIC_LIBS': 'ON'}
+
+            yield EvgTask(
+                name=f'{TAG}-{version}-{link_type}-{distro_str}',
+                run_on=find_large_distro(distro_name).name,
+                tags=[TAG, f'openssl-{version}', f'openssl-{link_type}', distro_name, compiler],
+                commands=[
+                    FetchSource.call(),
+                    FindCMakeLatest.call(),
+                    OpenSSLSetup.call(vars=vars),
+                    FunctionCall(func="run auth tests"),
+                ],
+            )
+
+    for distro_name, compiler, link_types, versions in OPENSSL_FIPS_MATRIX:
+        distro_str = make_distro_str(distro_name, compiler, None)
+
+        for link_type, version in product(link_types, versions):
+            vars = {'OPENSSL_VERSION': version, 'OPENSSL_ENABLE_FIPS': 'ON'}
+
+            if link_type == 'static':
+                vars |= {'OPENSSL_USE_STATIC_LIBS': 'ON'}
+
+            yield EvgTask(
+                name=f'{TAG}-fips-{version}-{link_type}-{distro_str}',
+                run_on=find_large_distro(distro_name).name,
+                tags=[TAG, f'openssl-fips-{version}', f'openssl-{link_type}', distro_name, compiler],
+                commands=[
+                    FetchSource.call(),
+                    FindCMakeLatest.call(),
+                    OpenSSLSetup.call(vars=vars),
+                    FunctionCall(func="run auth tests"),
+                ],
+            )
+
+
+def variants():
+    return [
+        BuildVariant(
+            name=f'{TAG}-matrix',
+            display_name='OpenSSL Compatibility Matrix',
+            tasks=[EvgTaskRef(name=f'.{TAG}')],
+        ),
+    ]

.evergreen/config_generator/components/openssl_static_compile.py

@@ -327,16 +327,30 @@ functions:
           - |
             # See SphinxBuild.cmake for EVG_DOCS_BUILD reasoning
             uv run --frozen --only-group docs env EVG_DOCS_BUILD=1 .evergreen/scripts/build-docs.sh
-  openssl-static-compile:
+  openssl-compat:
     - command: subprocess.exec
-      type: test
+      type: setup
       params:
         binary: bash
         working_dir: mongoc
-        add_expansions_to_env: true
+        include_expansions_in_env:
+          - OPENSSL_VERSION
+          - OPENSSL_ENABLE_FIPS
+          - OPENSSL_USE_STATIC_LIBS
+        args:
+          - -c
+          - .evergreen/scripts/openssl-compat-setup.sh
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        working_dir: mongoc
+        include_expansions_in_env:
+          - OPENSSL_VERSION
+          - OPENSSL_USE_STATIC_LIBS
         args:
           - -c
-          - .evergreen/scripts/compile-openssl-static.sh
+          - .evergreen/scripts/openssl-compat-check.sh
   restore-instance-profile:
     - command: subprocess.exec
       params: