[CDRIVER-4153] Beginning of Header Hygiene & Verification (#2053)

* Prepare some core header files for public use.

This change starts a new pattern for public API
headers: Headers that omit the "bson-" or "mongoc-"
prefix are intended for public `#include` directives, allowing users
to selectively import the APIs that they want.

This change required minimal modification of the existing code, as the
few headers that have been modified are very basic and have very few
dependencies.

* Fix C++ and C compatibility in some header files
* Add CMake code for header verification.

This adds a CMake utility based around
VERIFY_INTERFACE_HEADER_SETS to check that certain headers can be
included directly in a translation unit without any prerequesits, in
either C or C++ code.

This verification only concerns itself with headers that we have already
made part of the public API. Private header files and header files
that are not-yet-sanitized are not checked by this process.

The verification does not yet happen automatically. It is only enabled
for CMake 3.24 or newer, and requires the dev/CI to build the special
linting target the CMake generates.

* Earthly target and EVG tasks to run header verification
* Remove unused old script
* Rename bson.h -> bson-bcon.h

Author: vector-of-bool

.evergreen/config_generator/components/earthly.py

@@ -68,7 +68,9 @@ def os_split(env: EnvKey) -> tuple[str, None | str]:
         case "centos7":
             return "CentOS", "7.0"
         case _:
-            raise ValueError(f"Failed to split OS env key {env=} into a name+version pair (unrecognized)")
+            raise ValueError(
+                f"Failed to split OS env key {env=} into a name+version pair (unrecognized)"
+            )
 
 
 class EarthlyVariant(NamedTuple):
@@ -147,11 +149,15 @@ def suffix(self) -> str:
 
 # Authenticate with DevProd-provided Amazon ECR instance to use as pull-through cache for DockerHub.
 class DockerLoginAmazonECR(Function):
-    name = 'docker-login-amazon-ecr'
+    name = "docker-login-amazon-ecr"
     commands = [
         # Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
-        expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
-        ec2_assume_role(role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"),
+        expansions_update(
+            updates=[KeyValueParam(key="DOCKER_CONFIG", value="${workdir}/.docker")]
+        ),
+        ec2_assume_role(
+            role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"
+        ),
         subprocess_exec(
             binary="bash",
             command_type=EvgCommandType.SETUP,
@@ -163,7 +169,7 @@ class DockerLoginAmazonECR(Function):
             ],
             args=[
                 "-c",
-                'aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
+                "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com",
             ],
         ),
     ]
@@ -176,7 +182,9 @@ def task_filter(env: EarthlyVariant, conf: Configuration) -> bool:
     """
     match env, conf:
         # u16/u18/centos7 are not capable of building mongocxx
-        case e, (_sasl, _tls, cxx) if re.match(r"^Ubuntu 16|^Ubuntu 18|^CentOS 7", e.display_name):
+        case e, (_sasl, _tls, cxx) if re.match(
+            r"^Ubuntu 16|^Ubuntu 18|^CentOS 7", e.display_name
+        ):
             # Only build if C++ driver is test is disabled
             return cxx == "none"
         # Anything else: Allow it to run:
@@ -300,6 +308,13 @@ def tasks() -> Iterable[EvgTask]:
         if task is not None:
             yield task
 
+    yield EvgTask(
+        name="verify-headers",
+        commands=[earthly_exec(kind="test", target="verify-headers")],
+        tags=["pr-merge-gate"],
+        run_on=CONTAINER_RUN_DISTROS,
+    )
+
 
 def variants() -> Iterable[BuildVariant]:
     yield from (ev.as_evg_variant() for ev in all_possible(EarthlyVariant))

.evergreen/generated_configs/legacy-config.yml

@@ -16250,6 +16250,7 @@ buildvariants:
   - link-with-cmake
   - link-with-cmake-ssl
   - link-with-cmake-snappy
+  - verify-headers
   - name: link-with-cmake-mac
     distros:
     - macos-14-arm64

.evergreen/generated_configs/tasks.yml

@@ -6772,3 +6772,19 @@ tasks:
       - func: bootstrap-mongo-orchestration
       - func: run-simple-http-server
       - func: run-tests
+  - name: verify-headers
+    run_on:
+      - amazon2
+      - debian11-large
+      - debian12-large
+      - ubuntu2204-large
+      - ubuntu2404-large
+    tags: [pr-merge-gate]
+    commands:
+      - command: subprocess.exec
+        type: test
+        params:
+          binary: ./tools/earthly.sh
+          working_dir: mongoc
+          args:
+            - +verify-headers

.evergreen/legacy_config_generator/evergreen_config_lib/variants.py

@@ -50,6 +50,7 @@ def days(n: int) -> int:
             "link-with-cmake",
             "link-with-cmake-ssl",
             "link-with-cmake-snappy",
+            "verify-headers",
             OD([("name", "link-with-cmake-mac"), ("distros", ["macos-14-arm64"])]),
             OD([("name", "link-with-cmake-windows"), ("distros", ["windows-vsCurrent-large"])]),
             OD([("name", "link-with-cmake-windows-ssl"), ("distros", ["windows-vsCurrent-large"])]),
@@ -84,7 +85,7 @@ def days(n: int) -> int:
             "build-and-run-authentication-tests-openssl-1.0.1",
             "build-and-run-authentication-tests-openssl-1.0.2",
             "build-and-run-authentication-tests-openssl-1.1.0",
-            "build-and-run-authentication-tests-openssl-1.0.1-fips"
+            "build-and-run-authentication-tests-openssl-1.0.1-fips",
         ],
         {},
     ),
@@ -356,7 +357,7 @@ def days(n: int) -> int:
                 "name": "ocsp-openssl-1.0.1",
                 "execution_tasks": [".ocsp-openssl-1.0.1"],
             },
-        ]
+        ],
     ),
     Variant(
         "packaging",
@@ -380,7 +381,7 @@ def days(n: int) -> int:
             ".versioned-api .5.0",
             ".versioned-api .6.0",
             ".versioned-api .7.0",
-            ".versioned-api .8.0"
+            ".versioned-api .8.0",
         ],
         {},
     ),

.evergreen/scripts/check-files.py

@@ -39,7 +39,7 @@
     },
     {
         "name": "libbson",
-        "headers": list(BSON_PREFIX.glob("*.h")),
+        "headers": list(BSON_PREFIX.glob("bson-*.h")),
         "exclusions": [
             BSON_PREFIX / "bson-prelude.h",
             BSON_PREFIX / "bson.h",

.evergreen/scripts/check-preludes.py

@@ -28,6 +28,7 @@ include(FeatureSummary)
 include (MongoSettings)
 include (MongoPlatform)
 include (GeneratePkgConfig)
+include (VerifyHeaders)
 
 # Subcomponents:
 mongo_bool_setting(ENABLE_MONGOC "Enable the build of libmongoc libraries (The MongoDB C database driver)")

CMakeLists.txt

@@ -9,25 +9,11 @@ IMPORT ./tools/ AS tools
 # For target names, descriptions, and build parameters, run the "doc" Earthly subcommand.
 # Example use: <earthly> +build --env=u22 --sasl=off --tls=OpenSSL --c_compiler=gcc
 
-# build :
-#   Build libmongoc and libbson using the specified environment.
-#
-# The --env argument specifies the build environment among the `+env.xyz` environment
-# targets, using --purpose=build for the build environment. Refer to the target
-# list for a list of available build environments.
-build:
-    # env is an argument
-    ARG --required env
-    FROM --pass-args +env.$env --purpose=build
-    # The configuration to be built
-    ARG config=RelWithDebInfo
-    # The prefix at which to install the built result
-    ARG install_prefix=/opt/mongo-c-driver
-    # Build configuration parameters. Will be case-normalized for CMake usage.
-    ARG --required sasl
-    ARG --required tls
-    LET source_dir=/opt/mongoc/source
-    LET build_dir=/opt/mongoc/build
+# COPY_SOURCE :
+#   Copy source files required for the build into the specified "--into" directory
+COPY_SOURCE:
+    FUNCTION
+    ARG --required into
     COPY --dir \
         build/ \
         CMakeLists.txt \
@@ -37,8 +23,16 @@ build:
         src/ \
         THIRD_PARTY_NOTICES \
         VERSION_CURRENT \
-        "$source_dir"
-    ENV CCACHE_HOME=/root/.cache/ccache
+        "$into"
+
+# CONFIGURE :
+#   Configure the project in $source_dir into $build_dir with a common set of configuration options
+CONFIGURE:
+    FUNCTION
+    ARG --required source_dir
+    ARG --required build_dir
+    ARG --required tls
+    ARG --required sasl
     RUN cmake -S "$source_dir" -B "$build_dir" -G "Ninja Multi-Config" \
         -D ENABLE_MAINTAINER_FLAGS=ON \
         -D ENABLE_SHM_COUNTERS=ON \
@@ -50,6 +44,29 @@ build:
         -D ENABLE_COVERAGE=ON \
         -D ENABLE_DEBUG_ASSERTIONS=ON \
         -Werror
+
+# build :
+#   Build libmongoc and libbson using the specified environment.
+#
+# The --env argument specifies the build environment among the `+env.xyz` environment
+# targets, using --purpose=build for the build environment. Refer to the target
+# list for a list of available build environments.
+build:
+    # env is an argument
+    ARG --required env
+    FROM --pass-args +env.$env --purpose=build
+    # The configuration to be built
+    ARG config=RelWithDebInfo
+    # The prefix at which to install the built result
+    ARG install_prefix=/opt/mongo-c-driver
+    # Build configuration parameters. Will be case-normalized for CMake usage.
+    ARG --required sasl
+    ARG --required tls
+    LET source_dir=/opt/mongoc/source
+    LET build_dir=/opt/mongoc/build
+    DO +COPY_SOURCE --into=$source_dir
+    ENV CCACHE_HOME=/root/.cache/ccache
+    DO --pass-args +CONFIGURE --source_dir=$source_dir --build_dir=$build_dir
     RUN --mount=type=cache,target=$CCACHE_HOME \
         env CCACHE_BASE="$source_dir" \
             cmake --build $build_dir --config $config
@@ -399,6 +416,34 @@ vcpkg-base:
     COPY src/libmongoc/examples/cmake/vcpkg/ $src_dir
     WORKDIR $src_dir
 
+# verify-headers :
+#   Execute CMake header verification on the sources
+#
+#   See `earthly.rst` for more details.
+verify-headers:
+    # We test against multiple different platforms, because glibc/musl versions may
+    # rearrange their header contents and requirements, so we want to check against as
+    # many as possible.
+    BUILD +do-verify-headers-impl \
+        --from +env.alpine3.19 \
+        --from +env.u22 \
+        --from +env.centos7 \
+        --sasl=off --tls=off --cxx_compiler=gcc --c_compiler=gcc
+
+do-verify-headers-impl:
+    ARG --required from
+    # We don't really care about the specifics of the build env/settings, so set some
+    # reasonable defaults so the caller doesn't need to specify. In the future, it is
+    # possible that we will need to test other environments and build settings.
+    FROM --pass-args "$from" --purpose=build
+    # Add C++ so we can test as C++ headers
+    DO --pass-args tools+ADD_CXX_COMPILER
+    DO +COPY_SOURCE --into=/s
+    DO --pass-args +CONFIGURE --source_dir /s --build_dir /s/_build
+    # The "all_verify_interface_header_sets" target is created automatically
+    # by CMake for the VERIFY_INTERFACE_HEADER_SETS target property.
+    RUN cmake --build /s/_build --target all_verify_interface_header_sets
+
 # run :
 #   Run one or more targets simultaneously.
 #