PHPC-2011: Upgrade libmongoc to 1.20.0 and libmongocrypt to 1.3.0 (#1279)

* PHPC-2011: Bump libmongocrypt to 1.3.0

* PHPC-2011: Bump libmongoc to 1.20.0

* Script to update submodule sources in config.m4 and config.w32

Author: jmikola

CONTRIBUTING.md

@@ -136,7 +136,7 @@ reflect the new sources and/or package version.
 ```
 $ cd src/libmongoc
 $ git fetch
-$ git checkout 1.15.0
+$ git checkout 1.20.0
 ```
 
 During development, it may be necessary to temporarily point the libmongoc
@@ -175,9 +175,10 @@ collectively enumerate all of the the sources within the libmongoc submodule to
 include in a bundled build.
 
 These variables should each have a shell command in a preceding comment, which
-should be run to regenerate that particular list of source files. In the event
-that either libmongoc or libbson introduce a new source directory, that will
-need to be manually added (follow prior art).
+should be run to regenerate that particular list of source files. Each command
+may be run manually or `scripts/update-submodule-sources.php` may be used to
+update all variables. In the event that either libmongoc or libbson introduce a
+new source directory, that will need to be manually added (follow prior art).
 
 #### Update package dependencies
 
@@ -189,11 +190,11 @@ error message in the `pkg-config` blocks for both libmongoc and libbson.
 For example, the following lines might be updated for libmongoc:
 
 ```
-if $PKG_CONFIG libmongoc-1.0 --atleast-version 1.15.0; then
+if $PKG_CONFIG libmongoc-1.0 --atleast-version 1.20.0; then
 
 ...
 
-AC_MSG_ERROR(system libmongoc must be upgraded to version >= 1.15.0)
+AC_MSG_ERROR(system libmongoc must be upgraded to version >= 1.20.0)
 ```
 
 #### Update tested versions in Evergreen configuration
@@ -221,7 +222,7 @@ and by ensuring that the test suite passes. Once done, commit the changes to all
 of the above files/paths. For example:
 
 ```
-$ git commit -m "Bump libmongoc to 1.15.0" config.m4 config.w32 src/libmongoc src/LIBMONGOC_VERSION_CURRENT
+$ git commit -m "Bump libmongoc to 1.20.0" config.m4 config.w32 src/libmongoc src/LIBMONGOC_VERSION_CURRENT
 ```
 
 ### Updating libmongocrypt
@@ -231,12 +232,13 @@ To update libmongocrypt, the steps are similar to the above:
 ```
 $ cd src/libmongocrypt
 $ git fetch
-$ git checkout 1.0.1
+$ git checkout 1.3.0
 $ make libmongocrypt-version-current
 ```
 
-Package dependencies in  `config.m4` must also be updated, as do the sources in
-the PECL generation script.
+Package dependencies in  `config.m4` must also be updated (either manually or
+with `scripts/update-submodule-sources.php`), as do the sources in the PECL
+generation script.
 
 ## Releasing
 

config.m4

@@ -338,6 +338,7 @@ if test "$PHP_MONGODB" != "no"; then
     dnl This include modifies the value of $PHP_MONGODB_CLIENT_SIDE_ENCRYPTION to "yes"
     dnl or "no" depending on whether dependencies for libmongocrypt are fulfilled
     _include([scripts/autotools/libmongocrypt/CheckSSL.m4])
+    _include([scripts/autotools/libmongocrypt/Endian.m4])
     _include([scripts/autotools/libmongocrypt/Version.m4])
 
     PHP_MONGODB_BSON_VERSION_STRING="Bundled ($BSON_VERSION)"
@@ -443,7 +444,7 @@ if test "$PHP_MONGODB" != "no"; then
       AC_SUBST(MONGOCRYPT_ENABLE_TRACE, 1)
 
       dnl Generated with: find src/libmongocrypt/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 4- | sort -dz | tr '\000' ' '
-      PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c"
+      PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c mongocrypt-util.c"
 
       dnl Generated with: find src/libmongocrypt/src/crypto -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' '
       PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES="cng.c commoncrypto.c libcrypto.c none.c"
@@ -455,7 +456,7 @@ if test "$PHP_MONGODB" != "no"; then
       PHP_MONGODB_MONGOCRYPT_OS_WIN_SOURCES="os_mutex.c os_once.c"
 
       dnl Generated with: find src/libmongocrypt/kms-message/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' '
-      PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c"
+      PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kmip_reader_writer.c kms_kmip_request.c kms_kmip_response.c kms_kmip_response_parser.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c"
 
       PHP_MONGODB_LIBMONGOCRYPT_CFLAGS="$PHP_MONGODB_BUNDLED_CFLAGS $PHP_MONGODB_LIBMONGOCRYPT_CFLAGS"
       PHP_MONGODB_ADD_SOURCES([src/libmongocrypt/src/], $PHP_MONGODB_MONGOCRYPT_SOURCES, $PHP_MONGODB_LIBMONGOCRYPT_CFLAGS)

config.w32

@@ -264,6 +264,7 @@ if (PHP_MONGODB != "no") {
   if (PHP_MONGODB_CLIENT_SIDE_ENCRYPTION != "no" && mongoc_ssl_found) {
     mongoc_opts.MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION = 1;
 
+    ADD_FLAG("CFLAGS_MONGODB", "/D KMS_MESSAGE_LITTLE_ENDIAN=1");
     ADD_FLAG("CFLAGS_MONGODB", "/I" + configure_module_dirname + "/src/libmongocrypt/src");
     ADD_FLAG("CFLAGS_MONGODB", "/I" + configure_module_dirname + "/src/libmongocrypt/kms-message/src");
     ADD_FLAG("CFLAGS_MONGODB", "/I" + configure_module_dirname + "/src/libmongocrypt-compat");
@@ -279,7 +280,7 @@ if (PHP_MONGODB != "no") {
     };
 
     // Generated with: find src/libmongocrypt/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 4- | sort -dz | tr '\000' ' '
-    var PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c";
+    var PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c mongocrypt-util.c";
 
     // Generated with: find src/libmongocrypt/src/crypto -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' '
     var PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES="cng.c commoncrypto.c libcrypto.c none.c";
@@ -291,7 +292,7 @@ if (PHP_MONGODB != "no") {
     var PHP_MONGODB_MONGOCRYPT_OS_WIN_SOURCES="os_mutex.c os_once.c";
 
     // Generated with: find src/libmongocrypt/kms-message/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' '
-    var PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c";
+    var PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kmip_reader_writer.c kms_kmip_request.c kms_kmip_response.c kms_kmip_response_parser.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c";
 
     MONGODB_ADD_SOURCES("/src/libmongocrypt/src", PHP_MONGODB_MONGOCRYPT_SOURCES);
     MONGODB_ADD_SOURCES("/src/libmongocrypt/src/crypto", PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES);

scripts/autotools/libmongocrypt/Endian.m4

@@ -0,0 +1,6 @@
+AC_C_BIGENDIAN
+if test "x$ac_cv_c_bigendian" = "xyes"; then
+    PHP_MONGODB_BUNDLED_CFLAGS="$PHP_MONGODB_BUNDLED_CFLAGS -DKMS_MESSAGE_BIG_ENDIAN=1"
+else
+    PHP_MONGODB_BUNDLED_CFLAGS="$PHP_MONGODB_BUNDLED_CFLAGS -DKMS_MESSAGE_LITTLE_ENDIAN=1"
+fi

scripts/update-submodule-sources.php

@@ -0,0 +1,47 @@
+<?php
+
+$cmd = "find %s -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f %d- | sort -dz | tr '\\000' ' '";
+
+$vars = [
+    'PHP_MONGODB_COMMON_SOURCES' => 'src/libmongoc/src/common',
+    'PHP_MONGODB_KMS_MESSAGE_SOURCES' => 'src/libmongoc/src/kms-message/src',
+    'PHP_MONGODB_BSON_SOURCES' => 'src/libmongoc/src/libbson/src/bson',
+    'PHP_MONGODB_JSONSL_SOURCES' => 'src/libmongoc/src/libbson/src/jsonsl',
+    'PHP_MONGODB_MONGOC_SOURCES' => 'src/libmongoc/src/libmongoc/src/mongoc',
+    'PHP_MONGODB_ZLIB_SOURCES' => 'src/libmongoc/src/zlib-1.2.11',
+    'PHP_MONGODB_MONGOCRYPT_SOURCES' => 'src/libmongocrypt/src',
+    'PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES' => 'src/libmongocrypt/src/crypto',
+    'PHP_MONGODB_MONGOCRYPT_OS_POSIX_SOURCES' => 'src/libmongocrypt/src/os_posix',
+    'PHP_MONGODB_MONGOCRYPT_OS_WIN_SOURCES' => 'src/libmongocrypt/src/os_win',
+    'PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES' => 'src/libmongocrypt/kms-message/src',
+];
+
+$patterns = [];
+$replacements = [];
+
+foreach ($vars as $var => $path) {
+    $cutNth = 2 + substr_count($path, '/');
+
+    $files = trim(shell_exec(sprintf($cmd, $path, $cutNth)));
+
+    $patterns[] = sprintf('/(%s=")([^"]*)(";?)/', $var);
+    $replacements[] = '$1' . $files . '$3';
+}
+
+$files = [
+    realpath(__DIR__ . '/../config.m4') => count($patterns),
+    // config.w32 does not use PHP_MONGODB_ZLIB_SOURCES (PHPC-1111)
+    realpath(__DIR__ . '/../config.w32') => count($patterns) - 1,
+];
+
+foreach ($files as $file => $expectedCount) {
+    $replaced = preg_replace($patterns, $replacements, file_get_contents($file), 1, $count);
+
+    if ($count !== $expectedCount) {
+        fprintf(STDERR, "Skipping %s: Expected %d replacements but only matched %d\n", basename($file), $expectedCount, $count);
+        continue;
+    }
+
+    printf("Updated %s\n", basename($file));
+    file_put_contents($file, $replaced);
+}

src/LIBMONGOCRYPT_VERSION_CURRENT

@@ -1 +1 @@
-1.2.1
+1.3.0

src/LIBMONGOC_VERSION_CURRENT

@@ -1 +1 @@
-1.11.1-20211103+gitb93e345178
+1.20.0