Fix mingw-w64 GCC toolchain compatibility issues (#1054)

* Address -Wincompatible-pointer-types warnings
* Define __USE_MINGW_ANSI_STDIO=1
* Fix format string compatibility issues

Author: eramongodb

CMakeLists.txt

@@ -19,6 +19,12 @@ if (ENABLE_WINDOWS_STATIC_RUNTIME)
    set (CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 endif ()
 
+if (MINGW)
+   # MSVCRT-based mingw-w64 GCC requires this macro to enable C99 format specifiers. This does not apply to UCRT-based
+   # mingw-w64 GCC, but apply the workaround anyways until bson-compat.h no longer requires this.
+   add_compile_definitions ("__USE_MINGW_ANSI_STDIO=1")
+endif ()
+
 list (APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
 include (GNUInstallDirs)

kms-message/src/kms_message_private.h

@@ -124,8 +124,10 @@ struct _kms_response_parser_t {
       }                      \
    } while (0)
 
-#ifdef __GNUC__
-__attribute__((format(__printf__, 3, 4)))
+#if defined(__clang__)
+__attribute__((format(printf, 3, 4)))
+#elif defined(__GNUC__)
+__attribute__((format(gnu_printf, 3, 4)))
 #endif
 void
 kms_set_error (char *error, size_t size, const char *fmt, ...);

kms-message/src/kms_request_str.h

@@ -65,14 +65,18 @@ KMS_MSG_EXPORT (void)
 kms_request_str_append_lowercase (kms_request_str_t *str,
                                   kms_request_str_t *appended);
 
-#ifdef __GNUC__
-__attribute__((format(__printf__, 2, 3)))
+#if defined(__clang__)
+__attribute__((format(printf, 2, 3)))
+#elif defined(__GNUC__)
+__attribute__((format(gnu_printf, 2, 3)))
 #endif
 KMS_MSG_EXPORT (void)
 kms_request_str_appendf (kms_request_str_t *str, const char *format, ...);
 
-#ifdef __GNUC__
-__attribute__((format(__printf__, 2, 3)))
+#if defined(__clang__)
+__attribute__((format(printf, 2, 3)))
+#elif defined(__GNUC__)
+__attribute__((format(gnu_printf, 2, 3)))
 #endif
 KMS_MSG_EXPORT (void)
 kms_request_strdupf (kms_request_str_t *str, const char *format, ...);

src/crypto/cng.c

@@ -221,6 +221,7 @@ bool _native_crypto_aes_256_cbc_encrypt(aes_256_args_t args) {
 
     NTSTATUS nt_status;
 
+    ULONG bytes_written = args.bytes_written ? *args.bytes_written : 0u;
     nt_status = BCryptEncrypt(state->key_handle,
                               (PUCHAR)(args.in->data),
                               args.in->len,
@@ -229,8 +230,9 @@ bool _native_crypto_aes_256_cbc_encrypt(aes_256_args_t args) {
                               state->iv_len,
                               args.out->data,
                               args.out->len,
-                              args.bytes_written,
+                              &bytes_written,
                               0);
+    args.bytes_written ? (*args.bytes_written = bytes_written) : (void)0;
 
     if (nt_status != STATUS_SUCCESS) {
         CLIENT_ERR("error initializing cipher: 0x%x", (int)nt_status);
@@ -255,6 +257,7 @@ bool _native_crypto_aes_256_cbc_decrypt(aes_256_args_t args) {
 
     NTSTATUS nt_status;
 
+    ULONG bytes_written = args.bytes_written ? *args.bytes_written : 0u;
     nt_status = BCryptDecrypt(state->key_handle,
                               (PUCHAR)(args.in->data),
                               args.in->len,
@@ -263,8 +266,9 @@ bool _native_crypto_aes_256_cbc_decrypt(aes_256_args_t args) {
                               state->iv_len,
                               args.out->data,
                               args.out->len,
-                              args.bytes_written,
+                              &bytes_written,
                               0);
+    args.bytes_written ? (*args.bytes_written = bytes_written) : (void)0;
 
     if (nt_status != STATUS_SUCCESS) {
         CLIENT_ERR("error initializing cipher: 0x%x", (int)nt_status);
@@ -360,7 +364,7 @@ typedef struct {
 static bool _cng_ctr_crypto_generate(cng_ctr_encrypt_state *state, mongocrypt_status_t *status) {
     BSON_ASSERT(state);
 
-    uint32_t bytesEncrypted = 0;
+    ULONG bytesEncrypted = 0;
     NTSTATUS nt_status = BCryptEncrypt(state->key_handle,
                                        state->input_block,
                                        state->input_block_len,