[CDRIVER-4276] On-demand Credentials Callback for CSFLE (#1051)

This implements a credentials callback for client-side encryption.
This relates to CDRIVER-4276, although that ticket also concerns
itself with server auth on AWS, which is not implemented as part
of this change.

* Initial simple credentials callback impl
* Allow the user-provided callback to set an error
* Tests for KMS on-demand callback
* Only run callback tests if we have CSE enabled
* Add docs for new callback APIs

Author: vector-of-bool

src/libmongoc/doc/mongoc_auto_encryption_opts_set_kms_credential_provider_callback.rst

@@ -0,0 +1,32 @@
+:man_page: mongoc_auto_encryption_opts_set_kms_credential_provider_callback
+
+mongoc_auto_encryption_opts_set_kms_credential_provider_callback()
+==================================================================
+
+.. versionadded:: 1.23.0
+
+Synopsis
+--------
+
+.. code-block:: c
+
+  void
+  mongoc_auto_encryption_opts_set_kms_credential_provider_callback(
+    mongoc_auto_encryption_opts_t *opts,
+    mongoc_kms_credentials_provider_callback_fn fn,
+    void *userdata);
+
+Set the user-provided callback to provide KMS credentials on-demand when they
+are needed.
+
+Parameters
+----------
+
+- ``opts`` - The options object to update.
+- ``fn`` - The provider callback to set on the options object. May be ``NULL``
+  to clear the callback. Refer to:
+  :c:type:`mongoc_kms_credentials_provider_callback_fn`
+- ``userdata`` - An arbitrary pointer that will be passed along to the
+  callback function when it is called by libmongoc.
+
+.. seealso:: :doc:`mongoc_client_encryption_opts_set_kms_credential_provider_callback`

src/libmongoc/doc/mongoc_auto_encryption_opts_t.rst

@@ -31,6 +31,7 @@ Synopsis
     mongoc_auto_encryption_opts_set_keyvault_client_pool
     mongoc_auto_encryption_opts_set_keyvault_namespace
     mongoc_auto_encryption_opts_set_kms_providers
+    mongoc_auto_encryption_opts_set_kms_credential_provider_callback
     mongoc_auto_encryption_opts_set_schema_map
     mongoc_auto_encryption_opts_set_bypass_auto_encryption
     mongoc_auto_encryption_opts_set_extra

src/libmongoc/doc/mongoc_client_encryption_opts_set_kms_credential_provider_callback.rst

@@ -0,0 +1,60 @@
+:man_page: mongoc_client_encryption_opts_set_kms_credential_provider_callback
+
+mongoc_client_encryption_opts_set_kms_credential_provider_callback ()
+=====================================================================
+
+.. versionadded:: 1.23.0
+
+Synopsis
+--------
+
+.. code-block:: c
+
+  void
+  mongoc_client_encryption_opts_set_kms_credential_provider_callback (
+    mongoc_client_encryption_opts_t *opts,
+    mongoc_kms_credentials_provider_callback_fn fn,
+    void *userdata);
+
+Set the user-provided callback to provide KMS credentials on-demand when they
+are needed.
+
+Parameters
+----------
+
+- ``opts`` - The options object to update.
+- ``fn`` - The provider callback to set on the options object. May be ``NULL``
+  to clear the callback. Refer to:
+  :c:type:`mongoc_kms_credentials_provider_callback_fn`
+- ``userdata`` - An arbitrary pointer that will be passed along to the
+  callback function when it is called by libmongoc.
+
+.. seealso:: :doc:`mongoc_auto_encryption_opts_set_kms_credential_provider_callback`
+
+.. rubric:: Related:
+
+.. c:type:: mongoc_kms_credentials_provider_callback_fn
+
+  .. code-block:: c
+
+    typedef
+    bool (*mongoc_kms_credentials_provider_callback_fn) (void *userdata,
+                                                         const bson_t *params,
+                                                         bson_t *out,
+                                                         bson_error_t *error);
+
+  The type of a callback function for providing KMS providers data on-demand.
+
+  :parameters:
+
+    - ``userdata`` - The same userdata pointer provided to the ``userdata``
+      parameter when the callback was set.
+    - ``params`` - Parameters for the requested KMS credentials. Currently
+      empty.
+    - ``out`` - The output :symbol:`bson:bson_t` in which to write the new
+      KMS providers. When passed to the callback, this already points to an
+      empty BSON document which must be populated.
+    - ``error`` - An output parameter for indicating any errors that might
+      occur while generating the KMS credentials.
+
+  :return value: Must return ``true`` on success, ``false`` on failure.

src/libmongoc/doc/mongoc_client_encryption_opts_t.rst

@@ -25,6 +25,7 @@ Used to set options for :symbol:`mongoc_client_encryption_new()`.
     mongoc_client_encryption_opts_destroy
     mongoc_client_encryption_opts_set_keyvault_client
     mongoc_client_encryption_opts_set_keyvault_namespace
+    mongoc_client_encryption_opts_set_kms_credential_provider_callback
     mongoc_client_encryption_opts_set_kms_providers
     mongoc_client_encryption_opts_set_tls_opts
 

src/libmongoc/src/mongoc/mongoc-client-side-encryption.c

@@ -45,9 +45,20 @@ struct _mongoc_auto_encryption_opts_t {
    bson_t *encrypted_fields_map;
    bool bypass_auto_encryption;
    bool bypass_query_analysis;
+   mc_kms_credentials_callback creds_cb;
    bson_t *extra;
 };
 
+static void
+_set_creds_callback (mc_kms_credentials_callback *cb,
+                     mongoc_kms_credentials_provider_callback_fn fn,
+                     void *userdata)
+{
+   BSON_ASSERT (cb);
+   cb->fn = fn;
+   cb->userdata = userdata;
+}
+
 mongoc_auto_encryption_opts_t *
 mongoc_auto_encryption_opts_new (void)
 {
@@ -206,6 +217,15 @@ mongoc_auto_encryption_opts_set_extra (mongoc_auto_encryption_opts_t *opts,
    }
 }
 
+void
+mongoc_auto_encryption_opts_set_kms_credential_provider_callback (
+   mongoc_auto_encryption_opts_t *opts,
+   mongoc_kms_credentials_provider_callback_fn fn,
+   void *userdata)
+{
+   _set_creds_callback (&opts->creds_cb, fn, userdata);
+}
+
 /*--------------------------------------------------------------------------
  * Client Encryption options.
  *--------------------------------------------------------------------------
@@ -216,6 +236,7 @@ struct _mongoc_client_encryption_opts_t {
    char *keyvault_coll;
    bson_t *kms_providers;
    bson_t *tls_opts;
+   mc_kms_credentials_callback creds_cb;
 };
 
 mongoc_client_encryption_opts_t *
@@ -230,6 +251,7 @@ mongoc_client_encryption_opts_destroy (mongoc_client_encryption_opts_t *opts)
    if (!opts) {
       return;
    }
+   _set_creds_callback (&opts->creds_cb, NULL, NULL);
    bson_free (opts->keyvault_db);
    bson_free (opts->keyvault_coll);
    bson_destroy (opts->kms_providers);
@@ -287,6 +309,17 @@ mongoc_client_encryption_opts_set_tls_opts (
    opts->tls_opts = _bson_copy_or_null (tls_opts);
 }
 
+void
+mongoc_client_encryption_opts_set_kms_credential_provider_callback (
+   mongoc_client_encryption_opts_t *opts,
+   mongoc_kms_credentials_provider_callback_fn fn,
+   void *userdata)
+{
+   BSON_ASSERT_PARAM (opts);
+   opts->creds_cb.fn = fn;
+   opts->creds_cb.userdata = userdata;
+}
+
 /*--------------------------------------------------------------------------
  * Data key options.
  *--------------------------------------------------------------------------
@@ -1551,6 +1584,7 @@ _mongoc_cse_client_enable_auto_encryption (mongoc_client_t *client,
                             .extraOptions.cryptSharedLibRequired,
                          opts->bypass_auto_encryption,
                          opts->bypass_query_analysis,
+                         opts->creds_cb,
                          error);
    if (!client->topology->crypt) {
       GOTO (fail);
@@ -1712,6 +1746,7 @@ _mongoc_cse_client_pool_enable_auto_encryption (
                             .cryptSharedLibRequired,
                          opts->bypass_auto_encryption,
                          opts->bypass_query_analysis,
+                         opts->creds_cb,
                          error);
    if (!topology->crypt) {
       GOTO (fail);
@@ -1817,11 +1852,14 @@ mongoc_client_encryption_new (mongoc_client_encryption_opts_t *opts,
                          NULL /* No crypt_shared path */,
                          false /* crypt_shared not requried */,
                          true, /* bypassAutoEncryption (We are explicit) */
-                         false /* bypass_query_analysis. Not applicable. */,
+                         false,
+                         /* bypass_query_analysis. Not applicable. */
+                         opts->creds_cb,
                          error);
    if (!client_encryption->crypt) {
       goto fail;
    }
+
    success = true;
 
 fail:

src/libmongoc/src/mongoc/mongoc-client-side-encryption.h

@@ -40,6 +40,9 @@ BSON_BEGIN_DECLS
 
 typedef struct _mongoc_auto_encryption_opts_t mongoc_auto_encryption_opts_t;
 
+typedef bool (*mongoc_kms_credentials_provider_callback_fn) (
+   void *userdata, const bson_t *params, bson_t *out, bson_error_t *error);
+
 MONGOC_EXPORT (mongoc_auto_encryption_opts_t *)
 mongoc_auto_encryption_opts_new (void) BSON_GNUC_WARN_UNUSED_RESULT;
 
@@ -86,6 +89,12 @@ MONGOC_EXPORT (void)
 mongoc_auto_encryption_opts_set_extra (mongoc_auto_encryption_opts_t *opts,
                                        const bson_t *extra);
 
+MONGOC_EXPORT (void)
+mongoc_auto_encryption_opts_set_kms_credential_provider_callback (
+   mongoc_auto_encryption_opts_t *opts,
+   mongoc_kms_credentials_provider_callback_fn fn,
+   void *userdata);
+
 typedef struct _mongoc_client_encryption_opts_t mongoc_client_encryption_opts_t;
 typedef struct _mongoc_client_encryption_t mongoc_client_encryption_t;
 typedef struct _mongoc_client_encryption_encrypt_opts_t
@@ -118,6 +127,12 @@ MONGOC_EXPORT (void)
 mongoc_client_encryption_opts_set_tls_opts (
    mongoc_client_encryption_opts_t *opts, const bson_t *tls_opts);
 
+MONGOC_EXPORT (void)
+mongoc_client_encryption_opts_set_kms_credential_provider_callback (
+   mongoc_client_encryption_opts_t *opts,
+   mongoc_kms_credentials_provider_callback_fn fn,
+   void *userdata);
+
 MONGOC_EXPORT (mongoc_client_encryption_rewrap_many_datakey_result_t *)
 mongoc_client_encryption_rewrap_many_datakey_result_new (void)
    BSON_GNUC_WARN_UNUSED_RESULT;

src/libmongoc/src/mongoc/mongoc-crypt-private.h

@@ -21,10 +21,15 @@
 
 #include "mongoc-config.h"
 
-#ifdef MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION
-
 #include "mongoc.h"
 
+typedef struct mc_kms_credentials_callback {
+   mongoc_kms_credentials_provider_callback_fn fn;
+   void *userdata;
+} mc_kms_credentials_callback;
+
+#ifdef MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION
+
 /* For interacting with libmongocrypt */
 typedef struct __mongoc_crypt_t _mongoc_crypt_t;
 
@@ -42,6 +47,7 @@ _mongoc_crypt_new (const bson_t *kms_providers,
                    bool crypt_shared_lib_required,
                    bool bypass_auto_encryption,
                    bool bypass_query_analysis,
+                   mc_kms_credentials_callback creds_cb,
                    bson_error_t *error);
 
 void

src/libmongoc/src/mongoc/mongoc-crypt.c

@@ -35,6 +35,7 @@ struct __mongoc_crypt_t {
    mongoc_ssl_opt_t aws_tls_opt;
    mongoc_ssl_opt_t azure_tls_opt;
    mongoc_ssl_opt_t gcp_tls_opt;
+   mc_kms_credentials_callback creds_cb;
 };
 
 static void
@@ -575,6 +576,37 @@ _state_need_kms (_state_machine_t *state_machine, bson_error_t *error)
 #undef BUFFER_SIZE
 }
 
+static bool
+_state_need_kms_credentials (_state_machine_t *sm, bson_error_t *error)
+{
+   bson_t creds = BSON_INITIALIZER;
+   BSON_ASSERT (sm->crypt->creds_cb.fn);
+   const bson_t empty = BSON_INITIALIZER;
+
+   if (!sm->crypt->creds_cb.fn (
+          sm->crypt->creds_cb.userdata, &empty, &creds, error)) {
+      // The callback reports that it has failed
+      if (!error->code) {
+         bson_set_error (error,
+                         MONGOC_ERROR_CLIENT_SIDE_ENCRYPTION,
+                         MONGOC_ERROR_CLIENT_INVALID_ENCRYPTION_ARG,
+                         "Unknown error from user-provided callback for "
+                         "on-demand KMS credentials");
+      }
+      return false;
+   }
+
+   mongocrypt_binary_t *data = mongocrypt_binary_new_from_data (
+      (uint8_t *) bson_get_data (&creds), creds.len);
+   bool okay = mongocrypt_ctx_provide_kms_providers (sm->ctx, data);
+   if (!okay) {
+      _ctx_check_error (sm->ctx, error, true);
+   }
+   mongocrypt_binary_destroy (data);
+   bson_destroy (&creds);
+   return okay;
+}
+
 static bool
 _state_ready (_state_machine_t *state_machine,
               bson_t *result,
@@ -651,6 +683,11 @@ _state_machine_run (_state_machine_t *state_machine,
             goto fail;
          }
          break;
+      case MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS:
+         if (!_state_need_kms_credentials (state_machine, error)) {
+            goto fail;
+         }
+         break;
       case MONGOCRYPT_CTX_READY:
          bson_destroy (result);
          if (!_state_ready (state_machine, result, error)) {
@@ -914,6 +951,7 @@ _mongoc_crypt_new (const bson_t *kms_providers,
                    bool crypt_shared_lib_required,
                    bool bypass_auto_encryption,
                    bool bypass_query_analysis,
+                   mc_kms_credentials_callback creds_cb,
                    bson_error_t *error)
 {
    _mongoc_crypt_t *crypt;
@@ -941,6 +979,12 @@ _mongoc_crypt_new (const bson_t *kms_providers,
       goto fail;
    }
 
+   if (creds_cb.fn) {
+      // The user has provided a callback to lazily obtain KMS credentials. We
+      // need to opt-in to the libmongocrypt feature.
+      mongocrypt_setopt_use_need_kms_credentials_state (crypt->handle);
+   }
+
    if (schema_map) {
       schema_map_bin = mongocrypt_binary_new_from_data (
          (uint8_t *) bson_get_data (schema_map), schema_map->len);
@@ -1010,6 +1054,8 @@ _mongoc_crypt_new (const bson_t *kms_providers,
                   s);
    }
 
+   crypt->creds_cb = creds_cb;
+
    success = true;
 fail:
    mongocrypt_binary_destroy (local_masterkey_bin);