CDRIVER-4000 Test redaction of sensitive commands (#814)

* Sync command monitoring spec tests

This brings the spec tests up to date with mongodb/specifications#84ac002b

* Add support for 1.5 unified test format schema

This adds support for the "observeSensitiveCommands" monitoring option, as well as the new "auth" runOnRequirement.

* Redact sensitive commands for APM

* Test redaction of replies for sensitive commands

* Force redaction of replies for sensitive commands

Previously, the driver would not redact the reply to a hello command with speculative authentication unless the reply also was sensitive. As this makes the test completely useless, we've decided to always require redaction of replies when the command was redacted.

* Document is_redacted and force_redaction arguments to APM initialisers

* Make redaction helpers static

Author: alcaeus

src/libmongoc/src/mongoc/mongoc-apm-private.h

@@ -60,7 +60,8 @@ struct _mongoc_apm_command_started_t {
 
 struct _mongoc_apm_command_succeeded_t {
    int64_t duration;
-   const bson_t *reply;
+   bson_t *reply;
+   bool reply_owned;
    const char *command_name;
    int64_t request_id;
    int64_t operation_id;
@@ -73,7 +74,8 @@ struct _mongoc_apm_command_failed_t {
    int64_t duration;
    const char *command_name;
    const bson_error_t *error;
-   const bson_t *reply;
+   bson_t *reply;
+   bool reply_owned;
    int64_t request_id;
    int64_t operation_id;
    const mongoc_host_list_t *host;
@@ -153,12 +155,14 @@ mongoc_apm_command_started_init (mongoc_apm_command_started_t *event,
                                  int64_t operation_id,
                                  const mongoc_host_list_t *host,
                                  uint32_t server_id,
+                                 bool *is_redacted, /* out */
                                  void *context);
 
 void
 mongoc_apm_command_started_init_with_cmd (mongoc_apm_command_started_t *event,
                                           struct _mongoc_cmd_t *cmd,
                                           int64_t request_id,
+                                          bool *is_redacted, /* out */
                                           void *context);
 
 void
@@ -173,6 +177,7 @@ mongoc_apm_command_succeeded_init (mongoc_apm_command_succeeded_t *event,
                                    int64_t operation_id,
                                    const mongoc_host_list_t *host,
                                    uint32_t server_id,
+                                   bool force_redaction,
                                    void *context);
 
 void
@@ -188,11 +193,19 @@ mongoc_apm_command_failed_init (mongoc_apm_command_failed_t *event,
                                 int64_t operation_id,
                                 const mongoc_host_list_t *host,
                                 uint32_t server_id,
+                                bool force_redaction,
                                 void *context);
 
 void
 mongoc_apm_command_failed_cleanup (mongoc_apm_command_failed_t *event);
 
+bool
+mongoc_apm_is_sensitive_command (const char *command_name,
+                                 const bson_t *command);
+
+bool
+mongoc_apm_is_sensitive_reply (const char *command_name, const bson_t *reply);
+
 BSON_END_DECLS
 
 #endif /* MONGOC_APM_PRIVATE_H */

src/libmongoc/src/mongoc/mongoc-apm.c

@@ -17,6 +17,7 @@
 #include "mongoc-util-private.h"
 #include "mongoc-apm-private.h"
 #include "mongoc-cmd-private.h"
+#include "mongoc-handshake-private.h"
 
 /*
  * An Application Performance Management (APM) implementation, complying with
@@ -46,6 +47,24 @@ append_documents_from_cmd (const mongoc_cmd_t *cmd,
  * Private initializer / cleanup functions.
  */
 
+static void
+mongoc_apm_redact_command (bson_t *command);
+
+static void
+mongoc_apm_redact_reply (bson_t *reply);
+
+/*--------------------------------------------------------------------------
+ *
+ * mongoc_apm_command_started_init --
+ *
+ *       Initialises the command started event.
+ *
+ * Side effects:
+ *       If provided, is_redacted indicates whether the command document was
+ *       redacted to hide sensitive information.
+ *
+ *--------------------------------------------------------------------------
+ */
 void
 mongoc_apm_command_started_init (mongoc_apm_command_started_t *event,
                                  const bson_t *command,
@@ -55,6 +74,7 @@ mongoc_apm_command_started_init (mongoc_apm_command_started_t *event,
                                  int64_t operation_id,
                                  const mongoc_host_list_t *host,
                                  uint32_t server_id,
+                                 bool *is_redacted, /* out */
                                  void *context)
 {
    bson_iter_t iter;
@@ -87,6 +107,21 @@ mongoc_apm_command_started_init (mongoc_apm_command_started_t *event,
       event->command_owned = false;
    }
 
+   if (mongoc_apm_is_sensitive_command (command_name, command)) {
+      if (!event->command_owned) {
+         event->command = bson_copy (event->command);
+         event->command_owned = true;
+      }
+
+      if (is_redacted) {
+         *is_redacted = true;
+      }
+
+      mongoc_apm_redact_command (event->command);
+   } else if (is_redacted) {
+      *is_redacted = false;
+   }
+
    event->database_name = database_name;
    event->command_name = command_name;
    event->request_id = request_id;
@@ -97,10 +132,23 @@ mongoc_apm_command_started_init (mongoc_apm_command_started_t *event,
 }
 
 
+/*--------------------------------------------------------------------------
+ *
+ * mongoc_apm_command_started_init_with_cmd --
+ *
+ *       Initialises the command started event from a mongoc_cmd_t.
+ *
+ * Side effects:
+ *       If provided, is_redacted indicates whether the command document was
+ *       redacted to hide sensitive information.
+ *
+ *--------------------------------------------------------------------------
+ */
 void
 mongoc_apm_command_started_init_with_cmd (mongoc_apm_command_started_t *event,
                                           mongoc_cmd_t *cmd,
                                           int64_t request_id,
+                                          bool *is_redacted, /* out */
                                           void *context)
 {
    mongoc_apm_command_started_init (event,
@@ -111,6 +159,7 @@ mongoc_apm_command_started_init_with_cmd (mongoc_apm_command_started_t *event,
                                     cmd->operation_id,
                                     &cmd->server_stream->sd->host,
                                     cmd->server_stream->sd->id,
+                                    is_redacted,
                                     context);
 
    /* OP_MSG document sequence for insert, update, or delete? */
@@ -127,6 +176,18 @@ mongoc_apm_command_started_cleanup (mongoc_apm_command_started_t *event)
 }
 
 
+/*--------------------------------------------------------------------------
+ *
+ * mongoc_apm_command_succeeded_init --
+ *
+ *       Initialises the command succeeded event.
+ *
+ * Parameters:
+ *       @force_redaction: If true, the reply document is always redacted,
+ *       regardless of whether the command contains sensitive information.
+ *
+ *--------------------------------------------------------------------------
+ */
 void
 mongoc_apm_command_succeeded_init (mongoc_apm_command_succeeded_t *event,
                                    int64_t duration,
@@ -136,12 +197,23 @@ mongoc_apm_command_succeeded_init (mongoc_apm_command_succeeded_t *event,
                                    int64_t operation_id,
                                    const mongoc_host_list_t *host,
                                    uint32_t server_id,
+                                   bool force_redaction,
                                    void *context)
 {
    BSON_ASSERT (reply);
 
+   if (force_redaction || mongoc_apm_is_sensitive_reply (command_name, reply)) {
+      event->reply = bson_copy (reply);
+      event->reply_owned = true;
+
+      mongoc_apm_redact_reply (event->reply);
+   } else {
+      /* discard "const", we promise not to modify "reply" */
+      event->reply = (bson_t *) reply;
+      event->reply_owned = false;
+   }
+
    event->duration = duration;
-   event->reply = reply;
    event->command_name = command_name;
    event->request_id = request_id;
    event->operation_id = operation_id;
@@ -154,10 +226,24 @@ mongoc_apm_command_succeeded_init (mongoc_apm_command_succeeded_t *event,
 void
 mongoc_apm_command_succeeded_cleanup (mongoc_apm_command_succeeded_t *event)
 {
-   /* no-op */
+   if (event->reply_owned) {
+      bson_destroy (event->reply);
+   }
 }
 
 
+/*--------------------------------------------------------------------------
+ *
+ * mongoc_apm_command_failed_init --
+ *
+ *       Initialises the command failed event.
+ *
+ * Parameters:
+ *       @force_redaction: If true, the reply document is always redacted,
+ *       regardless of whether the command contains sensitive information.
+ *
+ *--------------------------------------------------------------------------
+ */
 void
 mongoc_apm_command_failed_init (mongoc_apm_command_failed_t *event,
                                 int64_t duration,
@@ -168,14 +254,25 @@ mongoc_apm_command_failed_init (mongoc_apm_command_failed_t *event,
                                 int64_t operation_id,
                                 const mongoc_host_list_t *host,
                                 uint32_t server_id,
+                                bool force_redaction,
                                 void *context)
 {
    BSON_ASSERT (reply);
 
+   if (force_redaction || mongoc_apm_is_sensitive_reply (command_name, reply)) {
+      event->reply = bson_copy (reply);
+      event->reply_owned = true;
+
+      mongoc_apm_redact_reply (event->reply);
+   } else {
+      /* discard "const", we promise not to modify "reply" */
+      event->reply = (bson_t *) reply;
+      event->reply_owned = false;
+   }
+
    event->duration = duration;
    event->command_name = command_name;
    event->error = error;
-   event->reply = reply;
    event->request_id = request_id;
    event->operation_id = operation_id;
    event->host = host;
@@ -187,7 +284,9 @@ mongoc_apm_command_failed_init (mongoc_apm_command_failed_t *event,
 void
 mongoc_apm_command_failed_cleanup (mongoc_apm_command_failed_t *event)
 {
-   /* no-op */
+   if (event->reply_owned) {
+      bson_destroy (event->reply);
+   }
 }
 
 
@@ -775,3 +874,70 @@ mongoc_apm_set_server_heartbeat_failed_cb (
 {
    callbacks->server_heartbeat_failed = cb;
 }
+
+static bool
+_mongoc_apm_is_sensitive_command_name (const char *command_name)
+{
+   return 0 == strcasecmp (command_name, "authenticate") ||
+          0 == strcasecmp (command_name, "saslStart") ||
+          0 == strcasecmp (command_name, "saslContinue") ||
+          0 == strcasecmp (command_name, "getnonce") ||
+          0 == strcasecmp (command_name, "createUser") ||
+          0 == strcasecmp (command_name, "updateUser") ||
+          0 == strcasecmp (command_name, "copydbgetnonce") ||
+          0 == strcasecmp (command_name, "copydbsaslstart") ||
+          0 == strcasecmp (command_name, "copydb");
+}
+
+bool
+mongoc_apm_is_sensitive_command (const char *command_name,
+                                 const bson_t *command)
+{
+   BSON_ASSERT (command);
+
+   if (_mongoc_apm_is_sensitive_command_name (command_name)) {
+      return true;
+   }
+
+   if (0 != strcasecmp (command_name, "hello") &&
+       0 != strcasecmp (command_name, HANDSHAKE_CMD_LEGACY_HELLO)) {
+      return false;
+   }
+
+   return bson_has_field (command, "speculativeAuthenticate");
+}
+
+void
+mongoc_apm_redact_command (bson_t *command)
+{
+   BSON_ASSERT (command);
+
+   /* Reinit the command to have an empty document */
+   bson_reinit (command);
+}
+
+bool
+mongoc_apm_is_sensitive_reply (const char *command_name, const bson_t *reply)
+{
+   BSON_ASSERT (reply);
+
+   if (_mongoc_apm_is_sensitive_command_name (command_name)) {
+      return true;
+   }
+
+   if (0 != strcasecmp (command_name, "hello") &&
+       0 != strcasecmp (command_name, HANDSHAKE_CMD_LEGACY_HELLO)) {
+      return false;
+   }
+
+   return bson_has_field (reply, "speculativeAuthenticate");
+}
+
+void
+mongoc_apm_redact_reply (bson_t *reply)
+{
+   BSON_ASSERT (reply);
+
+   /* Reinit the reply to have an empty document */
+   bson_reinit (reply);
+}

src/libmongoc/src/mongoc/mongoc-client.c

@@ -2360,6 +2360,7 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster,
                                     operation_id,
                                     &server_stream->sd->host,
                                     server_stream->sd->id,
+                                    NULL,
                                     client->apm_context);
 
    client->apm_callbacks.started (&event);
@@ -2406,6 +2407,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (
                                       operation_id,
                                       &server_stream->sd->host,
                                       server_stream->sd->id,
+                                      false,
                                       client->apm_context);
 
    client->apm_callbacks.succeeded (&event);
@@ -2448,6 +2450,7 @@ _mongoc_client_monitor_op_killcursors_failed (
                                    operation_id,
                                    &server_stream->sd->host,
                                    server_stream->sd->id,
+                                   false,
                                    client->apm_context);
 
    client->apm_callbacks.failed (&event);

src/libmongoc/src/mongoc/mongoc-cluster.c

@@ -508,6 +508,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster,
    bson_t encrypted = BSON_INITIALIZER;
    bson_t decrypted = BSON_INITIALIZER;
    mongoc_cmd_t encrypted_cmd;
+   bool is_redacted = false;
 
    server_stream = cmd->server_stream;
    server_id = server_stream->sd->id;
@@ -534,8 +535,11 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster,
    }
 
    if (callbacks->started) {
-      mongoc_apm_command_started_init_with_cmd (
-         &started_event, cmd, request_id, cluster->client->apm_context);
+      mongoc_apm_command_started_init_with_cmd (&started_event,
+                                                cmd,
+                                                request_id,
+                                                &is_redacted,
+                                                cluster->client->apm_context);
 
       callbacks->started (&started_event);
       mongoc_apm_command_started_cleanup (&started_event);
@@ -579,6 +583,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster,
                                          cmd->operation_id,
                                          &server_stream->sd->host,
                                          server_id,
+                                         is_redacted,
                                          cluster->client->apm_context);
 
       callbacks->succeeded (&succeeded_event);
@@ -595,6 +600,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster,
                                       cmd->operation_id,
                                       &server_stream->sd->host,
                                       server_id,
+                                      is_redacted,
                                       cluster->client->apm_context);
 
       callbacks->failed (&failed_event);