bunch o extra changes?

Author: baileympearson

.github/workflows/encryption-tests.yml

@@ -0,0 +1,37 @@
+name: Encryption Tests 
+
+on:
+  push: 
+    branches: ['master']
+  pull_request:
+    branches: [ 'master' ]
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+  id-token: write
+
+jobs:
+  run-tests:
+    permissions:
+      # required for all workflows
+      security-events: write
+      id-token: write
+      contents: write
+    runs-on: ubuntu-latest
+    name: Encryption tests
+    env:
+      FORCE_COLOR: true
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Setup node
+        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        with:
+          node-version: latest
+      - name: Install Dependencies
+        run: npm install
+      - name: Install mongodb-client-encryption
+        run: npm install mongodb-client-encryption
+      - name: Run Tests
+        run: npm run test-encryption

.gitignore

@@ -67,3 +67,6 @@ examples/ecommerce-netlify-functions/.netlify/state.json
 
 notes.md
 list.out
+
+data
+*.pid

CONTRIBUTING.md

@@ -46,6 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe
   * execute `npm run test-tsd` to run the typescript tests
   * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time.
   * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command.
+* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file.
 
 ## Documentation
 

lib/collection.js

@@ -81,7 +81,7 @@ Collection.prototype.onOpen = function() {
  * @api private
  */
 
-Collection.prototype.onClose = function() {};
+Collection.prototype.onClose = function() { };
 
 /**
  * Queues a method for later execution when its

lib/connection.js

@@ -607,7 +607,7 @@ Connection.prototype.bulkWrite = async function bulkWrite(ops, options) {
 
 Connection.prototype.createCollections = async function createCollections(options = {}) {
   const result = {};
-  const errorsMap = { };
+  const errorsMap = {};
 
   const { continueOnError } = options;
   delete options.continueOnError;
@@ -734,7 +734,7 @@ Connection.prototype.transaction = function transaction(fn, options) {
         throw err;
       }).
       finally(() => {
-        session.endSession().catch(() => {});
+        session.endSession().catch(() => { });
       });
   });
 };
@@ -1025,7 +1025,7 @@ Connection.prototype.openUri = async function openUri(uri, options) {
 
   for (const model of Object.values(this.models)) {
     // Errors handled internally, so safe to ignore error
-    model.init().catch(function $modelInitNoop() {});
+    model.init().catch(function $modelInitNoop() { });
   }
 
   // `createConnection()` calls this `openUri()` function without
@@ -1061,7 +1061,7 @@ Connection.prototype.openUri = async function openUri(uri, options) {
 // to avoid uncaught exceptions when using `on('error')`. See gh-14377.
 Connection.prototype.on = function on(event, callback) {
   if (event === 'error' && this.$initialConnection) {
-    this.$initialConnection.catch(() => {});
+    this.$initialConnection.catch(() => { });
   }
   return EventEmitter.prototype.on.call(this, event, callback);
 };
@@ -1083,7 +1083,7 @@ Connection.prototype.on = function on(event, callback) {
 // to avoid uncaught exceptions when using `on('error')`. See gh-14377.
 Connection.prototype.once = function on(event, callback) {
   if (event === 'error' && this.$initialConnection) {
-    this.$initialConnection.catch(() => {});
+    this.$initialConnection.catch(() => { });
   }
   return EventEmitter.prototype.once.call(this, event, callback);
 };
@@ -1412,7 +1412,7 @@ Connection.prototype.model = function model(name, schema, collection, options) {
     }
 
     // Errors handled internally, so safe to ignore error
-    model.init().catch(function $modelInitNoop() {});
+    model.init().catch(function $modelInitNoop() { });
 
     return model;
   }
@@ -1439,7 +1439,7 @@ Connection.prototype.model = function model(name, schema, collection, options) {
   }
 
   if (this === model.prototype.db
-      && (!collection || collection === model.collection.name)) {
+    && (!collection || collection === model.collection.name)) {
     // model already uses this connection.
 
     // only the first model with this name is cached to allow
@@ -1626,8 +1626,8 @@ Connection.prototype.authMechanismDoesNotRequirePassword = function authMechanis
  */
 Connection.prototype.optionsProvideAuthenticationData = function optionsProvideAuthenticationData(options) {
   return (options) &&
-      (options.user) &&
-      ((options.pass) || this.authMechanismDoesNotRequirePassword());
+    (options.user) &&
+    ((options.pass) || this.authMechanismDoesNotRequirePassword());
 };
 
 /**
@@ -1689,7 +1689,7 @@ Connection.prototype.createClient = function createClient() {
  */
 Connection.prototype.syncIndexes = async function syncIndexes(options = {}) {
   const result = {};
-  const errorsMap = { };
+  const errorsMap = {};
 
   const { continueOnError } = options;
   delete options.continueOnError;

lib/drivers/node-mongodb-native/connection.js

@@ -12,6 +12,7 @@ const pkg = require('../../../package.json');
 const processConnectionOptions = require('../../helpers/processConnectionOptions');
 const setTimeout = require('../../helpers/timers').setTimeout;
 const utils = require('../../utils');
+const { inferBSONType } = require('../../encryption_utils');
 
 /**
  * A [node-mongodb-native](https://github.com/mongodb/node-mongodb-native) connection implementation.
@@ -304,6 +305,17 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio
     };
   }
 
+
+  const { schemaMap, encryptedFieldsMap } = this._buildEncryptionSchemas(options);
+
+  if (Object.keys(schemaMap).length > 0) {
+    options.autoEncryption.schemaMap = schemaMap;
+  }
+
+  if (Object.keys(encryptedFieldsMap).length > 0) {
+    options.autoEncryption.encryptedFieldsMap = encryptedFieldsMap;
+  }
+
   this.readyState = STATES.connecting;
   this._connectionString = uri;
 
@@ -327,6 +339,103 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio
   return this;
 };
 
+/**
+ * Given a connection, which may or may not have encrypted models, build
+ * a schemaMap and/or an encryptedFieldsMap for the connection, combining all models
+ * into a single schemaMap and encryptedFields map.
+ *
+ * @returns a copy of the options object with a schemaMap and/or an encryptedFieldsMap added to the options' autoEncryption
+ * options.
+  */
+NativeConnection.prototype._buildEncryptionSchemas = function() {
+  const schemaMap = Object.values(this.models).filter(model => model.schema.encryptionType() === 'csfle').reduce(
+    schemaMapReducer.bind(this),
+    {}
+  );
+  const encryptedFieldsMap = Object.values(this.models).filter(model => model.schema.encryptionType() === 'qe').reduce(
+    encryptedFieldsMapReducer.bind(this),
+    {}
+  );
+
+  return {
+    schemaMap, encryptedFieldsMap
+  };
+
+  /**
+   * `schemaMap`s are JSON schemas, which use the following structure to represent objects:
+   *    { field: { bsonType: 'object', properties: { ... } } }
+   *
+   * for example, a schema that looks like this `{ a: { b: int32 } }` would be encoded as
+   * `{ a: { bsonType: 'object', properties: { b: < encryption configuration > } } }`
+   *
+   * This function takes an array of path segments, an output object (that gets mutated) and
+   * a value to associated with the full path, and constructs a valid CSFLE JSON schema path for
+   * the object.  This works for deeply nested properties as well.
+   *
+   * @param {string[]} path array of path components
+   * @param {object} object the object in which to build a JSON schema of `path`'s properties
+   * @param {object} value the value to associate with the path in object
+   */
+  function buildNestedPath(path, object, value) {
+    let i = 0, component = path[i];
+    for (; i < path.length - 1; ++i, component = path[i]) {
+      object[component] = object[component] == null ? {
+        bsonType: 'object',
+        properties: {}
+      } : object[component];
+      object = object[component].properties;
+    }
+    object[component] = value;
+  }
+
+  /**
+   * @param {object} schemaMap the accumulation schemaMap
+   * @param {Model} the model
+   * @returns
+   */
+  function schemaMapReducer(schemaMap, model) {
+    const { schema, collection: { collectionName } } = model;
+    const namespace = `${this.$dbName}.${collectionName}`;
+
+    function schemaMapPropertyReducer(accum, [path, propertyConfig]) {
+      const bsonType = inferBSONType(schema, path);
+      const pathComponents = path.split('.');
+      const configuration = { encrypt: { ...propertyConfig, bsonType } };
+      buildNestedPath(pathComponents, accum, configuration);
+      return accum;
+    }
+    const properties = Object.entries(schema.encryptedFields).reduce(
+      schemaMapPropertyReducer,
+      {});
+
+    schemaMap[namespace] = {
+      bsonType: 'object',
+      properties
+    };
+    return schemaMap;
+  }
+
+  /**
+   *
+   * @param {object} encryptedFieldsMap the accumulation encryptedFieldsMap
+   * @param {Model} the model
+   * @returns
+   */
+  function encryptedFieldsMapReducer(encryptedFieldsMap, { schema, collection: { collectionName } }) {
+    const namespace = `${this.$dbName}.${collectionName}`;
+    const fields = Object.entries(schema.encryptedFields).map(
+      ([path, config]) => {
+        const bsonType = inferBSONType(schema, path);
+        // { path, bsonType, keyId, queries? }
+        return { path, bsonType, ...config };
+      });
+
+    encryptedFieldsMap[namespace] = { fields };
+
+    return encryptedFieldsMap;
+  }
+};
+
 /*!
  * ignore
  */
@@ -347,7 +456,7 @@ NativeConnection.prototype.setClient = function setClient(client) {
 
   for (const model of Object.values(this.models)) {
     // Errors handled internally, so safe to ignore error
-    model.init().catch(function $modelInitNoop() {});
+    model.init().catch(function $modelInitNoop() { });
   }
 
   return this;
@@ -390,9 +499,9 @@ function _setClient(conn, client, options, dbName) {
   };
 
   const type = client &&
-  client.topology &&
-  client.topology.description &&
-  client.topology.description.type || '';
+    client.topology &&
+    client.topology.description &&
+    client.topology.description.type || '';
 
   if (type === 'Single') {
     client.on('serverDescriptionChanged', ev => {

lib/utils.js

@@ -89,19 +89,19 @@ exports.deepEqual = function deepEqual(a, b) {
   }
 
   if ((isBsonType(a, 'ObjectId') && isBsonType(b, 'ObjectId')) ||
-      (isBsonType(a, 'Decimal128') && isBsonType(b, 'Decimal128'))) {
+    (isBsonType(a, 'Decimal128') && isBsonType(b, 'Decimal128'))) {
     return a.toString() === b.toString();
   }
 
   if (a instanceof RegExp && b instanceof RegExp) {
     return a.source === b.source &&
-        a.ignoreCase === b.ignoreCase &&
-        a.multiline === b.multiline &&
-        a.global === b.global &&
-        a.dotAll === b.dotAll &&
-        a.unicode === b.unicode &&
-        a.sticky === b.sticky &&
-        a.hasIndices === b.hasIndices;
+      a.ignoreCase === b.ignoreCase &&
+      a.multiline === b.multiline &&
+      a.global === b.global &&
+      a.dotAll === b.dotAll &&
+      a.unicode === b.unicode &&
+      a.sticky === b.sticky &&
+      a.hasIndices === b.hasIndices;
   }
 
   if (a == null || b == null) {
@@ -287,8 +287,8 @@ exports.merge = function merge(to, from, options, path) {
         // base schema has a given path as a single nested but discriminator schema
         // has the path as a document array, or vice versa (gh-9534)
         if (options.isDiscriminatorSchemaMerge &&
-            (from[key].$isSingleNested && to[key].$isMongooseDocumentArray) ||
-            (from[key].$isMongooseDocumentArray && to[key].$isSingleNested)) {
+          (from[key].$isSingleNested && to[key].$isMongooseDocumentArray) ||
+          (from[key].$isMongooseDocumentArray && to[key].$isSingleNested)) {
           continue;
         } else if (from[key].instanceOfSchema) {
           if (to[key].instanceOfSchema) {
@@ -995,7 +995,7 @@ exports.getOption = function(name) {
  * ignore
  */
 
-exports.noop = function() {};
+exports.noop = function() { };
 
 exports.errorToPOJO = function errorToPOJO(error) {
   const isError = error instanceof Error;
@@ -1025,3 +1025,13 @@ exports.injectTimestampsOption = function injectTimestampsOption(writeOperation,
   }
   writeOperation.timestamps = timestampsOption;
 };
+
+exports.print = function(...args) {
+  const { inspect } = require('util');
+  console.error(
+    inspect(
+      ...args,
+      { depth: Infinity }
+    )
+  );
+};

package.json

@@ -102,6 +102,7 @@
     "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js",
     "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js",
     "test-tsd": "node ./test/types/check-types-filename && tsd",
+    "test-encryption": "bash scripts/run-encryption-tests.sh",
     "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}",
     "test-coverage": "nyc --reporter=html --reporter=text npm test",
     "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check"
@@ -142,4 +143,4 @@
       "target": "ES2017"
     }
   }
-}
+}