all changes

Author: baileympearson

.github/workflows/encryption-tests.yml

@@ -0,0 +1,37 @@
+name: Encryption Tests 
+
+on:
+  push: 
+    branches: ['master']
+  pull_request:
+    branches: [ 'master' ]
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+  id-token: write
+
+jobs:
+  run-tests:
+    permissions:
+      # required for all workflows
+      security-events: write
+      id-token: write
+      contents: write
+    runs-on: ubuntu-latest
+    name: Encryption tests
+    env:
+      FORCE_COLOR: true
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Setup node
+        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        with:
+          node-version: latest
+      - name: Install Dependencies
+        run: npm install
+      - name: Install mongodb-client-encryption
+        run: npm install mongodb-client-encryption
+      - name: Run Tests
+        run: npm run test-encryption

.gitignore

@@ -67,3 +67,6 @@ examples/ecommerce-netlify-functions/.netlify/state.json
 
 notes.md
 list.out
+
+data
+*.pid

CONTRIBUTING.md

@@ -46,6 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe
   * execute `npm run test-tsd` to run the typescript tests
   * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time.
   * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command.
+* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file.
 
 ## Documentation
 

package.json

@@ -102,6 +102,7 @@
     "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js",
     "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js",
     "test-tsd": "node ./test/types/check-types-filename && tsd",
+    "test-encryption": "bash scripts/run-encryption-tests.sh",
     "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}",
     "test-coverage": "nyc --reporter=html --reporter=text npm test",
     "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check"
@@ -142,4 +143,4 @@
       "target": "ES2017"
     }
   }
-}
+}

scripts/configure-cluster-with-encryption.sh

@@ -0,0 +1,40 @@
+# note: in order to use FLE with mongodb, we must
+#  have mongocryptd or the shared library downloaded
+#  have an enterprise server >= 4.2
+
+# this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server)
+
+export CWD=$(pwd);
+mkdir data
+cd data
+
+# note: 
+  # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing.  
+  # if you'd like to make changes to the cluster settings, edit the exported variables below.
+  # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh
+  # after this script is run, the data/ folder will notably contain the following:
+    # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path
+    # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod
+if [ ! -d "drivers-evergreen-tools/" ]; then
+ git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git"
+fi
+
+# configure cluster settings
+export DRIVERS_TOOLS=$CWD/data/drivers-evergreen-tools
+export MONGODB_VERSION=8.0
+export AUTH=true
+export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin
+export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo
+export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration
+export TOPOLOGY=sharded_cluster
+export SSL=nossl
+
+cd $DRIVERS_TOOLS
+rm -rf mongosh mongodb mo
+mkdir mo
+cd -
+
+rm expansions.sh 2> /dev/null
+
+# start cluster
+bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh

scripts/run-encryption-tests.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# sets up mongodb cluster and encryption configuration, adds relevant variables to the environment, and runs encryption tests
+
+export CWD=$(pwd);
+
+# set up mongodb cluster and encryption configuration if the data/ folder does not exist
+# note: for tooling, cluster set-up and configuration look into the 'scripts/configure-cluster-with-encryption.sh' script
+
+if [ -d "data" ]; then
+  cd data
+else
+  source $CWD/scripts/configure-cluster-with-encryption.sh
+fi
+
+# extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run
+read -r -d '' SOURCE_SCRIPT << EOM
+const fs = require('fs');
+const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' })
+	.trim().split('\\n');
+const regex = /^(?<key>.*): "(?<value>.*)"$/;
+const variables = file.map(
+	(line) => regex.exec(line.trim()).groups
+).map(
+	({key, value}) => \`export \${key}='\${value}'\`
+).join('\n');
+
+process.stdout.write(variables);
+process.stdout.write('\n');
+EOM
+
+node --eval "$SOURCE_SCRIPT" | tee expansions.sh
+source expansions.sh
+
+export MONGOOSE_TEST_URI=$MONGODB_URI
+
+# run encryption tests
+cd ..
+npx mocha --exit ./test/encryption/*.test.js

test/encryption/encryption.test.js

@@ -0,0 +1,96 @@
+'use strict';
+
+const assert = require('assert');
+const mdb = require('mongodb');
+const isBsonType = require('../../lib/helpers/isBsonType');
+
+const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64');
+
+describe('ci', () => {
+  describe('environmental variables', () => {
+    it('MONGOOSE_TEST_URI is set', async function() {
+      const uri = process.env.MONGOOSE_TEST_URI;
+      assert.ok(uri);
+    });
+
+    it('CRYPT_SHARED_LIB_PATH is set', async function() {
+      const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH;
+      assert.ok(shared_library_path);
+    });
+  });
+
+  describe('basic integration', () => {
+    let keyVaultClient;
+    let dataKey;
+    let encryptedClient;
+    let unencryptedClient;
+
+    beforeEach(async function() {
+      keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI);
+      await keyVaultClient.connect();
+      await keyVaultClient.db('keyvault').collection('datakeys');
+      const clientEncryption = new mdb.ClientEncryption(keyVaultClient, {
+        keyVaultNamespace: 'keyvault.datakeys',
+        kmsProviders: { local: { key: LOCAL_KEY } }
+      });
+      dataKey = await clientEncryption.createDataKey('local');
+
+      encryptedClient = new mdb.MongoClient(
+        process.env.MONGOOSE_TEST_URI,
+        {
+          autoEncryption: {
+            keyVaultNamespace: 'keyvault.datakeys',
+            kmsProviders: { local: { key: LOCAL_KEY } },
+            schemaMap: {
+              'db.coll': {
+                bsonType: 'object',
+                encryptMetadata: {
+                  keyId: [dataKey]
+                },
+                properties: {
+                  a: {
+                    encrypt: {
+                      bsonType: 'int',
+                      algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random',
+                      keyId: [dataKey]
+                    }
+                  }
+                }
+              }
+            },
+            extraOptions: {
+              cryptdSharedLibRequired: true,
+              cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH
+            }
+          }
+        }
+      );
+
+      unencryptedClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI);
+    });
+
+    afterEach(async function() {
+      await keyVaultClient.close();
+      await encryptedClient.close();
+      await unencryptedClient.close();
+    });
+
+    it('ci set-up should support basic mongodb auto-encryption integration', async() => {
+      await encryptedClient.connect();
+      const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 });
+
+      // client not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded
+      const encryptedResult = await unencryptedClient.db('db').collection('coll').findOne({ _id: insertedId });
+
+      assert.ok(encryptedResult);
+      assert.ok(encryptedResult.a);
+      assert.ok(isBsonType(encryptedResult.a, 'Binary'));
+      assert.ok(encryptedResult.a.sub_type === 6);
+
+      // when the encryptedClient runs a find, the original unencrypted value is returned
+      const unencryptedResult = await encryptedClient.db('db').collection('coll').findOne({ _id: insertedId });
+      assert.ok(unencryptedResult);
+      assert.ok(unencryptedResult.a === 1);
+    });
+  });
+});