feat(redact): add a dictionary of secrets to redact MCP-29 (#569)

* feat(redact): add a dictionary of secrets to redact

With this feature we support redacting known secrets from a
dictionary. This is value for environments like the MCP Server where
we create new users (user/password) or when we can infer secrets from
CLI arguments (--user, --pasword).

* chore: add support for arbitrarily deep object redacting

* chore: refactor to reuse the iteration, instead of iterating twice

* chore: fix linter warnings

* chore: fix an issue when the constructor is not a function

We don't want to return a truthy value if the ctor is a non function
value (like an object). Plain objects either have a plain object as
a prototype or a null prototype

Author: kmruiz

package-lock.json

@@ -58,6 +58,7 @@
     "@mongodb-js/tsconfig-devtools": "^1.0.3",
     "@types/chai": "^4.2.21",
     "@types/mocha": "^9.1.1",
+    "@types/regexp.escape": "^2.0.0",
     "@types/sinon-chai": "^3.2.5",
     "chai": "^4.5.0",
     "depcheck": "^1.4.7",
@@ -68,5 +69,8 @@
     "prettier": "^3.5.3",
     "sinon": "^9.2.3",
     "typescript": "^5.0.4"
+  },
+  "dependencies": {
+    "regexp.escape": "^2.0.1"
   }
 }

packages/mongodb-redact/package.json

@@ -24,6 +24,25 @@ const BIN_DATA = `db = db.getSiblingDB("__realm_sync")
 db.history.updateOne({"_id": ObjectId("63ed1d522d8573fa5c203660")}, {$set:{changeset:BinData(5, "iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAADMElEQVR4nOzVwQnAIBQFQYXff81RUkQCOyDj1YOPnbXWPmeTRef+/3O/OyBjzh3CD95BfqICMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMO0TAAD//2Anhf4QtqobAAAAAElFTkSuQmCC")}})`;
 
 describe('mongodb-redact', function () {
+  describe('Secrets', function () {
+    it('should redact provided secrets in strings', function () {
+      const res = redact('foo@bar', [{ value: 'foo@bar', kind: 'password' }]);
+      expect(res).to.equal('<password>');
+    });
+
+    it('should redact provided secrets in objects', function () {
+      const res = redact({ key: 'foo@bar' }, [
+        { value: 'foo@bar', kind: 'password' },
+      ]);
+      expect(res).to.deep.equal({ key: '<password>' });
+    });
+
+    it('should redact provided secrets in arrays', function () {
+      const res = redact(['foo@bar'], [{ value: 'foo@bar', kind: 'password' }]);
+      expect(res).to.deep.equal(['<password>']);
+    });
+  });
+
   describe('Types', function () {
     it('should work with string types', function () {
       const res = redact('[email protected]');

packages/mongodb-redact/src/index.spec.ts

@@ -1,40 +1,42 @@
 import { regexes } from './regexes';
+import { isPlainObject } from './utils';
+import { redactSecretsOnString } from './secrets';
+import type { Secret } from './secrets';
 
-const plainObjectTag = Object.prototype.toString.call({});
-function isPlainObject(val: unknown): val is object {
-  if (
-    typeof val !== 'object' ||
-    !val ||
-    Object.prototype.toString.call(val) !== plainObjectTag
-  ) {
-    return false;
-  }
-  const proto = Object.getPrototypeOf(val);
-  if (proto === null) return true;
-  if (!Object.prototype.hasOwnProperty.call(proto, 'constructor')) return false;
-  const ctor = proto.constructor;
-  if (typeof ctor !== 'function') return ctor;
-  // `ctor === Object` but this works across contexts
-  // (Object is special because Object.__proto__.__proto__ === Object.prototype),
-  const ctorPrototype = Object.getPrototypeOf(ctor);
-  return Object.getPrototypeOf(ctorPrototype) === ctor.prototype;
-}
-
-export function redact<T>(message: T): T {
+export function redact<T>(
+  message: T,
+  secrets: Secret[] | undefined = undefined,
+): T {
   if (isPlainObject(message)) {
     // recursively walk through all values of an object
-    return Object.fromEntries(
-      Object.entries(message).map(([key, value]) => [key, redact(value)]),
+    const newMessage = Object.fromEntries(
+      Object.entries(message).map(([key, value]) => [
+        key,
+        redact(value, secrets),
+      ]),
     ) as T;
+
+    // make sure we inherit the prototype so we don't add new behaviour to the object
+    // nobody is expecting
+    return Object.setPrototypeOf(
+      newMessage,
+      Object.getPrototypeOf(message) as object | null,
+    );
   }
+
   if (Array.isArray(message)) {
     // walk through array and redact each value
-    return message.map(redact) as T;
+    return message.map((msg) => redact(msg, secrets)) as T;
   }
   if (typeof message !== 'string') {
     // all non-string types can be safely returned
     return message;
   }
+
+  if (secrets) {
+    message = redactSecretsOnString(message, secrets);
+  }
+
   // apply all available regexes to the string
   for (const [regex, replacement] of regexes) {
     // The type here isn't completely accurate in case `T` is a specific string template
@@ -45,3 +47,4 @@ export function redact<T>(message: T): T {
 }
 
 export default redact;
+export type { Secret } from './secrets';

packages/mongodb-redact/src/index.ts

@@ -0,0 +1,96 @@
+import { expect } from 'chai';
+import { SECRET_KIND } from './secrets';
+import type { SecretKind } from './secrets';
+import { redact } from './index';
+
+describe('dictionary-based secret redaction', function () {
+  for (const kind of SECRET_KIND) {
+    it(`redacts content of kind '${kind}'`, function () {
+      const secret = '123456';
+      const content = '123456';
+
+      const redacted = redact(content, [{ value: secret, kind: kind }]);
+
+      expect(redacted).equal(`<${kind}>`);
+    });
+  }
+
+  for (const invalidValue of [null, undefined, false, 0]) {
+    it(`returns itself on an invalid value like ${String(invalidValue)}`, function () {
+      expect(redact(invalidValue as any, [])).equal(invalidValue);
+    });
+  }
+
+  it('redacts multiple coincidences of a secret', function () {
+    const secret = '123456';
+    const content = '123456 abc 123456';
+
+    const redacted = redact(content, [{ value: secret, kind: 'password' }]);
+
+    expect(redacted).to.equal('<password> abc <password>');
+  });
+
+  it('rejects unknown types', function () {
+    expect(() =>
+      redact('some content to redact', [
+        {
+          value: 'some',
+          kind: 'invalid' as SecretKind,
+        },
+      ]),
+    ).throw();
+  });
+
+  it('redacts secrets in between word boundaries', function () {
+    const secret = '123456';
+    const content = '.123456.';
+
+    const redacted = redact(content, [{ value: secret, kind: 'password' }]);
+
+    expect(redacted).equal('.<password>.');
+  });
+
+  it('does not redact content that seems a secret inside another word', function () {
+    const secret = '123456';
+    const content = 'abc123456def';
+
+    const redacted = redact(content, [{ value: secret, kind: 'password' }]);
+
+    expect(redacted).equal('abc123456def');
+  });
+
+  it('escapes values so using it in regexes is safe', function () {
+    const secret = '.+';
+    const content = '.abcdef.';
+
+    const redacted = redact(content, [{ value: secret, kind: 'password' }]);
+
+    expect(redacted).equal('.abcdef.');
+  });
+
+  it('redacts on arrays', function () {
+    const secret = 'abc';
+    const content = ['abc', 'cbd'];
+
+    const redacted = redact(content, [{ value: secret, kind: 'password' }]);
+
+    expect(redacted).deep.equal(['<password>', 'cbd']);
+  });
+
+  it('redacts on objects', function () {
+    const pwdSecret = '123456';
+    const usrSecret = 'admin';
+
+    const content = { pwd: pwdSecret, usr: usrSecret };
+
+    const redacted = redact(content, [
+      { value: pwdSecret, kind: 'password' },
+      { value: usrSecret, kind: 'user' },
+    ]);
+
+    expect(redacted).deep.equal({
+      pwd: '<password>',
+      usr: '<user>',
+    });
+  });
+});

packages/mongodb-redact/src/secrets.spec.ts

@@ -0,0 +1,40 @@
+// RegExp.escape is not widely available yet, so we are going to use regexp.escape
+// as it's extremely simple and at some point we can get rid of it easily.
+import escape from 'regexp.escape';
+
+export const SECRET_KIND = [
+  'base64',
+  'private key',
+  'user',
+  'password',
+  'email',
+  'ip',
+  'url',
+  'mongodb uri',
+] as const;
+
+export type SecretKind = (typeof SECRET_KIND)[number];
+
+export type Secret = {
+  readonly value: string;
+  readonly kind: SecretKind;
+};
+
+export function redactSecretsOnString<T extends string>(
+  content: T,
+  secrets: Secret[],
+): T {
+  let result = content;
+  for (const { value, kind } of secrets) {
+    if (!SECRET_KIND.includes(kind)) {
+      throw new Error(
+        `Unknown secret kind ${kind}. Valid types: ${SECRET_KIND.join(', ')}`,
+      );
+    }
+
+    const regex = new RegExp(`\\b${escape(value)}\\b`, 'g');
+    result = result.replace(regex, `<${kind}>`) as T;
+  }
+
+  return result;
+}

packages/mongodb-redact/src/secrets.ts

@@ -0,0 +1,20 @@
+const plainObjectTag = Object.prototype.toString.call({});
+
+export function isPlainObject(val: unknown): val is object {
+  if (
+    typeof val !== 'object' ||
+    !val ||
+    Object.prototype.toString.call(val) !== plainObjectTag
+  ) {
+    return false;
+  }
+  const proto = Object.getPrototypeOf(val);
+  if (proto === null) return true;
+  if (!Object.prototype.hasOwnProperty.call(proto, 'constructor')) return false;
+  const ctor = proto.constructor;
+  if (typeof ctor !== 'function') return false;
+  // `ctor === Object` but this works across contexts
+  // (Object is special because Object.__proto__.__proto__ === Object.prototype),
+  const ctorPrototype = Object.getPrototypeOf(ctor);
+  return Object.getPrototypeOf(ctorPrototype) === ctor.prototype;
+}

packages/mongodb-redact/src/utils.ts

@@ -3,7 +3,8 @@
   "compilerOptions": {
     "outDir": "dist",
     "allowJs": true,
-    "strict": true
+    "strict": true,
+    "esModuleInterop": true
   },
   "include": ["src/**/*"],
   "exclude": ["./src/**/*.spec.*"]