Skip to content

Commit a3556dd

Browse files
authored
fix(cli-repl): spawn one mongocryptd per FLE-using shell MONGOSH-595 (#746)
See the ticket for motivation.
1 parent becc1ab commit a3556dd

21 files changed

+651
-38
lines changed

packages/cli-repl/src/cli-repl.ts

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { MongoshInternalError, MongoshWarning } from '@mongosh/errors';
22
import { redactPassword } from '@mongosh/history';
33
import i18n from '@mongosh/i18n';
4-
import { bson } from '@mongosh/service-provider-core';
4+
import { bson, AutoEncryptionOptions } from '@mongosh/service-provider-core';
55
import { CliOptions, CliServiceProvider, MongoClientOptions } from '@mongosh/service-provider-server';
66
import Analytics from 'analytics-node';
77
import askpassword from 'askpassword';
@@ -12,6 +12,7 @@ import { Readable, Writable } from 'stream';
1212
import type { StyleDefinition } from './clr';
1313
import { ConfigManager, ShellHomeDirectory, ShellHomePaths } from './config-directory';
1414
import { CliReplErrors } from './error-codes';
15+
import { MongocryptdManager } from './mongocryptd-manager';
1516
import MongoshNodeRepl, { MongoshNodeReplOptions } from './mongosh-repl';
1617
import setupLoggerAndTelemetry from './setup-logger-and-telemetry';
1718
import { MongoshBus, UserConfig } from '@mongosh/types';
@@ -32,7 +33,8 @@ type AnalyticsOptions = {
3233
};
3334

3435
export type CliReplOptions = {
35-
shellCliOptions: CliOptions & { mongocryptdSpawnPath?: string },
36+
shellCliOptions: CliOptions;
37+
mongocryptdSpawnPaths?: string[][],
3638
input: Readable;
3739
output: Writable;
3840
shellHomePaths: ShellHomePaths;
@@ -47,6 +49,7 @@ class CliRepl {
4749
mongoshRepl: MongoshNodeRepl;
4850
bus: MongoshBus;
4951
cliOptions: CliOptions;
52+
mongocryptdManager: MongocryptdManager;
5053
shellHomeDirectory: ShellHomeDirectory;
5154
configDirectory: ConfigManager<UserConfig>;
5255
config: UserConfig = new UserConfig();
@@ -81,6 +84,11 @@ class CliRepl {
8184
.on('update-config', (config: UserConfig) =>
8285
this.bus.emit('mongosh:update-user', config.userId, config.enableTelemetry));
8386

87+
this.mongocryptdManager = new MongocryptdManager(
88+
options.mongocryptdSpawnPaths ?? [],
89+
this.shellHomeDirectory,
90+
this.bus);
91+
8492
// We can't really do anything meaningfull if the output stream is broken or
8593
// closed. To avoid throwing an error while writing to it, let's send it to
8694
// the telemetry instead
@@ -149,6 +157,15 @@ class CliRepl {
149157
this.warnAboutInaccessibleFile(err);
150158
}
151159

160+
if (driverOptions.autoEncryption) {
161+
const extraOptions = {
162+
...(driverOptions.autoEncryption.extraOptions ?? {}),
163+
...(await this.startMongocryptd())
164+
};
165+
166+
driverOptions.autoEncryption = { ...driverOptions.autoEncryption, extraOptions };
167+
}
168+
152169
const initialServiceProvider = await this.connect(driverUri, driverOptions);
153170
const initialized = await this.mongoshRepl.initialize(initialServiceProvider);
154171
const commandLineLoadFiles = this.listCommandLineLoadFiles();
@@ -374,6 +391,7 @@ class CliRepl {
374391
await promisify(analytics.flush.bind(analytics))();
375392
} catch { /* ignore */ }
376393
}
394+
await this.mongocryptdManager.close();
377395
this.bus.emit('mongosh:closed');
378396
}
379397

@@ -397,6 +415,10 @@ class CliRepl {
397415
clr(text: string, style: StyleDefinition): string {
398416
return this.mongoshRepl.clr(text, style);
399417
}
418+
419+
async startMongocryptd(): Promise<AutoEncryptionOptions['extraOptions']> {
420+
return await this.mongocryptdManager.start();
421+
}
400422
}
401423

402424
export default CliRepl;

packages/cli-repl/src/index.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import CliRepl from './cli-repl';
44
import clr from './clr';
55
import { getStoragePaths } from './config-directory';
66
import { MONGOSH_WIKI, TELEMETRY_GREETING_MESSAGE, USAGE } from './constants';
7-
import { getMongocryptdPath } from './mongocryptd-path';
7+
import { getMongocryptdPaths } from './mongocryptd-manager';
88
import { runSmokeTests } from './smoke-tests';
99

1010
export default CliRepl;
@@ -18,6 +18,6 @@ export {
1818
parseCliArgs,
1919
mapCliToDriver,
2020
getStoragePaths,
21-
getMongocryptdPath,
21+
getMongocryptdPaths,
2222
runSmokeTests
2323
};
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/* eslint-disable chai-friendly/no-unused-expressions */
2+
import Nanobus from 'nanobus';
3+
import { promises as fs } from 'fs';
4+
import path from 'path';
5+
import { promisify } from 'util';
6+
import { getMongocryptdPaths, MongocryptdManager } from './mongocryptd-manager';
7+
import type { MongoshBus } from '@mongosh/types';
8+
import { ShellHomeDirectory } from './config-directory';
9+
import { startTestServer } from '../../../testing/integration-testing-hooks';
10+
import { expect } from 'chai';
11+
12+
describe('getMongocryptdPaths', () => {
13+
it('always includes plain `mongocryptd`', async() => {
14+
expect(await getMongocryptdPaths()).to.deep.include(['mongocryptd']);
15+
});
16+
});
17+
18+
describe('MongocryptdManager', () => {
19+
let basePath: string;
20+
let bus: MongoshBus;
21+
let shellHomeDirectory: ShellHomeDirectory;
22+
let spawnPaths: string[][];
23+
let manager: MongocryptdManager;
24+
let events: { event: string, data: any }[];
25+
const makeManager = () => {
26+
manager = new MongocryptdManager(spawnPaths, shellHomeDirectory, bus);
27+
return manager;
28+
};
29+
30+
const fakeMongocryptdDir = path.resolve(__dirname, '..', 'test', 'fixtures', 'fake-mongocryptd');
31+
32+
beforeEach(() => {
33+
const nanobus = new Nanobus();
34+
events = [];
35+
nanobus.on('*', (event, data) => events.push({ event, data }));
36+
bus = nanobus;
37+
38+
spawnPaths = [];
39+
basePath = path.resolve(__dirname, '..', '..', '..', 'tmp', 'test', `${Date.now()}`, `${Math.random()}`);
40+
shellHomeDirectory = new ShellHomeDirectory({
41+
shellRoamingDataPath: basePath,
42+
shellLocalDataPath: basePath,
43+
shellRcPath: basePath
44+
});
45+
});
46+
afterEach(() => {
47+
manager?.close();
48+
});
49+
50+
it('does a no-op close when not initialized', () => {
51+
expect(makeManager().close().state).to.equal(null);
52+
});
53+
54+
for (const otherMongocryptd of ['none', 'missing', 'broken', 'weirdlog']) {
55+
for (const version of ['4.2', '4.4']) {
56+
for (const variant of ['withunix', 'nounix']) {
57+
// eslint-disable-next-line no-loop-func
58+
it(`spawns a working mongocryptd (${version}, ${variant}, other mongocryptd: ${otherMongocryptd})`, async() => {
59+
spawnPaths = [
60+
[
61+
process.execPath,
62+
path.resolve(fakeMongocryptdDir, `working-${version}-${variant}.js`)
63+
]
64+
];
65+
if (otherMongocryptd === 'missing') {
66+
spawnPaths.unshift([ path.resolve(fakeMongocryptdDir, 'nonexistent') ]);
67+
}
68+
if (otherMongocryptd === 'broken') {
69+
spawnPaths.unshift([ process.execPath, path.resolve(fakeMongocryptdDir, 'exit1') ]);
70+
}
71+
if (otherMongocryptd === 'weirdlog') {
72+
spawnPaths.unshift([ process.execPath, path.resolve(fakeMongocryptdDir, 'weirdlog') ]);
73+
}
74+
expect(await makeManager().start()).to.deep.equal({
75+
mongocryptdURI: variant === 'nounix' ?
76+
'mongodb://localhost:27020' :
77+
'mongodb://%2Ftmp%2Fmongocryptd.sock',
78+
mongocryptdBypassSpawn: true
79+
});
80+
81+
const tryspawns = events.filter(({ event }) => event === 'mongosh:mongocryptd-tryspawn');
82+
expect(tryspawns).to.have.lengthOf(otherMongocryptd === 'none' ? 1 : 2);
83+
});
84+
}
85+
}
86+
}
87+
88+
it('passes relevant arguments to mongocryptd', async() => {
89+
spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'writepidfile.js')]];
90+
await makeManager().start();
91+
const pidfile = path.join(manager.path, 'mongocryptd.pid');
92+
expect(JSON.parse(await fs.readFile(pidfile, 'utf8')).args).to.deep.equal([
93+
...spawnPaths[0],
94+
'--idleShutdownTimeoutSecs', '60',
95+
'--pidfilepath', pidfile,
96+
'--port', '0',
97+
...(process.platform !== 'win32' ? ['--unixSocketPrefix', path.dirname(pidfile)] : [])
98+
]);
99+
});
100+
101+
it('multiple start() calls are no-ops', async() => {
102+
spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'writepidfile.js')]];
103+
const manager = makeManager();
104+
await manager.start();
105+
const pid1 = manager.state.proc.pid;
106+
await manager.start();
107+
expect(manager.state.proc.pid).to.equal(pid1);
108+
});
109+
110+
it('handles synchronous throws from child_process.spawn', async() => {
111+
spawnPaths = [['']];
112+
try {
113+
await makeManager().start();
114+
expect.fail('missed exception');
115+
} catch (e) {
116+
expect(e.code).to.equal('ERR_INVALID_ARG_VALUE');
117+
}
118+
});
119+
120+
it('throws if no spawn paths are provided at all', async() => {
121+
spawnPaths = [];
122+
try {
123+
await makeManager().start();
124+
expect.fail('missed exception');
125+
} catch (e) {
126+
expect(e.name).to.equal('MongoshInternalError');
127+
}
128+
});
129+
130+
it('includes stderr in the log if stdout is unparseable', async() => {
131+
spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'weirdlog.js')]];
132+
try {
133+
await makeManager().start();
134+
expect.fail('missed exception');
135+
} catch (e) {
136+
expect(e.name).to.equal('MongoshInternalError');
137+
}
138+
const nostdoutErrors = events.filter(({ event, data }) => {
139+
return event === 'mongosh:mongocryptd-error' && data.cause === 'nostdout';
140+
});
141+
expect(nostdoutErrors).to.deep.equal([{
142+
event: 'mongosh:mongocryptd-error',
143+
data: { cause: 'nostdout', stderr: 'Diagnostic message!\n' }
144+
}]);
145+
});
146+
147+
it('cleans up previously created, empty directory entries', async() => {
148+
spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'writepidfile.js')]];
149+
150+
const manager = makeManager();
151+
await manager.start();
152+
const pidfile = path.join(manager.path, 'mongocryptd.pid');
153+
expect(JSON.parse(await fs.readFile(pidfile, 'utf8')).pid).to.be.a('number');
154+
manager.close();
155+
156+
// The file remains after close, but is gone after creating a new one:
157+
await fs.stat(pidfile);
158+
await makeManager().start();
159+
try {
160+
await fs.stat(pidfile);
161+
expect.fail('missed exception');
162+
} catch (e) {
163+
expect(e.code).to.equal('ENOENT');
164+
}
165+
});
166+
167+
context('with network testing', () => {
168+
const testServer = startTestServer('shared');
169+
170+
beforeEach(async() => {
171+
process.env.MONGOSH_TEST_PROXY_TARGET_PORT = await testServer.port();
172+
});
173+
afterEach(() => {
174+
delete process.env.MONGOSH_TEST_PROXY_TARGET_PORT;
175+
});
176+
177+
it('performs keepalive pings', async() => {
178+
spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'withnetworking.js')]];
179+
const manager = makeManager();
180+
manager.idleShutdownTimeoutSecs = 1;
181+
await manager.start();
182+
const pidfile = path.join(manager.path, 'mongocryptd.pid');
183+
await promisify(setTimeout)(2000);
184+
expect(JSON.parse(await fs.readFile(pidfile, 'utf8')).connections).to.be.greaterThan(1);
185+
});
186+
});
187+
});

0 commit comments

Comments
 (0)