Skip to content

Commit 8f0c99e

Browse files
addaleaxaddaleax
authored
feat: add csfle shared library MONGOSH-949 (#1279)
* feat: add packaging for CSFLE shared library MONGOSH-1116 (#1230) * feat: detect CSFLE library path and pass it to driver MONGOSH-1118 (#1246) - Add `--csfleLibrarySearchPath` and `csfleLibraryPath` command line flags to control CSFLE library search path behavior - This is necessary for us for testing, but also likely for users, because simply adding an entry to the `PATH` environment variable will not be enough to locate the library if it is in a non-standard location (unlike mongocryptd). - Use these options in our e2e/smoke tests - Download the shared library when required for testing - Add CSFLE path detection logic, which replaces our mongocryptd path detection logic - Pass the resulting AutoEncryptionExtraOptions down to the driver. * chore: bump mongodb-client-encryption, add ts-expect-error MONGOSH-1196 (#1266) This should be sufficient for bringing the CSFLE shared library work into a mergeable state. The ts-expect-error comments can likely be removed pretty soon as well, with the next driver bump. * fix(ci): rpm uses /usr/lib64 rather than /usr/lib * fixup: rc5 is minimum workingcsfle library version * fixup: drop ts-expect-error where unnecessary * fixup: update csfle distro lookup table, adjust new fle2 integration tests * fixup: monkey-patch mongodb-client-encryption for bypassQueryAnalysis * fixup: use minimum-glibc shared library in linux e2e tests, skip s390x fle testing * fixup: regen evergreen config, fix e2e fle test skipping * fixup: align new debian 11 dockerfile with docker changes
1 parent af0308e commit 8f0c99e

File tree

73 files changed

+756
-869
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+756
-869
lines changed

‎.evergreen.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5033,6 +5033,8 @@ functions:
50335033
set -e
50345034
set -x
50355035
{
5036+
export NODE_JS_VERSION=${node_js_version}
5037+
. .evergreen/setup-env.sh
50365038
. preload.sh
50375039
./scripts/docker/build.sh ${dockerfile}
50385040
./scripts/docker/run.sh ${dockerfile} --smokeTests
@@ -8394,8 +8396,13 @@ tasks:
83948396
vars:
83958397
source_distribution_build_variant: debian-x64
83968398
- func: write_preload_script
8399+
- func: install
8400+
vars:
8401+
node_js_version: "14.19.1"
8402+
npm_deps_mode: cli_build
83978403
- func: test_artifact_docker
83988404
vars:
8405+
node_js_version: "14.19.1"
83998406
dockerfile: ubuntu18.04-deb
84008407
- name: pkg_test_docker_ubuntu20_04_deb
84018408
tags: ["smoke-test"]
@@ -8408,8 +8415,13 @@ tasks:
84088415
vars:
84098416
source_distribution_build_variant: debian-x64
84108417
- func: write_preload_script
8418+
- func: install
8419+
vars:
8420+
node_js_version: "14.19.1"
8421+
npm_deps_mode: cli_build
84118422
- func: test_artifact_docker
84128423
vars:
8424+
node_js_version: "14.19.1"
84138425
dockerfile: ubuntu20.04-deb
84148426
- name: pkg_test_docker_debian9_deb
84158427
tags: ["smoke-test"]
@@ -8422,8 +8434,13 @@ tasks:
84228434
vars:
84238435
source_distribution_build_variant: debian-x64
84248436
- func: write_preload_script
8437+
- func: install
8438+
vars:
8439+
node_js_version: "14.19.1"
8440+
npm_deps_mode: cli_build
84258441
- func: test_artifact_docker
84268442
vars:
8443+
node_js_version: "14.19.1"
84278444
dockerfile: debian9-deb
84288445
- name: pkg_test_docker_debian10_deb
84298446
tags: ["smoke-test"]
@@ -8436,8 +8453,13 @@ tasks:
84368453
vars:
84378454
source_distribution_build_variant: debian-x64
84388455
- func: write_preload_script
8456+
- func: install
8457+
vars:
8458+
node_js_version: "14.19.1"
8459+
npm_deps_mode: cli_build
84398460
- func: test_artifact_docker
84408461
vars:
8462+
node_js_version: "14.19.1"
84418463
dockerfile: debian10-deb
84428464
- name: pkg_test_docker_debian11_deb
84438465
tags: ["smoke-test"]
@@ -8450,8 +8472,13 @@ tasks:
84508472
vars:
84518473
source_distribution_build_variant: debian-x64
84528474
- func: write_preload_script
8475+
- func: install
8476+
vars:
8477+
node_js_version: "14.19.1"
8478+
npm_deps_mode: cli_build
84538479
- func: test_artifact_docker
84548480
vars:
8481+
node_js_version: "14.19.1"
84558482
dockerfile: debian11-deb
84568483
- name: pkg_test_docker_centos7_rpm
84578484
tags: ["smoke-test"]
@@ -8464,8 +8491,13 @@ tasks:
84648491
vars:
84658492
source_distribution_build_variant: rhel7-x64
84668493
- func: write_preload_script
8494+
- func: install
8495+
vars:
8496+
node_js_version: "14.19.1"
8497+
npm_deps_mode: cli_build
84678498
- func: test_artifact_docker
84688499
vars:
8500+
node_js_version: "14.19.1"
84698501
dockerfile: centos7-rpm
84708502
- name: pkg_test_docker_amazonlinux2_rpm
84718503
tags: ["smoke-test"]
@@ -8478,8 +8510,13 @@ tasks:
84788510
vars:
84798511
source_distribution_build_variant: rhel7-x64
84808512
- func: write_preload_script
8513+
- func: install
8514+
vars:
8515+
node_js_version: "14.19.1"
8516+
npm_deps_mode: cli_build
84818517
- func: test_artifact_docker
84828518
vars:
8519+
node_js_version: "14.19.1"
84838520
dockerfile: amazonlinux2-rpm
84848521
- name: pkg_test_docker_rocky8_rpm
84858522
tags: ["smoke-test"]
@@ -8492,8 +8529,13 @@ tasks:
84928529
vars:
84938530
source_distribution_build_variant: rhel8-x64
84948531
- func: write_preload_script
8532+
- func: install
8533+
vars:
8534+
node_js_version: "14.19.1"
8535+
npm_deps_mode: cli_build
84958536
- func: test_artifact_docker
84968537
vars:
8538+
node_js_version: "14.19.1"
84978539
dockerfile: rocky8-rpm
84988540
- name: pkg_test_docker_fedora34_rpm
84998541
tags: ["smoke-test"]
@@ -8506,8 +8548,13 @@ tasks:
85068548
vars:
85078549
source_distribution_build_variant: rhel8-x64
85088550
- func: write_preload_script
8551+
- func: install
8552+
vars:
8553+
node_js_version: "14.19.1"
8554+
npm_deps_mode: cli_build
85098555
- func: test_artifact_docker
85108556
vars:
8557+
node_js_version: "14.19.1"
85118558
dockerfile: fedora34-rpm
85128559
- name: pkg_test_docker_suse12_rpm
85138560
tags: ["smoke-test"]
@@ -8520,8 +8567,13 @@ tasks:
85208567
vars:
85218568
source_distribution_build_variant: suse-x64
85228569
- func: write_preload_script
8570+
- func: install
8571+
vars:
8572+
node_js_version: "14.19.1"
8573+
npm_deps_mode: cli_build
85238574
- func: test_artifact_docker
85248575
vars:
8576+
node_js_version: "14.19.1"
85258577
dockerfile: suse12-rpm
85268578
- name: pkg_test_docker_suse15_rpm
85278579
tags: ["smoke-test"]
@@ -8534,8 +8586,13 @@ tasks:
85348586
vars:
85358587
source_distribution_build_variant: suse-x64
85368588
- func: write_preload_script
8589+
- func: install
8590+
vars:
8591+
node_js_version: "14.19.1"
8592+
npm_deps_mode: cli_build
85378593
- func: test_artifact_docker
85388594
vars:
8595+
node_js_version: "14.19.1"
85398596
dockerfile: suse15-rpm
85408597
- name: pkg_test_docker_amazonlinux1_rpm
85418598
tags: ["smoke-test"]
@@ -8548,8 +8605,13 @@ tasks:
85488605
vars:
85498606
source_distribution_build_variant: amzn1-x64
85508607
- func: write_preload_script
8608+
- func: install
8609+
vars:
8610+
node_js_version: "14.19.1"
8611+
npm_deps_mode: cli_build
85518612
- func: test_artifact_docker
85528613
vars:
8614+
node_js_version: "14.19.1"
85538615
dockerfile: amazonlinux1-rpm
85548616
- name: pkg_test_debextract_debian_arm64
85558617
tags: ["smoke-test"]

‎.evergreen/evergreen.yml.in‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,8 @@ functions:
521521
set -e
522522
set -x
523523
{
524+
export NODE_JS_VERSION=${node_js_version}
525+
. .evergreen/setup-env.sh
524526
. preload.sh
525527
./scripts/docker/build.sh ${dockerfile}
526528
./scripts/docker/run.sh ${dockerfile} --smokeTests
@@ -819,8 +821,13 @@ tasks:
819821
preload_script_path: preload.sh
820822
<% }; break;
821823
case 'docker': { %>
824+
- func: install
825+
vars:
826+
node_js_version: "<% out(NODE_JS_VERSION_14) %>"
827+
npm_deps_mode: cli_build
822828
- func: test_artifact_docker
823829
vars:
830+
node_js_version: "<% out(NODE_JS_VERSION_14) %>"
824831
dockerfile: <% out(dockerfile) %>
825832
<% }; break;
826833
// We don't have docker for platforms other than x64, so for those we just

‎config/build.conf.js‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
const path = require('path');
44
const os = require('os');
55

6+
const SHARED_LIBRARY_SUFFIX =
7+
process.platform === 'win32' ? 'dll' :
8+
process.platform === 'darwin' ? 'dylib' : 'so';
9+
610
/**
711
* The project root.
812
*/
@@ -45,11 +49,11 @@ const OUTPUT_DIR = path.join(ROOT, 'dist');
4549
const EXECUTABLE_PATH = path.join(OUTPUT_DIR, process.platform === 'win32' ? 'mongosh.exe' : 'mongosh');
4650

4751
/**
48-
* The name of the downloaded mongocryptd executable.
49-
* We use the name mongocryptd-mongosh to avoid conflicts with users
50-
* potentially installing the 'proper' mongocryptd package.
52+
* The path to the downloaded csfe shared library.
53+
* We use the name mongosh_csfle_v1 to avoid conflicts with users
54+
* potentially installing the 'proper' csfle shared library.
5155
*/
52-
const MONGOCRYPTD_PATH = path.resolve(TMP_DIR, 'mongocryptd-mongosh' + (process.platform === 'win32' ? '.exe' : ''));
56+
const CSFLE_LIBRARY_PATH = path.resolve(TMP_DIR, 'mongosh_csfle_v1.' + SHARED_LIBRARY_SUFFIX);
5357

5458
/**
5559
* Build info JSON data file.
@@ -105,7 +109,7 @@ module.exports = {
105109
repo: 'mongosh'
106110
},
107111
artifactUrlFile: process.env.ARTIFACT_URL_FILE,
108-
mongocryptdPath: MONGOCRYPTD_PATH,
112+
csfleLibraryPath: CSFLE_LIBRARY_PATH,
109113
packageInformation: {
110114
binaries: [
111115
{
@@ -120,11 +124,11 @@ module.exports = {
120124
}
121125
},
122126
{
123-
sourceFilePath: MONGOCRYPTD_PATH,
124-
category: 'libexec',
127+
sourceFilePath: CSFLE_LIBRARY_PATH,
128+
category: 'lib',
125129
license: {
126-
sourceFilePath: path.resolve(__dirname, '..', 'packaging', 'LICENSE-mongocryptd'),
127-
packagedFilePath: 'LICENSE-mongocryptd',
130+
sourceFilePath: path.resolve(__dirname, '..', 'packaging', 'LICENSE-csfle'),
131+
packagedFilePath: 'LICENSE-csfle',
128132
debCopyright: COPYRIGHT,
129133
debIdentifier: 'Proprietary',
130134
rpmIdentifier: 'Proprietary'

‎lerna.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"packages": [
3-
"packages/*"
3+
"packages/*",
4+
"scripts/docker"
45
],
56
"version": "0.0.0-dev.0"
67
}

‎package-lock.json‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@
124124
"lerna": "^4.0.0",
125125
"mocha": "^7.1.2",
126126
"mongodb": "^4.6.0",
127-
"mongodb-download-url": "^1.1.2",
127+
"mongodb-download-url": "^1.2.0",
128128
"mongodb-js-precommit": "^2.0.0",
129129
"nock": "^13.0.11",
130130
"node-codesign": "^0.3.3",

‎packages/arg-parser/src/arg-mapper.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,16 @@ function setAutoEncrypt<Key extends keyof AutoEncryptionOptions>(
3232
return setDriver(i, 'autoEncryption', autoEncryption);
3333
}
3434

35+
type AutoEncryptionExtraOptions = NonNullable<AutoEncryptionOptions['extraOptions']>;
36+
function setAutoEncryptExtra<Key extends keyof AutoEncryptionExtraOptions>(
37+
i: Readonly<ConnectionInfo>,
38+
key: Key,
39+
value: AutoEncryptionExtraOptions[Key]): ConnectionInfo {
40+
const extraOptions = i.driverOptions.autoEncryption?.extraOptions ?? {};
41+
extraOptions[key] = value;
42+
return setAutoEncrypt(i, 'extraOptions', extraOptions);
43+
}
44+
3545
type AWSKMSOptions = NonNullable<NonNullable<AutoEncryptionOptions['kmsProviders']>['aws']>;
3646
function setAWSKMS<Key extends keyof AWSKMSOptions>(
3747
i: Readonly<ConnectionInfo>,
@@ -95,6 +105,7 @@ const MAPPINGS: {
95105
awsSecretAccessKey: (i, v) => setAWSKMS(i, 'secretAccessKey', v),
96106
awsSessionToken: (i, v) => setAWSKMS(i, 'sessionToken', v),
97107
awsIamSessionToken: (i, v) => setAuthMechProp(i, 'AWS_SESSION_TOKEN', v),
108+
csfleLibraryPath: (i, v) => setAutoEncryptExtra(i, 'csflePath', v),
98109
gssapiServiceName: (i, v) => setAuthMechProp(i, 'SERVICE_NAME', v),
99110
sspiRealmOverride: (i, v) => setAuthMechProp(i, 'SERVICE_REALM', v),
100111
sspiHostnameCanonicalization:

‎packages/arg-parser/src/cli-options.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ export interface CliOptions {
1616
awsIamSessionToken?: string;
1717
awsSecretAccessKey?: string;
1818
awsSessionToken?: string;
19+
csfleLibraryPath?: string;
1920
db?: string;
2021
eval?: string;
2122
gssapiServiceName?: string;

‎packages/browser-repl/package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/build/src/compile/signable-compiler.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ export class SignableCompiler {
7373
path: await findModulePath('service-provider-server', 'os-dns-native'),
7474
requireRegexp: /\bos_dns_native\.node$/
7575
};
76+
const csfleLibraryVersionAddon = {
77+
path: await findModulePath('cli-repl', 'mongodb-csfle-library-version'),
78+
requireRegexp: /\bmongodb_csfle_library_version\.node$/
79+
};
7680
// Warning! Until https://jira.mongodb.org/browse/MONGOSH-990,
7781
// packages/service-provider-server *also* has a copy of these.
7882
// We use the versions included in packages/cli-repl here, so these
@@ -110,7 +114,8 @@ export class SignableCompiler {
110114
addons: [
111115
fleAddon,
112116
osDnsAddon,
113-
kerberosAddon
117+
kerberosAddon,
118+
csfleLibraryVersionAddon
114119
].concat(winCAAddon ? [
115120
winCAAddon
116121
] : []).concat(winConsoleProcessListAddon ? [

0 commit comments

Comments
 (0)