chore(ci): cache compiled TS result MONGOSH-870 (#1026)

Author: addaleax

.evergreen.yml

@@ -3,7 +3,7 @@ set -e
 
 cd $(pwd)
 
-source .evergreen/.setup_env
+source .evergreen/setup-env.sh
 
 if uname -a | grep -q 'Linux.*x86_64'; then
   rm -rf "tmp/.sccache"

.evergreen/compile-artifact.sh

@@ -88,7 +88,8 @@ exec_timeout_secs: 3600
 #
 # Current functions:
 #   checkout - Checks out the project from git.
-#   install - Installs Node and all dependencies.
+#   compile_ts - Installs Node and all dependencies, and run all prepublish scripts.
+#   install - Installs Node and all dependencies, and download the result of running all prepublish scripts.
 #   check - Performs linter and dependency checks.
 #   test - Runs all tests.
 #   test_vscode - Clones the vscode extension repository and runs its tests.
@@ -105,6 +106,30 @@ functions:
     - command: git.get_project
       params:
         directory: src
+  compile_ts:
+    - command: expansions.write
+      params:
+        file: tmp/expansions.yaml
+        redacted: true
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        script: |
+          export NODE_JS_VERSION=${node_js_version}
+          source .evergreen/install-node.sh
+          npm run evergreen-release bump
+          npm run bootstrap-ci
+          tar cvzf compiled-ts.tgz packages/*/{lib,dist}
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/compiled-ts.tgz
+        remote_file: mongosh/binaries/${revision}/${revision_order_id}/compiled-ts.tgz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/x-gzip
   install:
     - command: expansions.write
       params:
@@ -116,8 +141,21 @@ functions:
         shell: bash
         script: |
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.install_node
+          source .evergreen/install-node.sh
           npm run evergreen-release bump
+    - command: s3.get
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/compiled-ts.tgz
+        remote_file: mongosh/binaries/${revision}/${revision_order_id}/compiled-ts.tgz
+        bucket: mciuploads
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        script: |
+          tar xvzf compiled-ts.tgz
   check:
     - command: shell.exec
       params:
@@ -127,8 +165,8 @@ functions:
           set -e
           {
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.setup_env
-          npm --unsafe-perm=true run bootstrap-ci
+          source .evergreen/setup-env.sh
+          npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish
           npm run check-ci
           }
   test:
@@ -144,8 +182,8 @@ functions:
           set -e
           {
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.setup_env
-          npm --unsafe-perm=true run bootstrap-ci
+          source .evergreen/setup-env.sh
+          npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish
           npm run test-ci
           echo "Creating coverage tarball..."
           tar cvzf coverage.tgz coverage
@@ -171,8 +209,8 @@ functions:
           set -e
           {
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.setup_env
-          npm --unsafe-perm=true run bootstrap-ci
+          source .evergreen/setup-env.sh
+          npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish
           (cd scripts/docker && docker build -t ubuntu18.04-xvfb -f ubuntu18.04-xvfb.Dockerfile .)
           docker run \
             --rm -v $PWD:/tmp/build ubuntu18.04-xvfb \
@@ -191,8 +229,8 @@ functions:
           set -e
           {
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.setup_env
-          npm --unsafe-perm=true run bootstrap-ci
+          source .evergreen/setup-env.sh
+          npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish
           npm run test-connectivity
           }
   test_apistrict:
@@ -208,7 +246,7 @@ functions:
           set -e
           {
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.setup_env
+          source .evergreen/setup-env.sh
           npm run test-apistrict-ci
           }
 
@@ -447,7 +485,7 @@ functions:
           set -e
           {
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.setup_env
+          source .evergreen/setup-env.sh
           export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
           npm run evergreen-release draft
           }
@@ -465,18 +503,19 @@ functions:
           set -e
           {
           export NODE_JS_VERSION=${node_js_version}
-          source .evergreen/.setup_env
+          source .evergreen/setup-env.sh
           export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
           echo "registry=https://registry.npmjs.org/" > .npmrc
           echo "//registry.npmjs.org/:_authToken=${devtoolsbot_npm_token}" >> .npmrc
-          npm --unsafe-perm=true run bootstrap-ci
+          npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish
           npm run evergreen-release publish
           }
 
 # Tasks will show up as the individual blocks in the Evergreen UI that can
 # pass or fail.
 #
 # Current tasks:
+#   compile_ts - Do the initial compilation of TS sources.
 #   check - Performs linter and dependency checks.
 #   test_{version} - Runs all tests, against a specified mongod version.
 #   test_vscode - Run the vscode extension integration tests.
@@ -488,7 +527,17 @@ functions:
 #   release_publish - Publishes the npm packages and uploads the tarballs.
 #   pkg_test_* - Run tests on the release packages
 tasks:
+  - name: compile_ts
+    commands:
+      - func: checkout
+      - func: compile_ts
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_14) %>"
+
   - name: check
+    depends_on:
+      - name: compile_ts
+        variant: linux
     commands:
       - func: checkout
       - func: install
@@ -506,6 +555,9 @@ tasks:
        for (const { shortName: mShort, versionSpec: mVersion } of MONGODB_VERSIONS) { %>
   - name: test_<% const id = `m${mShort}_n${nShort}`; ALL_UNIT_TESTS.push(id); out(id) %>
     tags: <% out(["unit-test", ...(mShort === 'latest' ? ["mlatest"] : [])]) %>
+    depends_on:
+      - name: compile_ts
+        variant: linux
     commands:
       - func: checkout
       - func: install
@@ -523,6 +575,9 @@ tasks:
   ###
   - name: test_vscode
     tags: ["extra-integration-test"]
+    depends_on:
+      - name: compile_ts
+        variant: linux
     commands:
       - func: checkout
       - func: install
@@ -533,6 +588,9 @@ tasks:
           node_js_version: "<% out(NODE_JS_VERSION_12) %>"
   - name: test_connectivity
     tags: ["extra-integration-test"]
+    depends_on:
+      - name: compile_ts
+        variant: linux
     commands:
       - func: checkout
       - func: install
@@ -541,6 +599,9 @@ tasks:
       - func: test_connectivity
   - name: test_apistrict
     tags: ["extra-integration-test"]
+    depends_on:
+      - name: compile_ts
+        variant: linux
     commands:
       - func: checkout
       - func: install
@@ -552,6 +613,9 @@ tasks:
           mongosh_server_test_version: "latest-alpha"
           mongosh_test_force_api_strict: "1"
   - name: compile_artifact
+    depends_on:
+      - name: compile_ts
+        variant: linux
     commands:
       - func: checkout
       - func: install
@@ -677,6 +741,9 @@ tasks:
     tags: ["publish"]
     git_tag_only: true
     exec_timeout_secs: 7200
+    depends_on:
+      - name: compile_ts
+        variant: linux
     commands:
       - func: checkout
       - func: install
@@ -720,6 +787,7 @@ buildvariants:
     display_name: "Ubuntu 18.04 x64"
     run_on: ubuntu1804-small
     tasks:
+      - name: compile_ts
       - name: check
       <% for (const test of ALL_UNIT_TESTS) { %>
       - name: test_<% out(test) %>

.evergreen/evergreen.yml.in

@@ -28,7 +28,7 @@ else
   fi
 fi
 
-. "$BASEDIR/.setup_env"
+. "$BASEDIR/setup-env.sh"
 
 # We need the build package for various tasks, and can bootstrap the cli-repl
 # package on all hosts, including dependencies.
@@ -39,6 +39,8 @@ fi
 # to installing with --ignore-scripts (i.e. do not attempt to build addons)
 # and only do the TypeScript compilation step, which is sufficient for the
 # executable compilation step.
-npm --unsafe-perm=true run bootstrap-ci -- --scope @mongosh/build
-npm --unsafe-perm=true run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies || \
-  (npm --unsafe-perm=true run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies --ignore-scripts && npm --unsafe-perm=true run compile-cli)
+npm ci --verbose
+
+npm run bootstrap-ci -- --scope @mongosh/build --ignore-prepublish
+(npm run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies --ignore-prepublish && test -e packages/service-provider-server/node_modules/mongodb-client-encryption) || \
+  npm run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies --ignore-prepublish --ignore-scripts

.evergreen/.install_node renamed to .evergreen/install-node.sh

@@ -8,7 +8,7 @@ cd $(pwd)
 export NODE_JS_VERSION=${NODE_JS_VERSION}
 export ARTIFACT_URL_FILE="$PWD/artifact-url.txt"
 
-source .evergreen/.setup_env
+source .evergreen/setup-env.sh
 tar xvzf dist.tgz
 
 if [ "$(uname)" == Linux ]; then

.evergreen/package-and-upload-artifact.sh

@@ -2,7 +2,7 @@
 set -e
 export NODE_JS_VERSION=${NODE_JS_VERSION}
 
-source .evergreen/.setup_env
+source .evergreen/setup-env.sh
 tar xvzf dist.tgz
 dist/mongosh --version
 

.evergreen/run-e2e-tests.sh

@@ -34,6 +34,11 @@ if [ "$OS" == "Windows_NT" ]; then
   export EVERGREEN_EXPANSIONS_PATH="$(cygpath -w "$EVERGREEN_EXPANSIONS_PATH")"
 fi
 
+# On RHEL hosts, we run as root for some reason
+if [ `uname` = Linux ]; then
+  export npm_config_unsafe_perm=true
+fi
+
 echo "Full path:"
 echo $PATH
 

.evergreen/.setup_env renamed to .evergreen/setup-env.sh

@@ -21,4 +21,5 @@ tmp/expansions.yaml
 .evergreen/mongodb
 tmp/
 dist.tgz
+compiled-ts.tgz
 mongocryptd.pid

.gitignore

@@ -12,8 +12,7 @@
     "bootstrap-with-chromium": "lerna bootstrap --concurrency=1",
     "prebootstrap": "npm install",
     "bootstrap": "cross-env PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 lerna bootstrap --ignore-prepublish --concurrency=1",
-    "prebootstrap-ci": "npm ci",
-    "bootstrap-ci": "lerna bootstrap --ci --concurrency=1",
+    "bootstrap-ci": "lerna bootstrap --ci",
     "clean": "lerna clean -y && rm -Rf node_modules",
     "check": "lerna run check --since HEAD --exclude-dependents",
     "check-ci": "lerna run check",

package.json

@@ -6,7 +6,7 @@ import { once } from 'events';
 import { startTestServer } from '../../../../../testing/integration-testing-hooks';
 
 describe('java-shell tests', function() {
-  this.timeout(600_000);
+  this.timeout(1_000_000);
   const testServer = startTestServer('shared');
   const packageRoot = path.resolve(__dirname, '..', '..', '..') + '/';