chore(ci): improve log file collection logic (#2164)

Author: addaleax

.evergreen.yml

@@ -1,6 +1,12 @@
 
 exec_timeout_secs: 10800
 
+pre:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |
+        rm -rf "$HOME/.mongodb/mongosh"
 post_error_fails_task: true
 post:
   - command: shell.exec

.evergreen/evergreen.yml.in

@@ -68,6 +68,12 @@ const { RELEASE_PACKAGE_MATRIX } = require('../config/release-package-matrix');
 %>
 exec_timeout_secs: 10800
 
+pre:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |
+        rm -rf "$HOME/.mongodb/mongosh"
 post_error_fails_task: true
 post:
   - command: shell.exec

.evergreen/setup-env.sh

@@ -4,6 +4,7 @@ set -x
 export BASEDIR="$PWD/.evergreen"
 export PATH="/cygdrive/c/python/Python311/Scripts:/cygdrive/c/python/Python311:/cygdrive/c/Python311/Scripts:/cygdrive/c/Python311:/opt/python/3.6/bin:$BASEDIR/mingit/cmd:$BASEDIR/mingit/mingw64/libexec/git-core:$BASEDIR/git-2:$BASEDIR/npm-10/node_modules/.bin:$BASEDIR/node-v$NODE_JS_VERSION-win-x64:/opt/java/jdk16/bin:/opt/chefdk/gitbin:/cygdrive/c/cmake/bin:/opt/mongodbtoolchain/v3/bin:$PATH"
 
+export MONGOSH_TEST_ONLY_MAX_LOG_FILE_COUNT=100000
 export IS_MONGOSH_EVERGREEN_CI=1
 export DEBUG="mongodb*,$DEBUG"
 

packages/cli-repl/src/cli-repl.ts

@@ -192,7 +192,9 @@ export class CliRepl implements MongoshIOProvider {
     this.logManager = new MongoLogManager({
       directory: this.shellHomeDirectory.localPath('.'),
       retentionDays: 30,
-      maxLogFileCount: 100,
+      maxLogFileCount: +(
+        process.env.MONGOSH_TEST_ONLY_MAX_LOG_FILE_COUNT || 100
+      ),
       onerror: (err: Error) => this.bus.emit('mongosh:error', err, 'log'),
       onwarn: (err: Error, path: string) =>
         this.warnAboutInaccessibleFile(err, path),

packages/e2e-tests/test/e2e-analytics.spec.ts

@@ -13,7 +13,7 @@ describe('e2e Analytics Node', function () {
     { args: ['--replSet', replSetName] }
   );
 
-  after(TestShell.cleanup);
+  afterEach(TestShell.cleanup);
 
   before(async function () {
     if (process.env.MONGOSH_TEST_FORCE_API_STRICT) {

packages/e2e-tests/test/test-shell.ts

@@ -115,16 +115,31 @@ export class TestShell {
     await Promise.all(exitPromises);
   }
 
+  debugInformation() {
+    return {
+      pid: this.process.pid,
+      output: this.output,
+      rawOutput: this.rawOutput,
+      exitCode: this.process.exitCode,
+      signal: this.process.signalCode,
+    };
+  }
+
+  static async cleanupAfterAll(): Promise<void> {
+    let foundOpenShells = false;
+    for (const shell of TestShell._openShells) {
+      foundOpenShells = true;
+      console.error(shell.debugInformation());
+    }
+    await TestShell.killall();
+    if (foundOpenShells)
+      throw new Error('Open shells at end of test discovered!');
+  }
+
   static async cleanup(this: Mocha.Context): Promise<void> {
     if (this.currentTest?.state === 'failed') {
       for (const shell of TestShell._openShells) {
-        console.error({
-          pid: shell.process.pid,
-          output: shell.output,
-          rawOutput: shell.rawOutput,
-          exitCode: shell.process.exitCode,
-          signal: shell.process.signalCode,
-        });
+        console.error(shell.debugInformation());
       }
     }
     await TestShell.killall();
@@ -321,3 +336,6 @@ export class TestShell {
     return match.groups!.logId;
   }
 }
+
+// If any shells remain at the end of the script, print their full output
+globalThis?.after('ensure no hanging shells', TestShell.cleanupAfterAll);