Address flaky integration tests with retries (google-gemini#4604)

Author: skeshive

integration-tests/run-tests.js

@@ -61,6 +61,7 @@ async function main() {
     console.log(`\tFound test file: ${testFileName}`);
   }
 
+  const MAX_RETRIES = 3;
   let allTestsPassed = true;
 
   for (const testFile of testFiles) {
@@ -72,63 +73,97 @@ async function main() {
       `------------- Running test file: ${testFileName} ------------------------------`,
     );
 
-    const nodeArgs = ['--test'];
-    if (verbose) {
-      nodeArgs.push('--test-reporter=spec');
-    }
-    nodeArgs.push(testFile);
-
-    const child = spawn('node', nodeArgs, {
-      stdio: 'pipe',
-      env: {
-        ...process.env,
-        GEMINI_CLI_INTEGRATION_TEST: 'true',
-        INTEGRATION_TEST_FILE_DIR: testFileDir,
-        KEEP_OUTPUT: keepOutput.toString(),
-        VERBOSE: verbose.toString(),
-        TEST_FILE_NAME: testFileName,
-      },
-    });
-
-    let outputStream;
-    if (keepOutput) {
-      const outputFile = join(testFileDir, 'output.log');
-      outputStream = createWriteStream(outputFile);
-      console.log(`Output for ${testFileName} written to: ${outputFile}`);
-    }
-
-    child.stdout.on('data', (data) => {
-      if (verbose) {
-        process.stdout.write(data);
-      }
-      if (outputStream) {
-        outputStream.write(data);
+    let attempt = 0;
+    let testFilePassed = false;
+    let lastStdout = [];
+    let lastStderr = [];
+
+    while (attempt < MAX_RETRIES && !testFilePassed) {
+      attempt++;
+      if (attempt > 1) {
+        console.log(
+          `--- Retrying ${testFileName} (attempt ${attempt} of ${MAX_RETRIES}) ---`,
+        );
       }
-    });
 
-    child.stderr.on('data', (data) => {
+      const nodeArgs = ['--test'];
       if (verbose) {
-        process.stderr.write(data);
+        nodeArgs.push('--test-reporter=spec');
       }
-      if (outputStream) {
-        outputStream.write(data);
+      nodeArgs.push(testFile);
+
+      const child = spawn('node', nodeArgs, {
+        stdio: 'pipe',
+        env: {
+          ...process.env,
+          GEMINI_CLI_INTEGRATION_TEST: 'true',
+          INTEGRATION_TEST_FILE_DIR: testFileDir,
+          KEEP_OUTPUT: keepOutput.toString(),
+          VERBOSE: verbose.toString(),
+          TEST_FILE_NAME: testFileName,
+        },
+      });
+
+      let outputStream;
+      if (keepOutput) {
+        const outputFile = join(testFileDir, `output-attempt-${attempt}.log`);
+        outputStream = createWriteStream(outputFile);
+        console.log(`Output for ${testFileName} written to: ${outputFile}`);
       }
-    });
 
-    const exitCode = await new Promise((resolve) => {
-      child.on('close', (code) => {
+      const stdout = [];
+      const stderr = [];
+
+      child.stdout.on('data', (data) => {
+        if (verbose) {
+          process.stdout.write(data);
+        } else {
+          stdout.push(data);
+        }
         if (outputStream) {
-          outputStream.end(() => {
-            resolve(code);
-          });
+          outputStream.write(data);
+        }
+      });
+
+      child.stderr.on('data', (data) => {
+        if (verbose) {
+          process.stderr.write(data);
         } else {
-          resolve(code);
+          stderr.push(data);
+        }
+        if (outputStream) {
+          outputStream.write(data);
         }
       });
-    });
 
-    if (exitCode !== 0) {
-      console.error(`Test file failed: ${testFileName}`);
+      const exitCode = await new Promise((resolve) => {
+        child.on('close', (code) => {
+          if (outputStream) {
+            outputStream.end(() => {
+              resolve(code);
+            });
+          } else {
+            resolve(code);
+          }
+        });
+      });
+
+      if (exitCode === 0) {
+        testFilePassed = true;
+      } else {
+        lastStdout = stdout;
+        lastStderr = stderr;
+      }
+    }
+
+    if (!testFilePassed) {
+      console.error(
+        `Test file failed after ${MAX_RETRIES} attempts: ${testFileName}`,
+      );
+      if (!verbose) {
+        process.stdout.write(Buffer.concat(lastStdout).toString('utf8'));
+        process.stderr.write(Buffer.concat(lastStderr).toString('utf8'));
+      }
       allTestsPassed = false;
     }
   }