Support retrying failed tests from previous build (#15)

Author: yqiu24

javascript/dist/queue/BaseRunner.d.ts

@@ -332,12 +332,14 @@ export declare class BaseRunner {
     recordFailedTest(testName: string, testSuite: string): Promise<void>;
     recordPassingTest(testName: string, testSuite: string): Promise<void>;
     getFailedTests(): Promise<string>;
+    getFailedTestNamesFromPreviousBuild(): Promise<string[]>;
     waitForMaster(): Promise<void>;
     getMasterStatus(): Promise<string | null>;
     isInitialized(): Promise<boolean>;
     size(): Promise<number>;
     progress(): Promise<number>;
     toArray(): Promise<string[]>;
     key(...args: string[]): string;
+    retriedBuildKey(...args: string[]): string;
     private createRedisScripts;
 }

javascript/dist/queue/BaseRunner.js

@@ -58,6 +58,15 @@ class BaseRunner {
         const failures = Object.values(failedTests).map(test => JSON.parse(test));
         return JSON.stringify(failures);
     }
+    async getFailedTestNamesFromPreviousBuild() {
+        const previousBuildId = this.config.retriedBuildId;
+        if (!previousBuildId) {
+            return [];
+        }
+        const failedTests = await this.client.hGetAll(this.retriedBuildKey('error-reports'));
+        const failures = Object.values(failedTests).map(test => JSON.parse(test).test_name);
+        return failures;
+    }
     async waitForMaster() {
         if (this.isMaster) {
             return;
@@ -115,6 +124,13 @@ class BaseRunner {
         const uniqueID = this.config.namespace ? `${this.config.namespace}:#${this.config.buildId}` : this.config.buildId;
         return ['build', uniqueID, ...args].join(':');
     }
+    retriedBuildKey(...args) {
+        if (!Array.isArray(args)) {
+            args = [args];
+        }
+        const uniqueID = this.config.namespace ? `${this.config.namespace}:#${this.config.retriedBuildId}` : this.config.retriedBuildId;
+        return ['build', uniqueID, ...args].join(':');
+    }
     createRedisScripts() {
         return {
             acknowledge: (0, redis_1.defineScript)({

javascript/dist/queue/Configuration.d.ts

@@ -11,6 +11,7 @@ export type InitConfig = {
     inactiveWorkersTimeout?: number;
     namespace?: string;
     failureFile?: string;
+    retriedBuildId?: string;
 };
 export declare class Configuration {
     buildId: string;
@@ -25,7 +26,8 @@ export declare class Configuration {
     inactiveWorkersTimeout: number;
     namespace?: string;
     failureFile?: string;
-    constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, }: {
+    retriedBuildId?: string;
+    constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, retriedBuildId, }: {
         buildId: string;
         workerId: string;
         seed?: string;
@@ -38,6 +40,7 @@ export declare class Configuration {
         inactiveWorkersTimeout?: number;
         namespace?: string;
         failureFile?: string;
+        retriedBuildId?: string;
     });
     static fromEnv(): any;
     globalMaxRequeues(testCount: number): number;

javascript/dist/queue/Configuration.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Configuration = void 0;
 class Configuration {
-    constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, }) {
+    constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, retriedBuildId, }) {
         this.buildId = buildId;
         this.workerId = workerId;
         this.seed = seed;
@@ -15,6 +15,7 @@ class Configuration {
         this.inactiveWorkersTimeout = inactiveWorkersTimeout ?? this.timeout;
         this.namespace = namespace;
         this.failureFile = failureFile;
+        this.retriedBuildId = retriedBuildId;
     }
     static fromEnv() {
         const buildId = process.env['CIRCLE_BUILD_URL'] ||

javascript/dist/queue/Worker.js

@@ -61,10 +61,18 @@ class Worker extends BaseRunner_1.BaseRunner {
         await this.client.release(this.key('running'), this.key('worker', this.config.workerId, 'queue'), this.key('owners'));
     }
     async populate(tests, seed) {
-        if (seed !== undefined) {
-            tests = (0, utils_1.shuffleArray)(tests, seed);
+        if (this.config.retriedBuildId) {
+            console.log(`[ci-queue] Retrying failed tests for build ${this.config.retriedBuildId}`);
+            const failedTests = await this.getFailedTestNamesFromPreviousBuild();
+            console.log(`[ci-queue] Failed tests: ${failedTests}`);
+            await this.push(failedTests);
+        }
+        else {
+            if (seed !== undefined) {
+                tests = (0, utils_1.shuffleArray)(tests, seed);
+            }
+            await this.push(tests);
         }
-        await this.push(tests);
     }
     shutdown() {
         this.shutdownRequired = true;

javascript/src/queue/BaseRunner.ts

@@ -75,6 +75,18 @@ export class BaseRunner {
     return JSON.stringify(failures);
   }
 
+  async getFailedTestNamesFromPreviousBuild(): Promise<string[]> {
+    const previousBuildId = this.config.retriedBuildId;
+    if (!previousBuildId) {
+      return [];
+    }
+
+    const failedTests = await this.client.hGetAll(this.retriedBuildKey('error-reports'));
+    const failures = Object.values(failedTests).map(test => JSON.parse(test).test_name);
+
+    return failures;
+  }
+
   async waitForMaster(): Promise<void> {
     if (this.isMaster) {
       return;
@@ -142,6 +154,14 @@ export class BaseRunner {
     return ['build', uniqueID, ...args].join(':');
   }
 
+  retriedBuildKey(...args: string[]): string {
+    if (!Array.isArray(args)) {
+      args = [args];
+    }
+    const uniqueID = this.config.namespace ? `${this.config.namespace}:#${this.config.retriedBuildId}` : this.config.retriedBuildId;
+    return ['build', uniqueID, ...args].join(':');
+  }
+
   private createRedisScripts() {
     return {
       acknowledge: defineScript({

javascript/src/queue/Configuration.ts

@@ -11,6 +11,7 @@ export type InitConfig = {
   inactiveWorkersTimeout?: number;
   namespace?: string;
   failureFile?: string;
+  retriedBuildId?: string;
 };
 
 export class Configuration {
@@ -26,6 +27,7 @@ export class Configuration {
   inactiveWorkersTimeout: number;
   namespace?: string;
   failureFile?: string;
+  retriedBuildId?: string;
   constructor({
     buildId,
     workerId,
@@ -39,6 +41,7 @@ export class Configuration {
     inactiveWorkersTimeout,
     namespace,
     failureFile,
+    retriedBuildId,
   }: {
     buildId: string;
     workerId: string;
@@ -52,6 +55,7 @@ export class Configuration {
     inactiveWorkersTimeout?: number;
     namespace?: string;
     failureFile?: string;
+    retriedBuildId?: string;
   }) {
     this.buildId = buildId;
     this.workerId = workerId;
@@ -65,6 +69,7 @@ export class Configuration {
     this.inactiveWorkersTimeout = inactiveWorkersTimeout ?? this.timeout;
     this.namespace = namespace;
     this.failureFile = failureFile;
+    this.retriedBuildId = retriedBuildId;
   }
 
   static fromEnv() {

javascript/src/queue/Worker.ts

@@ -92,10 +92,17 @@ export class Worker extends BaseRunner {
   }
 
   async populate(tests: string[], seed?: number) {
-    if (seed !== undefined) {
-      tests = shuffleArray(tests, seed);
-    }
-    await this.push(tests);
+    if (this.config.retriedBuildId) {
+      console.log(`[ci-queue] Retrying failed tests for build ${this.config.retriedBuildId}`);
+      const failedTests = await this.getFailedTestNamesFromPreviousBuild();
+      console.log(`[ci-queue] Failed tests: ${failedTests}`);
+      await this.push(failedTests);
+    } else {
+      if (seed !== undefined) {
+        tests = shuffleArray(tests, seed);
+      }
+      await this.push(tests);
+    } 
   }
 
   shutdown() {