Skip to content

Commit 8b07033

Browse files
kunal-figmakunal-figma
authored
Merge pull request #27 from figma/kunal/dynamic-test-timeout
Support dynamic test timeouts
2 parents 2992b0b + 3b18548 commit 8b07033

File tree

14 files changed

+130
-23
lines changed

14 files changed

+130
-23
lines changed

‎javascript/dist/queue/BaseRunner.d.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,19 +310,20 @@ export declare class BaseRunner {
310310
reserve: {
311311
NUMBER_OF_KEYS: number;
312312
SCRIPT: string;
313-
transformArguments(this: void, queueKey: string, setKey: string, processedKey: string, workerQueueKey: string, ownersKey: string, currentTime: number): string[];
313+
transformArguments(this: void, queueKey: string, setKey: string, processedKey: string, workerQueueKey: string, ownersKey: string, testGroupTimeoutKey: string, currentTime: number, useDynamicDeadline: boolean): string[];
314314
transformReply(this: void, reply: string | null | undefined): string | null | undefined;
315315
} & import("@redis/client/dist/lib/lua-script").SHA1;
316316
reserveLost: {
317317
NUMBER_OF_KEYS: number;
318318
SCRIPT: string;
319-
transformArguments(this: void, setKey: string, completedKey: string, workerQueueKey: string, ownersKey: string, currentTime: number, timeout: number): string[];
319+
transformArguments(this: void, setKey: string, completedKey: string, workerQueueKey: string, ownersKey: string, testGroupTimeoutKey: string, currentTime: number, timeout: number, useDynamicDeadline: boolean): string[];
320320
transformReply(this: void, reply: string | null | undefined): string | null | undefined;
321321
} & import("@redis/client/dist/lib/lua-script").SHA1;
322322
}>;
323323
totalTestCount: number;
324324
private queueInitialized?;
325325
constructor(redisUrl: string, config: Configuration);
326+
useDynamicDeadline(): boolean;
326327
connect(): Promise<void>;
327328
disconnect(): Promise<void>;
328329
isExhausted(): Promise<boolean>;
@@ -339,6 +340,7 @@ export declare class BaseRunner {
339340
size(): Promise<number>;
340341
progress(): Promise<number>;
341342
toArray(): Promise<string[]>;
343+
testGroupTimeoutKey(): string;
342344
key(...args: string[]): string;
343345
retriedBuildKey(...args: string[]): string;
344346
private createRedisScripts;

‎javascript/dist/queue/BaseRunner.js‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ class BaseRunner {
1212
this.client = (0, redis_1.createClient)({ url: redisUrl, scripts: this.createRedisScripts() });
1313
this.config = config;
1414
}
15+
useDynamicDeadline() {
16+
return Boolean(this.config.useDynamicDeadline);
17+
}
1518
async connect() {
1619
try {
1720
await this.client.connect();
@@ -127,6 +130,9 @@ class BaseRunner {
127130
.flatMap((t) => t)
128131
.reverse();
129132
}
133+
testGroupTimeoutKey() {
134+
return this.key('test_group_timeouts');
135+
}
130136
key(...args) {
131137
if (!Array.isArray(args)) {
132138
args = [args];
@@ -176,33 +182,37 @@ class BaseRunner {
176182
},
177183
}),
178184
reserve: (0, redis_1.defineScript)({
179-
NUMBER_OF_KEYS: 5,
185+
NUMBER_OF_KEYS: 6,
180186
SCRIPT: (0, node_fs_1.readFileSync)(`${__dirname}/../../../redis/reserve.lua`).toString(),
181-
transformArguments(queueKey, setKey, processedKey, workerQueueKey, ownersKey, currentTime) {
187+
transformArguments(queueKey, setKey, processedKey, workerQueueKey, ownersKey, testGroupTimeoutKey, currentTime, useDynamicDeadline) {
182188
return [
183189
queueKey,
184190
setKey,
185191
processedKey,
186192
workerQueueKey,
187193
ownersKey,
194+
testGroupTimeoutKey,
188195
currentTime.toString(),
196+
useDynamicDeadline.toString(),
189197
];
190198
},
191199
transformReply(reply) {
192200
return reply;
193201
},
194202
}),
195203
reserveLost: (0, redis_1.defineScript)({
196-
NUMBER_OF_KEYS: 4,
204+
NUMBER_OF_KEYS: 5,
197205
SCRIPT: (0, node_fs_1.readFileSync)(`${__dirname}/../../../redis/reserve_lost.lua`).toString(),
198-
transformArguments(setKey, completedKey, workerQueueKey, ownersKey, currentTime, timeout) {
206+
transformArguments(setKey, completedKey, workerQueueKey, ownersKey, testGroupTimeoutKey, currentTime, timeout, useDynamicDeadline) {
199207
return [
200208
setKey,
201209
completedKey,
202210
workerQueueKey,
203211
ownersKey,
212+
testGroupTimeoutKey,
204213
currentTime.toString(),
205214
timeout.toString(),
215+
useDynamicDeadline.toString(),
206216
];
207217
},
208218
transformReply(reply) {

‎javascript/dist/queue/Configuration.d.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ export declare class Configuration {
2727
namespace?: string;
2828
failureFile?: string;
2929
retriedBuildId?: string;
30-
constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, retriedBuildId, }: {
30+
useDynamicDeadline?: boolean;
31+
constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, retriedBuildId, useDynamicDeadline, }: {
3132
buildId: string;
3233
workerId: string;
3334
seed?: string;
@@ -41,6 +42,7 @@ export declare class Configuration {
4142
namespace?: string;
4243
failureFile?: string;
4344
retriedBuildId?: string;
45+
useDynamicDeadline?: boolean;
4446
});
4547
static fromEnv(): any;
4648
globalMaxRequeues(testCount: number): number;

‎javascript/dist/queue/Configuration.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
Object.defineProperty(exports, "__esModule", { value: true });
33
exports.Configuration = void 0;
44
class Configuration {
5-
constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, retriedBuildId, }) {
5+
constructor({ buildId, workerId, seed, redisTTL, maxRequeues, requeueTolerance, maxTestsAllowedToFail, timeout, reportTimeout, inactiveWorkersTimeout, namespace, failureFile, retriedBuildId, useDynamicDeadline, }) {
66
this.buildId = buildId;
77
this.workerId = workerId;
88
this.seed = seed;
@@ -16,6 +16,7 @@ class Configuration {
1616
this.namespace = namespace;
1717
this.failureFile = failureFile;
1818
this.retriedBuildId = retriedBuildId;
19+
this.useDynamicDeadline = useDynamicDeadline ?? false;
1920
}
2021
static fromEnv() {
2122
const buildId = process.env['CIRCLE_BUILD_URL'] ||

‎javascript/dist/queue/Test.d.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
export interface TestSpec {
2+
name: string;
3+
timeout: number;
4+
}

‎javascript/dist/queue/Test.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
"use strict";
2+
Object.defineProperty(exports, "__esModule", { value: true });

‎javascript/dist/queue/Worker.d.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { BaseRunner } from './BaseRunner';
22
import { Configuration } from './Configuration';
3+
import { TestSpec } from './Test';
34
export declare class Worker extends BaseRunner {
45
private shutdownRequired;
56
private currentlyReservedTest;
@@ -9,7 +10,7 @@ export declare class Worker extends BaseRunner {
910
acknowledge(test: string): Promise<boolean>;
1011
requeue(test: string, offset?: number): Promise<boolean>;
1112
release(): Promise<void>;
12-
populate(tests: string[], seed?: number): Promise<boolean>;
13+
populate(tests: TestSpec[], seed?: number): Promise<boolean>;
1314
shutdown(): void;
1415
private throwOnMismatchingTest;
1516
private reserveTest;

‎javascript/dist/queue/Worker.js‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,18 @@ class Worker extends BaseRunner_1.BaseRunner {
7474
return false;
7575
}
7676
console.log(`[ci-queue] Failed test groups: ${failedTestGroups}`);
77-
await this.push(failedTestGroups);
77+
const failedTestGroupTimeouts = [];
78+
for (const failedTestGroup of failedTestGroups) {
79+
const testSpec = tests.find((test) => test.name === failedTestGroup);
80+
if (testSpec) {
81+
failedTestGroupTimeouts.push(testSpec);
82+
}
83+
else {
84+
console.log("Failed to find timeout for test group", failedTestGroup, ". defaulting to", this.config.timeout);
85+
failedTestGroupTimeouts.push({ name: failedTestGroup, timeout: this.config.timeout });
86+
}
87+
}
88+
await this.push(failedTestGroupTimeouts);
7889
return true;
7990
}
8091
console.log(`[ci-queue] Populating tests`);
@@ -105,24 +116,31 @@ class Worker extends BaseRunner_1.BaseRunner {
105116
return reservedTest;
106117
}
107118
async tryToReserveTest() {
108-
return await this.client.reserve(this.key('queue'), this.key('running'), this.key('processed'), this.key('worker', this.config.workerId, 'queue'), this.key('owners'), Date.now() / 1000);
119+
return await this.client.reserve(this.key('queue'), this.key('running'), this.key('processed'), this.key('worker', this.config.workerId, 'queue'), this.key('owners'), this.testGroupTimeoutKey(), Date.now() / 1000, this.useDynamicDeadline());
109120
}
110121
async tryToReserveLostTest() {
111-
const lostTest = await this.client.reserveLost(this.key('running'), this.key('completed'), this.key('worker', this.config.workerId, 'queue'), this.key('owners'), Date.now() / 1000, this.config.timeout);
122+
const lostTest = await this.client.reserveLost(this.key('running'), this.key('completed'), this.key('worker', this.config.workerId, 'queue'), this.key('owners'), this.testGroupTimeoutKey(), Date.now() / 1000, this.config.timeout, this.useDynamicDeadline());
112123
return lostTest;
113124
}
114-
async push(tests) {
125+
async push(testSpecs) {
126+
const tests = testSpecs.map((testSpec) => testSpec.name);
127+
const testTimeoutMap = testSpecs.reduce((acc, testSpec) => {
128+
acc[testSpec.name] = testSpec.timeout;
129+
return acc;
130+
}, {});
115131
this.totalTestCount = tests.length;
116132
this.isMaster = await this.client.setNX(this.key('master-status'), 'setup');
117133
if (this.isMaster) {
118134
await this.client
119135
.multi()
120136
.lPush(this.key('queue'), tests)
137+
.hSet(this.testGroupTimeoutKey(), testTimeoutMap)
121138
.set(this.key('total'), this.totalTestCount)
122139
.set(this.key('master-status'), 'ready')
123140
.expire(this.key('queue'), this.config.redisTTL)
124141
.expire(this.key('total'), this.config.redisTTL)
125142
.expire(this.key('master-status'), this.config.redisTTL)
143+
.expire(this.testGroupTimeoutKey(), this.config.redisTTL)
126144
.exec();
127145
}
128146
await this.client.sAdd(this.key('workers'), [this.config.workerId]);

‎javascript/src/queue/BaseRunner.ts‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ export class BaseRunner {
1616
this.config = config;
1717
}
1818

19+
useDynamicDeadline(): boolean {
20+
return Boolean(this.config.useDynamicDeadline)
21+
}
22+
1923
async connect() {
2024
try {
2125
await this.client.connect();
@@ -58,7 +62,7 @@ export class BaseRunner {
5862
Buffer.from(fullTestName).toString('binary'),
5963
Buffer.from(payload).toString('binary')
6064
);
61-
65+
6266
await this.client.expire(this.key('error-reports'), this.config.redisTTL);
6367
console.log(`[ci-queue] Incrementing failed test count for ${testName}`);
6468
await this.client.incr(this.key('test_failed_count'));
@@ -84,7 +88,7 @@ export class BaseRunner {
8488
try {
8589
const failedTests = await this.client.hGetAll(this.retriedBuildKey('error-reports'));
8690
console.log(`[ci-queue] Failed tests`, failedTests);
87-
const failedTestGroups = Object.keys(failedTests).length > 0
91+
const failedTestGroups = Object.keys(failedTests).length > 0
8892
? Object.values(failedTests).map(test => JSON.parse(test).test_group)
8993
: [];
9094
return failedTestGroups;
@@ -154,6 +158,10 @@ export class BaseRunner {
154158
.reverse() as string[];
155159
}
156160

161+
testGroupTimeoutKey() {
162+
return this.key('test_group_timeouts')
163+
}
164+
157165
key(...args: string[]): string {
158166
if (!Array.isArray(args)) {
159167
args = [args];
@@ -222,47 +230,55 @@ export class BaseRunner {
222230
}),
223231

224232
reserve: defineScript({
225-
NUMBER_OF_KEYS: 5,
233+
NUMBER_OF_KEYS: 6,
226234
SCRIPT: readFileSync(`${__dirname}/../../../redis/reserve.lua`).toString(),
227235
transformArguments(
228236
queueKey: string,
229237
setKey: string,
230238
processedKey: string,
231239
workerQueueKey: string,
232240
ownersKey: string,
241+
testGroupTimeoutKey: string,
233242
currentTime: number,
243+
useDynamicDeadline: boolean,
234244
) {
235245
return [
236246
queueKey,
237247
setKey,
238248
processedKey,
239249
workerQueueKey,
240250
ownersKey,
251+
testGroupTimeoutKey,
241252
currentTime.toString(),
253+
useDynamicDeadline.toString(),
242254
];
243255
},
244256
transformReply(reply: string | null | undefined) {
245257
return reply;
246258
},
247259
}),
248260
reserveLost: defineScript({
249-
NUMBER_OF_KEYS: 4,
261+
NUMBER_OF_KEYS: 5,
250262
SCRIPT: readFileSync(`${__dirname}/../../../redis/reserve_lost.lua`).toString(),
251263
transformArguments(
252264
setKey: string,
253265
completedKey: string,
254266
workerQueueKey: string,
255267
ownersKey: string,
268+
testGroupTimeoutKey: string,
256269
currentTime: number,
257270
timeout: number,
271+
useDynamicDeadline: boolean
258272
) {
259273
return [
260274
setKey,
261275
completedKey,
262276
workerQueueKey,
263277
ownersKey,
278+
testGroupTimeoutKey,
264279
currentTime.toString(),
265280
timeout.toString(),
281+
useDynamicDeadline.toString(),
266282
];
267283
},
268284
transformReply(reply: string | null | undefined) {

‎javascript/src/queue/Configuration.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ export class Configuration {
2828
namespace?: string;
2929
failureFile?: string;
3030
retriedBuildId?: string;
31+
useDynamicDeadline?: boolean;
3132
constructor({
3233
buildId,
3334
workerId,
@@ -42,6 +43,7 @@ export class Configuration {
4243
namespace,
4344
failureFile,
4445
retriedBuildId,
46+
useDynamicDeadline,
4547
}: {
4648
buildId: string;
4749
workerId: string;
@@ -56,6 +58,7 @@ export class Configuration {
5658
namespace?: string;
5759
failureFile?: string;
5860
retriedBuildId?: string;
61+
useDynamicDeadline?: boolean;
5962
}) {
6063
this.buildId = buildId;
6164
this.workerId = workerId;
@@ -70,6 +73,7 @@ export class Configuration {
7073
this.namespace = namespace;
7174
this.failureFile = failureFile;
7275
this.retriedBuildId = retriedBuildId;
76+
this.useDynamicDeadline = useDynamicDeadline ?? false
7377
}
7478

7579
static fromEnv() {

0 commit comments

Comments
 (0)