Skip to content

Commit b06835a

Browse files
timmywiltimmywil
committed
feat: support run config for generating multiple runs
1 parent a075efc commit b06835a

File tree

7 files changed

+90
-57
lines changed

7 files changed

+90
-57
lines changed

‎bin/command.js‎

Lines changed: 47 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { resolve } from "node:path";
66
import { browsers } from "../flags/browsers.js";
77
import { getPlan, listBrowsers, stopWorkers } from "../browserstack/api.js";
88
import { buildBrowserFromString } from "../browserstack/buildBrowserFromString.js";
9-
import { run } from "../run.js";
9+
import { run as runTests } from "../run.js";
1010
import readYAML from "../lib/readYAML.js";
1111
import { createTestServer } from "../createTestServer.js";
1212

@@ -19,6 +19,31 @@ function parseFlags( flags ) {
1919
.map( ( value ) => `${ key }=${ value }` ) );
2020
}
2121

22+
// Get all possible combinations of flag values.
23+
// Example: { "jquery": [ "1.12.4", "3.5.1" ], "jquery-migrate": [ "dev", "min" ] }
24+
// -> [ "jquery=1.12.4&jquery-migrate=dev", "jquery=3.5.1&jquery-migrate=dev",
25+
// "jquery=1.12.4&jquery-migrate=min", "jquery=3.5.1&jquery-migrate=min" ]
26+
function parseRuns( runs ) {
27+
const results = [];
28+
29+
function dfs( run, keys, startIndex ) {
30+
if ( startIndex === keys.length ) {
31+
if ( run.length > 0 ) {
32+
results.push( run.join( "&" ) );
33+
}
34+
return;
35+
}
36+
const key = keys[ startIndex ];
37+
const values = runs[ key ];
38+
for ( const value of values ) {
39+
dfs( run.concat( `${ key }=${ value }` ), keys, startIndex + 1 );
40+
}
41+
}
42+
43+
dfs( [], Object.keys( runs ?? [] ), 0 );
44+
return results;
45+
}
46+
2247
async function parseMiddleware( config, argv ) {
2348
const middleware = await Promise.all( [
2449
...( config.middleware ?? [] ),
@@ -41,12 +66,15 @@ yargs( process.argv.slice( 2 ) )
4166
yargs.option( "config-file", {
4267
alias: "c",
4368
type: "string",
69+
example: "jtr-config.yml",
4470
description: "Path to a YAML configuration file. " +
45-
"Use this to avoid passing options via the command line."
71+
"Use this to avoid passing options via the command line. " +
72+
"jquery-test-runner will automatically search for jtr.yml or jtr.yaml."
4673
} )
4774
.option( "base-url", {
4875
alias: "u",
4976
type: "string",
77+
example: "/tests/",
5078
description: "Base URL for the test server. " +
5179
"Expected to always start and end with a slash (/). " +
5280
"Defaults to \"/test/\"."
@@ -59,15 +87,17 @@ yargs( process.argv.slice( 2 ) )
5987
} )
6088
.option( "flag", {
6189
alias: "f",
90+
example: "module=core",
6291
type: "array",
6392
description: "Add a universal flag to be added as a query parameter " +
64-
"to the test URL for all test pages."
93+
"to the test URL for all test pages. e.g. --flag module=core"
6594
} )
66-
.option( "isolated-flag", {
67-
alias: "i",
95+
.option( "run", {
6896
type: "array",
69-
description: "Add an isolated flag to be added as a query parameter " +
70-
"to the test URL for each. Each isolated flag creates a new test page."
97+
example: "module=core&esmodules=true",
98+
description: "Reuse the same tunnel and browser by adding more runs with " +
99+
"different flags. Each run is a separate test run. These have the same " +
100+
"format as the --flag option."
71101
} )
72102
.option( "browser", {
73103
alias: "b",
@@ -80,7 +110,7 @@ yargs( process.argv.slice( 2 ) )
80110
"Defaults to Chrome."
81111
} )
82112
.option( "middleware", {
83-
alias: "mw",
113+
alias: "m",
84114
type: "array",
85115
description: "Add middleware to the test server by passing " +
86116
"the path to a module that exports a middleware factory function. " +
@@ -144,27 +174,26 @@ yargs( process.argv.slice( 2 ) )
144174
} );
145175
},
146176
handler: async( { configFile, ...argv } ) => {
147-
console.log( "Running tests..." );
148177
const config = await readYAML( configFile );
149178
const flag = [
150179
...parseFlags( config.flags ),
151180
...( config.flag ?? [] ),
152181
...( argv.flag ?? [] )
153182
];
154-
const isolatedFlag = [
155-
...parseFlags( config.isolatedFlags ),
156-
...( config.isolatedFlag ?? [] ),
157-
...( argv.isolatedFlag ?? [] )
183+
const run = [
184+
...parseRuns( config.runs ),
185+
...( config.run ?? [] ),
186+
...( argv.run ?? [] )
158187
];
159188
const middleware = await parseMiddleware( config, argv );
160189

161-
return run( {
190+
return runTests( {
162191
...config,
163192
testUrl: config.testUrls,
164193
...argv,
165194
flag,
166-
isolatedFlag,
167-
middleware
195+
middleware,
196+
run
168197
} );
169198
}
170199
} )
@@ -196,7 +225,7 @@ yargs( process.argv.slice( 2 ) )
196225
description: "Whether to log requests to the console. Default: false."
197226
} )
198227
.option( "middleware", {
199-
alias: "mw",
228+
alias: "m",
200229
type: "array",
201230
description: "Add middleware to the test server by passing " +
202231
"the path to a module that exports a middleware factory function. " +
@@ -218,7 +247,7 @@ yargs( process.argv.slice( 2 ) )
218247

219248
const port = argv.port ?? config.port ?? DEFAULT_PORT;
220249
return app.listen( { port, host: "0.0.0.0" }, function() {
221-
console.log( `Open tests at http://localhost:${ port }/` );
250+
console.log( `Open tests at http://localhost:${ port }${ baseUrl }` );
222251
} );
223252
}
224253
} )

‎lib/buildTestUrl.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ export function buildTestUrl( {
44
baseUrl,
55
browserstack,
66
flags,
7-
isolatedFlag,
7+
run,
88
jsdom,
99
port,
1010
reportId,
@@ -15,7 +15,7 @@ export function buildTestUrl( {
1515
}
1616

1717
const query = new URLSearchParams();
18-
const allFlags = [ ...flags, ...( isolatedFlag ? [ isolatedFlag ] : [] ) ];
18+
const allFlags = [ ...flags, ...( run ? run.split( "&" ) : [] ) ];
1919
for ( const flag of allFlags ) {
2020
const [ key, value ] = flag.split( "=" );
2121

‎lib/readYAML.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Read a yaml configuration file using the `readYAML` function.
22
// The path is expected to either be absolute or relative to the current working directory.
33

4-
import { readFile } from "node:fs/promises";
4+
import { readFile, stat } from "node:fs/promises";
55
import { resolve } from "node:path";
66
import { parse } from "yaml";
77

@@ -10,7 +10,13 @@ const rkebab = /-([a-z])/g;
1010

1111
export default async function readYAML( path ) {
1212
if ( !path ) {
13-
return {};
13+
if ( await stat( resolve( process.cwd(), "jtr.yml" ) ).catch( () => false ) ) {
14+
path = "jtr.yml";
15+
} else if ( await stat( resolve( process.cwd(), "jtr.yaml" ) ).catch( () => false ) ) {
16+
path = "jtr.yaml";
17+
} else {
18+
return {};
19+
}
1420
}
1521

1622
const contents = await readFile( resolve( process.cwd(), path ), "utf8" );

‎package-lock.json‎

Lines changed: 1 addition & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎queue.js‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ export function retryTest( reportId, maxRetries ) {
4545
test.retries++;
4646
if ( test.retries <= maxRetries ) {
4747
console.log(
48-
`\nRetrying test ${ reportId } for ${ chalk.yellow(
49-
test.options.modules.join( ", " )
50-
) }...${ test.retries }`
48+
`\nRetrying test ${ reportId } with URL ${ chalk.yellow( test.url ) }...${
49+
test.retries
50+
}`
5151
);
5252
return test;
5353
}
@@ -63,8 +63,8 @@ export async function hardRetryTest( reportId, maxHardRetries ) {
6363
test.hardRetries++;
6464
if ( test.hardRetries <= maxHardRetries ) {
6565
console.log(
66-
`\nHard retrying test ${ reportId } for ${ chalk.yellow(
67-
test.options.modules.join( ", " )
66+
`\nHard retrying test ${ reportId } with URL ${ chalk.yellow(
67+
test.url
6868
) }...${ test.hardRetries }`
6969
);
7070
await restartBrowser( test.browser );

‎reporter.js‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import chalk from "chalk";
2-
import { getBrowserString } from "./lib/getBrowserString.js";
32
import { prettyMs } from "./lib/prettyMs.js";
43
import * as Diff from "diff";
54

@@ -15,7 +14,7 @@ function serializeForDiff( value ) {
1514
return `${ value }`;
1615
}
1716

18-
export function reportTest( test, { browser, headless, id } ) {
17+
export function reportTest( test, { fullBrowser, id } ) {
1918
if ( test.status === "passed" ) {
2019

2120
// Write to console without newlines
@@ -25,7 +24,7 @@ export function reportTest( test, { browser, headless, id } ) {
2524

2625
let message = `${ chalk.bold( `${ test.suiteName }: ${ test.name }` ) }`;
2726
message += `\nTest ${ test.status } on ${ chalk.yellow(
28-
getBrowserString( browser, headless )
27+
fullBrowser
2928
) } (${ chalk.bold( id ) }).`;
3029

3130
// test.assertions only contains passed assertions;
@@ -114,13 +113,10 @@ export function reportTest( test, { browser, headless, id } ) {
114113
}
115114
}
116115

117-
export function reportEnd( result, { browser, flags, headless, id, isolatedFlag } ) {
118-
const fullBrowser = getBrowserString( browser, headless );
119-
const allFlags = [ ...flags, ...( isolatedFlag ? [ isolatedFlag ] : [] ) ];
120-
116+
export function reportEnd( result, { fullBrowser, id, url } ) {
121117
console.log(
122118
`\n\nTests finished in ${ prettyMs( result.runtime ) } ` +
123-
`with flags "${ chalk.yellow( allFlags.join( "&" ) ) }" ` +
119+
`with URL ${ chalk.yellow( url ) } ` +
124120
`in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( id ) })...`
125121
);
126122
console.log(

‎run.js‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ export async function run( {
2929
flag: flags = [],
3030
hardRetries,
3131
headless,
32-
isolatedFlag: isolatedFlags = [],
3332
middleware = [],
3433
retries = 0,
34+
run: runs = [],
3535
runId,
3636
testUrl: testUrls = [],
3737
verbose
@@ -242,25 +242,17 @@ export async function run( {
242242
}
243243
}
244244

245-
function queueRun( browser, { isolatedFlag, testUrl } = {} ) {
245+
function queueRun( browser, { run, testUrl } = {} ) {
246246
const fullBrowser = getBrowserString( browser, headless );
247247
const reportId = generateHash(
248-
`${ hashValue }-${ isolatedFlag }-${ testUrl }-${ fullBrowser }`
248+
`${ hashValue }-${ run }-${ testUrl }-${ fullBrowser }`
249249
);
250-
reports[ reportId ] = {
251-
browser,
252-
flags,
253-
headless,
254-
id: reportId,
255-
isolatedFlag,
256-
testUrl
257-
};
258250

259251
const url = buildTestUrl( {
260252
baseUrl,
261253
browserstack,
262254
flags,
263-
isolatedFlag,
255+
run,
264256
jsdom: browser.browser === "jsdom",
265257
port,
266258
reportId,
@@ -270,30 +262,41 @@ export async function run( {
270262
const options = {
271263
baseUrl,
272264
browserstack,
265+
runId,
273266
concurrency,
274267
debug,
275268
flags,
276269
headless,
277-
isolatedFlag,
270+
run,
278271
reportId,
279-
runId,
280272
testUrl,
281273
tunnelId,
282274
verbose
283275
};
284276

277+
reports[ reportId ] = {
278+
browser,
279+
flags,
280+
fullBrowser,
281+
headless,
282+
id: reportId,
283+
run,
284+
testUrl,
285+
url
286+
};
287+
285288
addRun( url, browser, options );
286289
}
287290

288291
for ( const browser of browsers ) {
289-
if ( isolatedFlags.length > 0 ) {
290-
isolatedFlags.forEach( ( isolatedFlag ) => {
292+
if ( runs.length > 0 ) {
293+
runs.forEach( ( run ) => {
291294
if ( testUrls.length > 0 ) {
292295
testUrls.forEach( ( testUrl ) => {
293-
queueRun( browser, { isolatedFlag, testUrl } );
296+
queueRun( browser, { run, testUrl } );
294297
} );
295298
} else {
296-
queueRun( browser, { isolatedFlag } );
299+
queueRun( browser, { run } );
297300
}
298301
} );
299302
} else if ( testUrls.length > 0 ) {
@@ -306,7 +309,7 @@ export async function run( {
306309
}
307310

308311
try {
309-
console.log( `Starting Run ${ runId }...` );
312+
console.log( `Starting test run ${ runId }...` );
310313
await runAll();
311314
} catch ( error ) {
312315
console.error( error );
@@ -322,7 +325,7 @@ export async function run( {
322325
stop = true;
323326
const reportFlags = [
324327
...report.flags,
325-
...( report.isolatedFlag ? [ report.isolatedFlag ] : [] )
328+
...( report.run ? [ report.run ] : [] )
326329
];
327330
console.error(
328331
chalk.red(

0 commit comments

Comments
 (0)