Skip to content

Commit 1403e77

Browse files
mydeaLuca Forstner
mydea
and
Luca Forstner
authored
test: Make E2E tests async & parallelizable (#7466)
Co-authored-by: Luca Forstner <[email protected]>
1 parent 9ecd152 commit 1403e77

40 files changed

+659
-469
lines changed

‎.github/workflows/build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ jobs:
185185
name: Build
186186
needs: [job_get_metadata, job_install_deps]
187187
runs-on: ubuntu-20.04
188-
timeout-minutes: 20
188+
timeout-minutes: 30
189189
steps:
190190
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
191191
uses: actions/checkout@v3

‎packages/e2e-tests/.eslintrc.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module.exports = {
33
node: true,
44
},
55
extends: ['../../.eslintrc.js'],
6-
ignorePatterns: ['test-applications/**'],
6+
ignorePatterns: ['test-applications/**', 'tmp/**'],
77
parserOptions: {
88
sourceType: 'module',
99
},

‎packages/e2e-tests/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
.env
2+
tmp

‎packages/e2e-tests/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ To get you started with the recipe, you can copy the following into `test-recipe
5454
{
5555
"$schema": "../../test-recipe-schema.json",
5656
"testApplicationName": "My New Test Application",
57-
"buildCommand": "yarn install --pure-lockfile",
57+
"buildCommand": "yarn install --network-concurrency 1",
5858
"tests": [
5959
{
6060
"testName": "My new test",

‎packages/e2e-tests/lib/buildApp.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/* eslint-disable no-console */
2+
3+
import * as fs from 'fs-extra';
4+
import * as path from 'path';
5+
6+
import { DEFAULT_BUILD_TIMEOUT_SECONDS } from './constants';
7+
import type { Env, RecipeInstance } from './types';
8+
import { spawnAsync } from './utils';
9+
10+
export async function buildApp(appDir: string, recipeInstance: RecipeInstance, env: Env): Promise<void> {
11+
const { recipe, label, dependencyOverrides } = recipeInstance;
12+
13+
const packageJsonPath = path.resolve(appDir, 'package.json');
14+
15+
if (dependencyOverrides) {
16+
// Override dependencies
17+
const packageJson: { dependencies?: Record<string, string> } = JSON.parse(
18+
fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }),
19+
);
20+
packageJson.dependencies = packageJson.dependencies
21+
? { ...packageJson.dependencies, ...dependencyOverrides }
22+
: dependencyOverrides;
23+
fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2), {
24+
encoding: 'utf-8',
25+
});
26+
}
27+
28+
if (recipe.buildCommand) {
29+
console.log(`Running build command for test application "${label}"`);
30+
31+
const buildResult = await spawnAsync(recipe.buildCommand, {
32+
cwd: appDir,
33+
timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
34+
env: {
35+
...process.env,
36+
...env,
37+
} as unknown as NodeJS.ProcessEnv,
38+
});
39+
40+
if (buildResult.error) {
41+
console.log(`Build failed for test application "${label}"`);
42+
43+
// Prepends some text to the output build command's output so we can distinguish it from logging in this script
44+
console.log(buildResult.stdout.replace(/^/gm, ' [BUILD OUTPUT] '));
45+
console.log(buildResult.stderr.replace(/^/gm, ' [BUILD OUTPUT] '));
46+
47+
console.log('[BUILD ERROR] ', buildResult.error);
48+
throw buildResult.error;
49+
}
50+
51+
if (recipe.buildAssertionCommand) {
52+
console.log(`Running build assertion for test application "${label}"`);
53+
54+
const buildAssertionResult = await spawnAsync(
55+
recipe.buildAssertionCommand,
56+
{
57+
cwd: appDir,
58+
timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
59+
env: {
60+
...process.env,
61+
...env,
62+
} as unknown as NodeJS.ProcessEnv,
63+
},
64+
buildResult.stdout,
65+
);
66+
67+
if (buildAssertionResult.error) {
68+
console.log(`Build assertion failed for test application "${label}"`);
69+
70+
// Prepends some text to the output build command's output so we can distinguish it from logging in this script
71+
console.log(buildAssertionResult.stdout.replace(/^/gm, ' [BUILD ASSERTION OUTPUT] '));
72+
console.log(buildAssertionResult.stderr.replace(/^/gm, ' [BUILD ASSERTION OUTPUT] '));
73+
74+
console.log('[BUILD ASSERTION ERROR] ', buildAssertionResult.error);
75+
76+
throw buildAssertionResult.error;
77+
}
78+
}
79+
}
80+
}

‎packages/e2e-tests/lib/buildRecipeInstances.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import * as fs from 'fs';
2+
3+
import type { Recipe, RecipeInput, RecipeInstance } from './types';
4+
5+
export function buildRecipeInstances(recipePaths: string[]): RecipeInstance[] {
6+
const recipes = buildRecipes(recipePaths);
7+
const recipeInstances: RecipeInstance[] = [];
8+
9+
const basePort = 3001;
10+
11+
recipes.forEach((recipe, i) => {
12+
recipe.versions.forEach(version => {
13+
const dependencyOverrides =
14+
Object.keys(version.dependencyOverrides).length > 0 ? version.dependencyOverrides : undefined;
15+
const dependencyOverridesInformationString = dependencyOverrides
16+
? ` (Dependency overrides: ${JSON.stringify(dependencyOverrides)})`
17+
: '';
18+
19+
recipeInstances.push({
20+
label: `${recipe.testApplicationName}${dependencyOverridesInformationString}`,
21+
recipe,
22+
dependencyOverrides,
23+
port: basePort + i,
24+
});
25+
});
26+
});
27+
28+
return recipeInstances;
29+
}
30+
31+
function buildRecipes(recipePaths: string[]): Recipe[] {
32+
return recipePaths.map(recipePath => buildRecipe(recipePath));
33+
}
34+
35+
function buildRecipe(recipePath: string): Recipe {
36+
const recipe: RecipeInput = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
37+
38+
const versions = process.env.CANARY_E2E_TEST
39+
? recipe.canaryVersions ?? []
40+
: recipe.versions ?? [{ dependencyOverrides: {} }];
41+
42+
return {
43+
...recipe,
44+
path: recipePath,
45+
versions,
46+
};
47+
}

‎packages/e2e-tests/lib/constants.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
export const TEST_REGISTRY_CONTAINER_NAME = 'verdaccio-e2e-test-registry';
2+
export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5;
3+
export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2;
4+
export const VERDACCIO_VERSION = '5.22.1';
5+
export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
6+
export const TMP_DIR = 'tmp';

‎packages/e2e-tests/lib/runAllTestApps.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/* eslint-disable no-console */
2+
import { buildRecipeInstances } from './buildRecipeInstances';
3+
import { buildAndTestApp } from './runTestApp';
4+
import type { RecipeInstance, RecipeTestResult } from './types';
5+
6+
export async function runAllTestApps(
7+
recipePaths: string[],
8+
envVarsToInject: Record<string, string | undefined>,
9+
): Promise<void> {
10+
const maxParallel = process.env.CI ? 2 : 5;
11+
12+
const recipeInstances = buildRecipeInstances(recipePaths);
13+
14+
const results = await shardPromises(
15+
recipeInstances,
16+
recipeInstance => buildAndTestApp(recipeInstance, envVarsToInject),
17+
maxParallel,
18+
);
19+
20+
console.log('--------------------------------------');
21+
console.log('Test Result Summary:');
22+
23+
results.forEach(result => {
24+
if (result.buildFailed) {
25+
console.log(`● BUILD FAILED - ${result.label} (${result.recipe.path}`);
26+
} else {
27+
console.log(`● BUILD SUCCEEDED - ${result.label}`);
28+
result.tests.forEach(testResult => {
29+
console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
30+
});
31+
}
32+
});
33+
34+
const failed = results.filter(result => result.buildFailed || result.testFailed);
35+
36+
if (failed.length) {
37+
console.log(`${failed.length} test(s) failed.`);
38+
process.exit(1);
39+
}
40+
41+
console.log('All tests succeeded. 🎉');
42+
}
43+
44+
// Always run X promises at a time
45+
function shardPromises(
46+
recipes: RecipeInstance[],
47+
callback: (recipe: RecipeInstance) => Promise<RecipeTestResult>,
48+
maxParallel: number,
49+
): Promise<RecipeTestResult[]> {
50+
return new Promise(resolve => {
51+
console.log(`Running a total of ${recipes.length} jobs, with up to ${maxParallel} jobs in parallel...`);
52+
const results: RecipeTestResult[] = [];
53+
const remaining = recipes.slice();
54+
const running: Promise<unknown>[] = [];
55+
56+
function runNext(): void {
57+
if (running.length < maxParallel && remaining.length > 0) {
58+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
59+
const next = remaining.shift()!;
60+
const promise = callback(next);
61+
62+
console.log(`Running job ${next.label}, ${remaining.length} remaining...`);
63+
64+
running.push(promise);
65+
66+
promise
67+
.then(result => results.push(result))
68+
.finally(() => {
69+
const pos = running.indexOf(promise);
70+
running.splice(pos, 1);
71+
72+
runNext();
73+
});
74+
} else if (remaining.length === 0 && running.length === 0) {
75+
resolve(results);
76+
}
77+
}
78+
79+
// Initial runs
80+
for (let i = 0; i < maxParallel; i++) {
81+
runNext();
82+
}
83+
});
84+
}

‎packages/e2e-tests/lib/runTestApp.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/* eslint-disable no-console */
2+
3+
import * as fs from 'fs-extra';
4+
import * as path from 'path';
5+
6+
import { buildApp } from './buildApp';
7+
import { TMP_DIR } from './constants';
8+
import { testApp } from './testApp';
9+
import type { Env, RecipeInstance, RecipeTestResult } from './types';
10+
11+
let tmpDirCount = 0;
12+
13+
// This should never throw, we always return a result here
14+
export async function buildAndTestApp(
15+
recipeInstance: RecipeInstance,
16+
envVarsToInject: Record<string, string | undefined>,
17+
): Promise<RecipeTestResult> {
18+
const { recipe, port } = recipeInstance;
19+
const recipeDirname = path.dirname(recipe.path);
20+
21+
const targetDir = path.join(TMP_DIR, `${recipe.testApplicationName}-${tmpDirCount++}`);
22+
23+
await fs.copy(recipeDirname, targetDir);
24+
25+
const env: Env = {
26+
...envVarsToInject,
27+
PORT: port.toString(),
28+
};
29+
30+
try {
31+
await buildApp(targetDir, recipeInstance, env);
32+
} catch (error) {
33+
await fs.remove(targetDir);
34+
35+
return {
36+
...recipeInstance,
37+
buildFailed: true,
38+
testFailed: false,
39+
tests: [],
40+
};
41+
}
42+
43+
// This cannot throw, we always return a result here
44+
const results = await testApp(targetDir, recipeInstance, env);
45+
46+
// Cleanup
47+
await fs.remove(targetDir);
48+
49+
return {
50+
...recipeInstance,
51+
buildFailed: false,
52+
testFailed: results.some(result => result.result !== 'PASS'),
53+
tests: results,
54+
};
55+
}

‎packages/e2e-tests/lib/testApp.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/* eslint-disable no-console */
2+
3+
import { DEFAULT_TEST_TIMEOUT_SECONDS } from './constants';
4+
import type { Env, RecipeInstance, TestDef, TestResult } from './types';
5+
import { spawnAsync } from './utils';
6+
7+
export async function testApp(appDir: string, recipeInstance: RecipeInstance, env: Env): Promise<TestResult[]> {
8+
const { recipe } = recipeInstance;
9+
10+
const results: TestResult[] = [];
11+
for (const test of recipe.tests) {
12+
results.push(await runTest(appDir, recipeInstance, test, env));
13+
}
14+
15+
return results;
16+
}
17+
18+
async function runTest(appDir: string, recipeInstance: RecipeInstance, test: TestDef, env: Env): Promise<TestResult> {
19+
const { recipe, label } = recipeInstance;
20+
console.log(`Running test command for test application "${label}", test "${test.testName}"`);
21+
22+
const testResult = await spawnAsync(test.testCommand, {
23+
cwd: appDir,
24+
timeout: (recipe.testTimeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
25+
env: {
26+
...process.env,
27+
...env,
28+
} as unknown as NodeJS.ProcessEnv,
29+
});
30+
31+
if (testResult.error) {
32+
console.log(`Test failed for test application "${label}", test "${test.testName}"`);
33+
34+
// Prepends some text to the output test command's output so we can distinguish it from logging in this script
35+
console.log(testResult.stdout.replace(/^/gm, ' [TEST OUTPUT] '));
36+
console.log(testResult.stderr.replace(/^/gm, ' [TEST OUTPUT] '));
37+
38+
console.log('[TEST ERROR] ', testResult.error);
39+
40+
return {
41+
testName: test.testName,
42+
result: testResult.error?.message.includes('ETDIMEDOUT') ? 'TIMEOUT' : 'FAIL',
43+
};
44+
}
45+
46+
return {
47+
testName: test.testName,
48+
result: 'PASS',
49+
};
50+
}

0 commit comments

Comments
 (0)