Skip to content

Commit 1186717

Browse files
frostebiteclaude
andcommitted
feat(testing): implement test workflow engine with YAML suites, taxonomy filtering, and structured results (#790)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 8d81236 commit 1186717

File tree

13 files changed

+2693
-3
lines changed

13 files changed

+2693
-3
lines changed

action.yml

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,8 @@ inputs:
182182
required: false
183183
default: ''
184184
description:
185-
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with the
186-
keys image, secrets (name, value object array), command line string)'
185+
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
186+
the keys image, secrets (name, value object array), command line string)'
187187
awsStackName:
188188
default: 'game-ci'
189189
required: false
@@ -279,6 +279,23 @@ inputs:
279279
description:
280280
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
281281
fixes.'
282+
testSuitePath:
283+
description: 'Path to YAML test suite definition file'
284+
required: false
285+
testSuiteEvent:
286+
description: 'CI event name for suite selection (pr, push, release)'
287+
required: false
288+
testTaxonomyPath:
289+
description: 'Path to custom taxonomy definition YAML'
290+
required: false
291+
testResultFormat:
292+
description: 'Test result output format: junit, json, or both'
293+
required: false
294+
default: 'junit'
295+
testResultPath:
296+
description: 'Directory for structured test result output'
297+
required: false
298+
default: './test-results'
282299

283300
outputs:
284301
volume:

dist/index.js

Lines changed: 994 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

dist/index.js.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/index.ts

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output
33
import { Cli } from './model/cli/cli';
44
import MacBuilder from './model/mac-builder';
55
import PlatformSetup from './model/platform-setup';
6+
import { TestWorkflowService } from './model/orchestrator/services/test-workflow';
67

78
async function runMain() {
89
try {
@@ -17,6 +18,23 @@ async function runMain() {
1718
const { workspace, actionFolder } = Action;
1819

1920
const buildParameters = await BuildParameters.create();
21+
22+
// If a test suite path is provided, use the test workflow engine
23+
// instead of the standard build execution path
24+
if (buildParameters.testSuitePath) {
25+
core.info('[TestWorkflow] Test suite path detected, using test workflow engine');
26+
const results = await TestWorkflowService.executeTestSuite(buildParameters.testSuitePath, buildParameters);
27+
28+
const totalFailed = results.reduce((sum, r) => sum + r.failed, 0);
29+
if (totalFailed > 0) {
30+
core.setFailed(`Test workflow completed with ${totalFailed} failure(s)`);
31+
} else {
32+
core.info('[TestWorkflow] All test runs passed');
33+
}
34+
35+
return;
36+
}
37+
2038
const baseImage = new ImageTag(buildParameters);
2139

2240
let exitCode = -1;

src/model/build-parameters.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,12 @@ class BuildParameters {
107107
public unityHubVersionOnMac!: string;
108108
public dockerWorkspacePath!: string;
109109

110+
public testSuitePath!: string;
111+
public testSuiteEvent!: string;
112+
public testTaxonomyPath!: string;
113+
public testResultFormat!: string;
114+
public testResultPath!: string;
115+
110116
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
111117
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
112118
}
@@ -242,6 +248,11 @@ class BuildParameters {
242248
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
243249
unityHubVersionOnMac: Input.unityHubVersionOnMac,
244250
dockerWorkspacePath: Input.dockerWorkspacePath,
251+
testSuitePath: Input.testSuitePath,
252+
testSuiteEvent: Input.testSuiteEvent,
253+
testTaxonomyPath: Input.testTaxonomyPath,
254+
testResultFormat: Input.testResultFormat,
255+
testResultPath: Input.testResultPath,
245256
};
246257
}
247258

src/model/input.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,26 @@ class Input {
282282
return Input.getInput('skipActivation')?.toLowerCase() ?? 'false';
283283
}
284284

285+
static get testSuitePath(): string {
286+
return Input.getInput('testSuitePath') ?? '';
287+
}
288+
289+
static get testSuiteEvent(): string {
290+
return Input.getInput('testSuiteEvent') ?? '';
291+
}
292+
293+
static get testTaxonomyPath(): string {
294+
return Input.getInput('testTaxonomyPath') ?? '';
295+
}
296+
297+
static get testResultFormat(): string {
298+
return Input.getInput('testResultFormat') ?? 'junit';
299+
}
300+
301+
static get testResultPath(): string {
302+
return Input.getInput('testResultPath') ?? './test-results';
303+
}
304+
285305
public static ToEnvVarFormat(input: string) {
286306
if (input.toUpperCase() === input) {
287307
return input;
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
export { TestSuiteParser } from './test-suite-parser';
2+
export { TaxonomyFilterService } from './taxonomy-filter-service';
3+
export { TestResultReporter } from './test-result-reporter';
4+
export { TestWorkflowService } from './test-workflow-service';
5+
export {
6+
TestSuiteDefinition,
7+
TestRunDefinition,
8+
TaxonomyDimension,
9+
TaxonomyDefinition,
10+
TestResult,
11+
TestFailure,
12+
} from './test-workflow-types';
Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
import fs from 'node:fs';
2+
import YAML from 'yaml';
3+
import { TaxonomyDimension, TaxonomyDefinition } from './test-workflow-types';
4+
5+
/**
6+
* Manages test taxonomy dimensions and builds filter arguments for
7+
* the Unity test runner CLI. Supports comma-separated value lists,
8+
* regex patterns (/pattern/), and hierarchical dot-notation matching.
9+
*/
10+
export class TaxonomyFilterService {
11+
/**
12+
* Built-in taxonomy dimensions that are always available.
13+
* Projects may extend these via a custom taxonomy file.
14+
*/
15+
private static readonly BUILT_IN_DIMENSIONS: TaxonomyDimension[] = [
16+
{ name: 'Scope', values: ['Unit', 'Integration', 'System', 'End To End'] },
17+
{ name: 'Maturity', values: ['Trusted', 'Adolescent', 'Experimental'] },
18+
{ name: 'FeedbackSpeed', values: ['Fast', 'Moderate', 'Slow'] },
19+
{ name: 'Execution', values: ['Synchronous', 'Asynchronous', 'Coroutine'] },
20+
{ name: 'Rigor', values: ['Strict', 'Normal', 'Relaxed'] },
21+
{ name: 'Determinism', values: ['Deterministic', 'NonDeterministic'] },
22+
{ name: 'IsolationLevel', values: ['Full', 'Partial', 'None'] },
23+
];
24+
25+
/**
26+
* Load taxonomy dimensions: built-in dimensions plus any custom dimensions
27+
* from an optional taxonomy file.
28+
*/
29+
static loadTaxonomy(filePath?: string): TaxonomyDimension[] {
30+
const dimensions = [...TaxonomyFilterService.BUILT_IN_DIMENSIONS];
31+
32+
if (filePath && fs.existsSync(filePath)) {
33+
const content = fs.readFileSync(filePath, 'utf8');
34+
const parsed = YAML.parse(content) as TaxonomyDefinition;
35+
36+
if (parsed?.extensible_groups && Array.isArray(parsed.extensible_groups)) {
37+
for (const group of parsed.extensible_groups) {
38+
if (group.name && Array.isArray(group.values)) {
39+
// If a custom dimension has the same name as a built-in, merge values
40+
const existing = dimensions.find((d) => d.name === group.name);
41+
if (existing) {
42+
const existingValues = new Set(existing.values);
43+
for (const value of group.values) {
44+
if (!existingValues.has(value)) {
45+
existing.values.push(value);
46+
}
47+
}
48+
} else {
49+
dimensions.push({ name: group.name, values: [...group.values] });
50+
}
51+
}
52+
}
53+
}
54+
}
55+
56+
return dimensions;
57+
}
58+
59+
/**
60+
* Convert a filter map to Unity test runner CLI args (--testFilter).
61+
*
62+
* Each filter dimension becomes a category expression. Multiple values in one
63+
* dimension are OR'd; multiple dimensions are AND'd. The result is a single
64+
* --testFilter string suitable for passing to Unity's test runner CLI.
65+
*
66+
* Regex patterns (values wrapped in /.../) are converted to category regex
67+
* expressions supported by the Unity test runner.
68+
*/
69+
static buildFilterArgs(filters: Record<string, string>): string {
70+
if (!filters || Object.keys(filters).length === 0) {
71+
return '';
72+
}
73+
74+
const categoryExpressions: string[] = [];
75+
76+
for (const [dimension, valueSpec] of Object.entries(filters)) {
77+
const expression = TaxonomyFilterService.buildDimensionExpression(dimension, valueSpec);
78+
if (expression) {
79+
categoryExpressions.push(expression);
80+
}
81+
}
82+
83+
if (categoryExpressions.length === 0) {
84+
return '';
85+
}
86+
87+
// Unity test runner uses --testFilter with category expressions
88+
// Multiple dimensions are AND'd by joining with ';'
89+
const filterString = categoryExpressions.join(';');
90+
return `--testFilter "${filterString}"`;
91+
}
92+
93+
/**
94+
* Build a filter expression for a single taxonomy dimension.
95+
*/
96+
private static buildDimensionExpression(dimension: string, valueSpec: string): string {
97+
if (!valueSpec || valueSpec.trim() === '') {
98+
return '';
99+
}
100+
101+
const trimmed = valueSpec.trim();
102+
103+
// Check if the value is a regex pattern: /pattern/
104+
if (trimmed.startsWith('/') && trimmed.endsWith('/') && trimmed.length > 2) {
105+
const pattern = trimmed.slice(1, -1);
106+
return `${dimension}=~${pattern}`;
107+
}
108+
109+
// Comma-separated values: OR'd together
110+
const values = trimmed
111+
.split(',')
112+
.map((v) => v.trim())
113+
.filter((v) => v.length > 0);
114+
115+
if (values.length === 0) {
116+
return '';
117+
}
118+
119+
if (values.length === 1) {
120+
return `${dimension}=${values[0]}`;
121+
}
122+
123+
// Multiple values: use pipe-separated OR syntax
124+
return `${dimension}=${values.join('|')}`;
125+
}
126+
127+
/**
128+
* Check if a test's taxonomy metadata matches the given filter criteria.
129+
*
130+
* A test matches if ALL filter dimensions match (AND across dimensions).
131+
* Within a single dimension, the test must match ANY of the specified values (OR).
132+
* Regex patterns are matched as regular expressions.
133+
* Hierarchical dot-notation supports prefix matching (e.g., filter "Combat.Melee"
134+
* matches test category "Combat.Melee.Sword").
135+
*/
136+
static matchesFilter(testCategories: Record<string, string>, filters: Record<string, string>): boolean {
137+
for (const [dimension, valueSpec] of Object.entries(filters)) {
138+
const testValue = testCategories[dimension];
139+
140+
// If the test has no value for this dimension, it does not match
141+
if (testValue === undefined || testValue === null) {
142+
return false;
143+
}
144+
145+
if (!TaxonomyFilterService.matchesDimensionFilter(testValue, valueSpec)) {
146+
return false;
147+
}
148+
}
149+
150+
return true;
151+
}
152+
153+
/**
154+
* Check if a single test category value matches a dimension filter spec.
155+
*/
156+
private static matchesDimensionFilter(testValue: string, valueSpec: string): boolean {
157+
const trimmed = valueSpec.trim();
158+
159+
// Regex pattern
160+
if (trimmed.startsWith('/') && trimmed.endsWith('/') && trimmed.length > 2) {
161+
const pattern = trimmed.slice(1, -1);
162+
try {
163+
const regex = new RegExp(pattern);
164+
return regex.test(testValue);
165+
} catch {
166+
// Invalid regex, treat as literal
167+
return testValue === trimmed;
168+
}
169+
}
170+
171+
// Comma-separated values
172+
const values = trimmed
173+
.split(',')
174+
.map((v) => v.trim())
175+
.filter((v) => v.length > 0);
176+
177+
return values.some((filterValue) => {
178+
// Exact match
179+
if (testValue === filterValue) {
180+
return true;
181+
}
182+
183+
// Hierarchical dot-notation prefix match
184+
// Filter "Combat.Melee" matches test "Combat.Melee" and "Combat.Melee.Sword"
185+
if (filterValue.includes('.') || testValue.includes('.')) {
186+
if (testValue.startsWith(filterValue + '.') || testValue === filterValue) {
187+
return true;
188+
}
189+
// Also allow the test to be a prefix of the filter for upward matching
190+
if (filterValue.startsWith(testValue + '.')) {
191+
return true;
192+
}
193+
}
194+
195+
return false;
196+
});
197+
}
198+
}

0 commit comments

Comments
 (0)