Skip to content

Commit 5c976a6

Browse files
plemarquandplemarquand
authored
Associate XCTest suite start/stop with test item (#1152)
Capture the XCTest output associated with suites and assocaite it with the test item that appears in the test run history. This makes it easy to see the start/end time output of a suite by clicking on it. In order to capture the test target for a suite we need to wait for the first test that executes within that suite in order to get the target. For whatever reason, XCTest does not print a suites target, only a test's when it starts. The approach is if a suite starts, queue output up until the first test run, then associate the last line of queued output with the test target found on the started test. Lines before the last line are not assocaited with a test (these are typically the top level suite like "Selected Tests" or the test target name. Issue: #1148
1 parent c3b64e0 commit 5c976a6

File tree

5 files changed

+244
-31
lines changed

5 files changed

+244
-31
lines changed

‎src/TestExplorer/TestParsers/TestRunState.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,14 @@ import * as vscode from "vscode";
2020
export interface ITestRunState {
2121
// excess data from previous parse that was not processed
2222
excess?: string;
23+
24+
// the currently running suite, with the test target included, i.e: TestTarget.Suite
25+
// note that TestTarget is only present on Darwin.
26+
activeSuite?: string;
27+
28+
// output captured before a test has run in a suite
29+
pendingSuiteOutput?: string[];
30+
2331
// failed test state
2432
failedTest?: {
2533
testIndex: number;

‎src/TestExplorer/TestParsers/XCTestOutputParser.ts

Lines changed: 70 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,11 @@ export const darwinTestRegex = {
4343
error: /^(.+):(\d+):\serror:\s-\[(\S+)\s(.*)\] : (.*)$/,
4444
// Regex "<path/to/test>:<line number>: -[<test target> <class.function>] : Test skipped"
4545
skipped: /^(.+):(\d+):\s-\[(\S+)\s(.*)\] : Test skipped/,
46-
// Regex "Test Suite '-[<test target> <class.function>]' started"
46+
// Regex "Test Suite '<class>' started"
4747
startedSuite: /^Test Suite '(.*)' started/,
48-
// Regex "Test Suite '-[<test target> <class.function>]' passed"
48+
// Regex "Test Suite '<class>' passed"
4949
passedSuite: /^Test Suite '(.*)' passed/,
50-
// Regex "Test Suite '-[<test target> <class.function>]' failed"
50+
// Regex "Test Suite '<class>' failed"
5151
failedSuite: /^Test Suite '(.*)' failed/,
5252
};
5353

@@ -61,11 +61,11 @@ export const nonDarwinTestRegex = {
6161
error: /^(.+):(\d+):\serror:\s*(.*)\.(.*) : (.*)/,
6262
// Regex "<path/to/test>:<line number>: <class>.<function> : Test skipped"
6363
skipped: /^(.+):(\d+):\s*(.*)\.(.*) : Test skipped/,
64-
// Regex "Test Suite '-[<test target> <class.function>]' started"
64+
// Regex "Test Suite '<class>' started"
6565
startedSuite: /^Test Suite '(.*)' started/,
66-
// Regex "Test Suite '-[<test target> <class.function>]' passed"
66+
// Regex "Test Suite '<class>' passed"
6767
passedSuite: /^Test Suite '(.*)' passed/,
68-
// Regex "Test Suite '-[<test target> <class.function>]' failed"
68+
// Regex "Test Suite '<class>' failed"
6969
failedSuite: /^Test Suite '(.*)' failed/,
7070
};
7171

@@ -178,6 +178,12 @@ export class XCTestOutputParser implements IXCTestOutputParser {
178178
const startedMatch = this.regex.started.exec(line);
179179
if (startedMatch) {
180180
const testName = `${startedMatch[1]}/${startedMatch[2]}`;
181+
182+
// Save the active TestTarget.SuiteClass.
183+
// Note that TestTarget is only present on Darwin.
184+
runState.activeSuite = startedMatch[1];
185+
this.processPendingSuiteOutput(runState, startedMatch[1]);
186+
181187
const startedTestIndex = runState.getTestItemIndex(testName, undefined);
182188
this.startTest(startedTestIndex, runState);
183189
this.appendTestOutput(startedTestIndex, line, runState);
@@ -232,40 +238,86 @@ export class XCTestOutputParser implements IXCTestOutputParser {
232238
// Regex "Test Suite '-[<test target> <class.function>]' started"
233239
const startedSuiteMatch = this.regex.startedSuite.exec(line);
234240
if (startedSuiteMatch) {
235-
this.startTestSuite(startedSuiteMatch[1], runState);
236-
this.appendTestOutput(undefined, line, runState);
241+
this.startTestSuite(startedSuiteMatch[1], line, runState);
237242
continue;
238243
}
239244
// Regex "Test Suite '-[<test target> <class.function>]' passed"
240245
const passedSuiteMatch = this.regex.passedSuite.exec(line);
241246
if (passedSuiteMatch) {
242-
this.passTestSuite(passedSuiteMatch[1], runState);
243-
this.appendTestOutput(undefined, line, runState);
247+
this.completeSuite(runState, line, this.passTestSuite);
244248
continue;
245249
}
246250
// Regex "Test Suite '-[<test target> <class.function>]' failed"
247251
const failedSuiteMatch = this.regex.failedSuite.exec(line);
248252
if (failedSuiteMatch) {
249-
this.failTestSuite(failedSuiteMatch[1], runState);
250-
this.appendTestOutput(undefined, line, runState);
253+
this.completeSuite(runState, line, this.failTestSuite);
251254
continue;
252255
}
253256
// unrecognised output could be the continuation of a previous error message
254257
this.continueErrorMessage(line, runState);
255258
}
256259
}
257260

261+
/**
262+
* Process the buffered lines captured before a test case has started.
263+
*/
264+
private processPendingSuiteOutput(runState: ITestRunState, suite?: string) {
265+
// If we have a qualified suite name captured from a runninng test
266+
// process the lines captured before the test started, associating the
267+
// line line with the suite.
268+
if (runState.pendingSuiteOutput) {
269+
const startedSuiteIndex = suite ? runState.getTestItemIndex(suite, undefined) : -1;
270+
const totalLines = runState.pendingSuiteOutput.length - 1;
271+
for (let i = 0; i <= totalLines; i++) {
272+
const line = runState.pendingSuiteOutput[i];
273+
274+
// Only the last line of the captured output should be associated with the suite
275+
const associateLineWithSuite = i === totalLines && startedSuiteIndex !== -1;
276+
277+
this.appendTestOutput(
278+
associateLineWithSuite ? startedSuiteIndex : undefined,
279+
line,
280+
runState
281+
);
282+
}
283+
runState.pendingSuiteOutput = [];
284+
}
285+
}
286+
287+
/** Mark a suite as complete */
288+
private completeSuite(
289+
runState: ITestRunState,
290+
line: string,
291+
resultMethod: (name: string, runState: ITestRunState) => void
292+
) {
293+
let suiteIndex: number | undefined;
294+
if (runState.activeSuite) {
295+
resultMethod(runState.activeSuite, runState);
296+
suiteIndex = runState.getTestItemIndex(runState.activeSuite, undefined);
297+
}
298+
299+
// If no tests have run we may have output still in the buffer.
300+
// If activeSuite is undefined we finished an empty suite
301+
// and we still want to flush the buffer.
302+
this.processPendingSuiteOutput(runState, runState.activeSuite);
303+
304+
runState.activeSuite = undefined;
305+
this.appendTestOutput(suiteIndex, line, runState);
306+
}
307+
258308
/** Get Test parsing regex for current platform */
259309
private get platformTestRegex(): TestRegex {
260-
if (process.platform === "darwin") {
261-
return darwinTestRegex;
262-
} else {
263-
return nonDarwinTestRegex;
264-
}
310+
return process.platform === "darwin" ? darwinTestRegex : nonDarwinTestRegex;
265311
}
266312

267313
/** Flag a test suite has started */
268-
private startTestSuite(name: string, runState: ITestRunState) {
314+
private startTestSuite(name: string, line: string, runState: ITestRunState) {
315+
// Buffer the output to this point until the first test
316+
// starts, at which point we can determine the target.
317+
runState.pendingSuiteOutput = runState.pendingSuiteOutput
318+
? [...runState.pendingSuiteOutput, line]
319+
: [line];
320+
269321
runState.startedSuite(name);
270322
}
271323

‎src/TestExplorer/TestRunner.ts

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ export class TestRunner {
323323

324324
/**
325325
* If the request has no test items to include in the run,
326-
* default to usig all the items in the `TestController`.
326+
* default to using all the items in the `TestController`.
327327
*/
328328
private ensureRequestIncludesTests(request: vscode.TestRunRequest): vscode.TestRunRequest {
329329
if ((request.include?.length ?? 0) > 0) {
@@ -1261,13 +1261,9 @@ export class TestRunnerTestRunState implements ITestRunState {
12611261
return;
12621262
}
12631263

1264-
const uri = this.testRun.testItems[index].uri;
1265-
const range = this.testRun.testItems[index].range;
1266-
let location: vscode.Location | undefined;
1267-
if (uri && range) {
1268-
location = new vscode.Location(uri, range);
1269-
}
1270-
1271-
this.testRun.appendOutputToTest(output, this.testRun.testItems[index], location);
1264+
const testItem = this.testRun.testItems[index];
1265+
const { uri, range } = testItem;
1266+
const location = uri && range ? new vscode.Location(uri, range) : undefined;
1267+
this.testRun.appendOutputToTest(output, testItem, location);
12721268
}
12731269
}

‎test/integration-tests/testexplorer/MockTestRunState.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ export class TestRunState implements ITestRunState {
6767
lineNumber: number;
6868
complete: boolean;
6969
};
70+
allOutput: string[] = [];
7071

7172
public testItemFinder: ITestItemFinder;
7273

@@ -123,6 +124,7 @@ export class TestRunState implements ITestRunState {
123124
if (index !== undefined) {
124125
this.testItemFinder.tests[index].output.push(output);
125126
}
127+
this.allOutput.push(output);
126128
}
127129

128130
// started suite

‎test/integration-tests/testexplorer/XCTestOutputParser.test.ts

Lines changed: 159 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,10 @@ Test Case '-[MyTests.MyTests`;
264264
});
265265

266266
test("Suite", () => {
267-
const testRunState = new TestRunState(["MyTests", "MyTests.MyTests/testPass"], true);
267+
const testRunState = new TestRunState(
268+
["MyTests.MyTests", "MyTests.MyTests/testPass"],
269+
true
270+
);
268271
const input = `Test Suite 'MyTests' started at 2024-08-26 13:19:25.325.
269272
Test Case '-[MyTests.MyTests testPass]' started.
270273
Test Case '-[MyTests.MyTests testPass]' passed (0.001 seconds).
@@ -273,19 +276,171 @@ Test Suite 'MyTests' passed at 2024-08-26 13:19:25.328.
273276
`;
274277
outputParser.parseResult(input, testRunState);
275278

279+
const testOutput = inputToTestOutput(input);
276280
assert.deepEqual(testRunState.tests, [
277281
{
278-
name: "MyTests",
279-
output: [],
282+
name: "MyTests.MyTests",
283+
output: [testOutput[0], testOutput[3]],
280284
status: TestStatus.passed,
281285
},
282286
{
283287
name: "MyTests.MyTests/testPass",
284288
status: TestStatus.passed,
285289
timing: { duration: 0.001 },
286-
output: inputToTestOutput(input).slice(1, -2), // trim the suite text
290+
output: [testOutput[1], testOutput[2]],
291+
},
292+
]);
293+
assert.deepEqual(inputToTestOutput(input), testRunState.allOutput);
294+
});
295+
296+
test("Empty Suite", () => {
297+
const testRunState = new TestRunState([], true);
298+
const input = `Test Suite 'Selected tests' started at 2024-10-19 15:23:29.594.
299+
Test Suite 'EmptyAppPackageTests.xctest' started at 2024-10-19 15:23:29.595.
300+
Test Suite 'EmptyAppPackageTests.xctest' passed at 2024-10-19 15:23:29.595.
301+
Executed 0 tests, with 0 failures (0 unexpected) in 0.000 (0.000) seconds
302+
Test Suite 'Selected tests' passed at 2024-10-19 15:23:29.596.
303+
Executed 0 tests, with 0 failures (0 unexpected) in 0.000 (0.001) seconds
304+
warning: No matching test cases were run`;
305+
306+
outputParser.parseResult(input, testRunState);
307+
308+
assert.deepEqual(testRunState.tests, []);
309+
assert.deepEqual(inputToTestOutput(input), testRunState.allOutput);
310+
});
311+
312+
test("Multiple Suites", () => {
313+
const testRunState = new TestRunState(
314+
[
315+
"MyTests.TestSuite1",
316+
"MyTests.TestSuite1/testFirst",
317+
"MyTests.TestSuite2",
318+
"MyTests.TestSuite2/testSecond",
319+
],
320+
true
321+
);
322+
const input = `Test Suite 'All tests' started at 2024-10-20 21:54:32.568.
323+
Test Suite 'EmptyAppPackageTests.xctest' started at 2024-10-20 21:54:32.570.
324+
Test Suite 'TestSuite1' started at 2024-10-20 21:54:32.570.
325+
Test Case '-[MyTests.TestSuite1 testFirst]' started.
326+
Test Case '-[MyTests.TestSuite1 testFirst]' passed (0.000 seconds).
327+
Test Suite 'TestSuite1' passed at 2024-10-20 21:54:32.570.
328+
Executed 1 test, with 0 failures (0 unexpected) in 0.000 (0.001) seconds
329+
Test Suite 'TestSuite2' started at 2024-10-20 21:54:32.570.
330+
Test Case '-[MyTests.TestSuite2 testSecond]' started.
331+
Test Case '-[MyTests.TestSuite2 testSecond]' passed (0.000 seconds).
332+
Test Suite 'TestSuite2' passed at 2024-10-20 21:54:32.571.
333+
Executed 1 test, with 0 failures (0 unexpected) in 0.000 (0.000) seconds
334+
Test Suite 'EmptyAppPackageTests.xctest' passed at 2024-10-20 21:54:32.571.
335+
Executed 2 tests, with 0 failures (0 unexpected) in 0.001 (0.001) seconds
336+
Test Suite 'All tests' passed at 2024-10-20 21:54:32.571.
337+
Executed 2 tests, with 0 failures (0 unexpected) in 0.001 (0.002) seconds`;
338+
339+
outputParser.parseResult(input, testRunState);
340+
341+
const testOutput = inputToTestOutput(input);
342+
assert.deepEqual(testRunState.tests, [
343+
{
344+
name: "MyTests.TestSuite1",
345+
output: [testOutput[2], testOutput[5]],
346+
status: "passed",
347+
},
348+
{
349+
name: "MyTests.TestSuite1/testFirst",
350+
output: [testOutput[3], testOutput[4]],
351+
status: "passed",
352+
timing: {
353+
duration: 0,
354+
},
355+
},
356+
{
357+
name: "MyTests.TestSuite2",
358+
output: [testOutput[7], testOutput[10]],
359+
status: "passed",
360+
},
361+
{
362+
name: "MyTests.TestSuite2/testSecond",
363+
output: [testOutput[8], testOutput[9]],
364+
status: "passed",
365+
timing: {
366+
duration: 0,
367+
},
368+
},
369+
]);
370+
assert.deepEqual(inputToTestOutput(input), testRunState.allOutput);
371+
});
372+
373+
test("Multiple Suites with Failed Test", () => {
374+
const testRunState = new TestRunState(
375+
[
376+
"MyTests.TestSuite1",
377+
"MyTests.TestSuite1/testFirst",
378+
"MyTests.TestSuite2",
379+
"MyTests.TestSuite2/testSecond",
380+
],
381+
true
382+
);
383+
const input = `Test Suite 'Selected tests' started at 2024-10-20 22:01:46.206.
384+
Test Suite 'EmptyAppPackageTests.xctest' started at 2024-10-20 22:01:46.207.
385+
Test Suite 'TestSuite1' started at 2024-10-20 22:01:46.207.
386+
Test Case '-[MyTests.TestSuite1 testFirst]' started.
387+
Test Case '-[MyTests.TestSuite1 testFirst]' passed (0.000 seconds).
388+
Test Suite 'TestSuite1' passed at 2024-10-20 22:01:46.208.
389+
Executed 1 test, with 0 failures (0 unexpected) in 0.000 (0.000) seconds
390+
Test Suite 'TestSuite2' started at 2024-10-20 22:01:46.208.
391+
Test Case '-[MyTests.TestSuite2 testSecond]' started.
392+
/Users/user/Developer/MyTests/MyTests.swift:13: error: -[MyTests.TestSuite2 testSecond] : failed
393+
Test Case '-[MyTests.TestSuite2 testSecond]' failed (0.000 seconds).
394+
Test Suite 'TestSuite2' failed at 2024-10-20 22:01:46.306.
395+
Executed 1 test, with 1 failure (0 unexpected) in 0.000 (0.000) seconds
396+
Test Suite 'EmptyAppPackageTests.xctest' failed at 2024-10-20 22:01:46.306.
397+
Executed 2 tests, with 1 failure (0 unexpected) in 0.001 (0.001) seconds
398+
Test Suite 'Selected tests' failed at 2024-10-20 22:01:46.306.
399+
Executed 2 tests, with 1 failure (0 unexpected) in 0.002 (0.002) seconds`;
400+
outputParser.parseResult(input, testRunState);
401+
402+
const testOutput = inputToTestOutput(input);
403+
assert.deepEqual(testRunState.tests, [
404+
{
405+
name: "MyTests.TestSuite1",
406+
output: [testOutput[2], testOutput[5]],
407+
status: "passed",
408+
},
409+
{
410+
name: "MyTests.TestSuite1/testFirst",
411+
output: [testOutput[3], testOutput[4]],
412+
status: "passed",
413+
timing: {
414+
duration: 0,
415+
},
416+
},
417+
{
418+
name: "MyTests.TestSuite2",
419+
output: [testOutput[7], testOutput[11]],
420+
status: "failed",
421+
},
422+
{
423+
name: "MyTests.TestSuite2/testSecond",
424+
output: [testOutput[8], testOutput[9], testOutput[10]],
425+
status: "failed",
426+
timing: {
427+
duration: 0,
428+
},
429+
issues: [
430+
{
431+
message: "failed",
432+
location: sourceLocationToVSCodeLocation(
433+
"/Users/user/Developer/MyTests/MyTests.swift",
434+
13,
435+
0
436+
),
437+
isKnown: false,
438+
diff: undefined,
439+
},
440+
],
287441
},
288442
]);
443+
assert.deepEqual(inputToTestOutput(input), testRunState.allOutput);
289444
});
290445

291446
suite("Diffs", () => {

0 commit comments

Comments
 (0)