Skip to content

Commit 82fd327

Browse files
jensjohaCommit Queue
jensjoha
authored and
Commit Queue
committed
[CFE] Tests for the coverage tests
More specifically, tests for the coverage merger tool, i.e. allowing us to write tests for missing coverage being reported in a certain way, and for the automatic update of comments working like we want it to. As this testing is added after-the-fact the (initial at least) testing is very sparse. Change-Id: I5ae372b4ad56511859ad6f616c99835d1ccd140f Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/443960 Reviewed-by: Johnni Winther <[email protected]> Commit-Queue: Jens Johansen <[email protected]>
1 parent 0f92bba commit 82fd327

18 files changed

+331
-5
lines changed

‎pkg/front_end/test/coverage_merger_suite.dart‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:async';
6+
import 'dart:io';
7+
8+
import 'package:testing/testing.dart'
9+
show
10+
Chain,
11+
ChainContext,
12+
Result,
13+
Step,
14+
TestDescription,
15+
ExpectationSet,
16+
Expectation;
17+
18+
import '../tool/coverage_merger.dart' as coverageMerger;
19+
import 'coverage_helper.dart';
20+
import 'testing/environment_keys.dart';
21+
import 'testing_utils.dart' show checkEnvironment;
22+
import 'utils/kernel_chain.dart';
23+
import 'utils/suite_utils.dart';
24+
import 'vm_service_coverage.dart';
25+
26+
void main([List<String> arguments = const []]) => internalMain(createContext,
27+
arguments: arguments,
28+
displayName: "coverage merger suite",
29+
configurationPath: "../testing.json");
30+
31+
const List<Map<String, String>> EXPECTATIONS = [
32+
{
33+
"name": "ExpectationFileMismatch",
34+
"group": "Fail",
35+
},
36+
{
37+
"name": "ExpectationFileMissing",
38+
"group": "Fail",
39+
},
40+
];
41+
42+
Future<Context> createContext(Chain suite, Map<String, String> environment) {
43+
const Set<String> knownEnvironmentKeys = {
44+
EnvironmentKeys.updateExpectations,
45+
};
46+
checkEnvironment(environment, knownEnvironmentKeys);
47+
48+
return new Future.value(new Context(suite.name, environment));
49+
}
50+
51+
class CheckCoverageData extends Step<TestDescription, void, Context> {
52+
const CheckCoverageData();
53+
54+
@override
55+
String get name => "CheckCoverageData";
56+
57+
@override
58+
Future<Result<void>> run(TestDescription description, Context context) async {
59+
Directory tmpDir =
60+
Directory.systemTemp.createTempSync("coverage_merger_test");
61+
try {
62+
Directory dartToolDir =
63+
new Directory.fromUri(tmpDir.uri.resolve(".dart_tool/"));
64+
dartToolDir.createSync(recursive: true);
65+
File packageConfig = new File.fromUri(
66+
tmpDir.uri.resolve(".dart_tool/package_config.json"));
67+
// We claim it being called 'front_end' as that's (currently at least) the
68+
// only package we process (almost) all files for.
69+
packageConfig.writeAsStringSync("""{
70+
"configVersion": 2,
71+
"packages": [
72+
{
73+
"name": "front_end",
74+
"rootUri": "../",
75+
"packageUri": "lib/",
76+
"languageVersion": "3.8"
77+
}
78+
]
79+
}""");
80+
Directory libDir = new Directory.fromUri(tmpDir.uri.resolve("lib/"));
81+
libDir.createSync(recursive: true);
82+
File main = new File.fromUri(tmpDir.uri.resolve("lib/main.dart"));
83+
File sourceFile = new File.fromUri(description.uri);
84+
main.writeAsStringSync(sourceFile.readAsStringSync().trim());
85+
86+
CollectingCoverageHelper helper = new CollectingCoverageHelper();
87+
await helper.start(
88+
[
89+
"--disable-dart-dev",
90+
"--enable-asserts",
91+
"--pause_isolates_on_exit",
92+
main.path
93+
],
94+
stdoutReceiver: (String line) {
95+
print(" > $line");
96+
},
97+
stderrReceiver: (String line) {
98+
print("e> $line");
99+
},
100+
);
101+
Coverage coverage = await helper.completer.future;
102+
Directory coverageDir =
103+
new Directory.fromUri(tmpDir.uri.resolve("coverage/"));
104+
coverageDir.createSync(recursive: true);
105+
File coverageFile =
106+
new File.fromUri(tmpDir.uri.resolve("coverage/coverage.json"));
107+
coverage.writeToFile(coverageFile);
108+
109+
Map<Uri, coverageMerger.CoverageInfo> coverageData =
110+
(await coverageMerger.mergeFromDirUri(
111+
packageConfig.uri,
112+
coverageDir.uri,
113+
silent: true,
114+
extraCoverageIgnores: ["coverage-ignore(suite):"],
115+
extraCoverageBlockIgnores: ["coverage-ignore-block(suite):"],
116+
addAndRemoveCommentsInFiles: false,
117+
stdoutReceiver: (String line) {
118+
print(" > $line");
119+
},
120+
stderrReceiver: (String line) {
121+
print("e> $line");
122+
},
123+
))!;
124+
if (coverageData.values.first.error) {
125+
print("Warning: Got an error.");
126+
}
127+
128+
Result<TestDescription> expectMatch =
129+
await context.match<TestDescription>(
130+
".visualization.expect",
131+
coverageData.values.first.visualization.trim(),
132+
description.uri,
133+
description);
134+
if (expectMatch.outcome != Expectation.pass) return expectMatch;
135+
136+
coverageData = (await coverageMerger.mergeFromDirUri(
137+
packageConfig.uri,
138+
coverageDir.uri,
139+
silent: true,
140+
extraCoverageIgnores: ["coverage-ignore(suite):"],
141+
extraCoverageBlockIgnores: ["coverage-ignore-block(suite):"],
142+
addAndRemoveCommentsInFiles: true,
143+
stdoutReceiver: (String line) {
144+
print(" > $line");
145+
},
146+
stderrReceiver: (String line) {
147+
print("e> $line");
148+
},
149+
))!;
150+
151+
print("Reading ${main.path}");
152+
153+
String outputWithComments = main.readAsStringSync().trim();
154+
return await context.match<TestDescription>(".commented.expect",
155+
outputWithComments, description.uri, description);
156+
} finally {
157+
try {
158+
tmpDir.deleteSync(recursive: true);
159+
} catch (e) {
160+
// Wait a little and retry.
161+
sleep(const Duration(milliseconds: 42));
162+
try {
163+
tmpDir.deleteSync(recursive: true);
164+
} catch (e) {
165+
print('Warning: $e');
166+
}
167+
}
168+
}
169+
}
170+
}
171+
172+
class CollectingCoverageHelper extends CoverageHelper {
173+
Completer<Coverage> completer = new Completer();
174+
175+
CollectingCoverageHelper() : super(doPrint: false, forceCompilation: true);
176+
177+
@override
178+
void gotCoverage(Coverage coverage) {
179+
completer.complete(coverage);
180+
}
181+
}
182+
183+
class Context extends ChainContext with MatchContext {
184+
final String suiteName;
185+
186+
@override
187+
final List<Step> steps = const <Step>[
188+
const CheckCoverageData(),
189+
];
190+
191+
@override
192+
final bool updateExpectations;
193+
194+
@override
195+
final ExpectationSet expectationSet =
196+
new ExpectationSet.fromJsonList(EXPECTATIONS);
197+
198+
Context(this.suiteName, Map<String, String> environment)
199+
: updateExpectations =
200+
environment[EnvironmentKeys.updateExpectations] == "true";
201+
202+
@override
203+
bool get canBeFixWithUpdateExpectations => true;
204+
205+
@override
206+
String get updateExpectationsOption =>
207+
'${EnvironmentKeys.updateExpectations}=true';
208+
}

‎pkg/front_end/test/coverage_merger_suite.status‎

Whitespace-only changes.

‎pkg/front_end/test/coverage_merger_tests/assert_message_auto_ignored.dart‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
void main(List<String> args) {
2+
assert(args.isEmpty || args.isNotEmpty,
3+
"$args was neither empty nor not empty.");
4+
print("The message above will never execute and is never covered.");
5+
}

‎pkg/front_end/test/coverage_merger_tests/assert_message_auto_ignored.dart.commented.expect‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
void main(List<String> args) {
2+
assert(
3+
args.isEmpty ||
4+
// Coverage-ignore(suite): Not run.
5+
args.isNotEmpty,
6+
"$args was neither empty nor not empty.",
7+
);
8+
print("The message above will never execute and is never covered.");
9+
}

‎pkg/front_end/test/coverage_merger_tests/assert_message_auto_ignored.dart.visualization.expect‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package:front_end/main.dart: 80% (1 misses)
2+
package:front_end/main.dart:2:
3+
In 'main':
4+
assert(args.isEmpty || args.isNotEmpty,
5+
^^^^^^^^^^

‎pkg/front_end/test/coverage_merger_tests/if_body_not_reached.dart‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
void main() {
2+
if (1 + 1 == 3) {
3+
print("condition reached, but body not reached.");
4+
} else {
5+
print("Else part reached.");
6+
}
7+
}

‎pkg/front_end/test/coverage_merger_tests/if_body_not_reached.dart.commented.expect‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
void main() {
2+
if (1 + 1 == 3) {
3+
// Coverage-ignore-block(suite): Not run.
4+
print("condition reached, but body not reached.");
5+
} else {
6+
print("Else part reached.");
7+
}
8+
}

‎pkg/front_end/test/coverage_merger_tests/if_body_not_reached.dart.visualization.expect‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package:front_end/main.dart: 80% (1 misses)
2+
package:front_end/main.dart:3:
3+
In 'main':
4+
print("condition reached, but body not reached.");
5+
^^^^^

‎pkg/front_end/test/coverage_merger_tests/if_dangling_uncovered.dart‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
void main() {
2+
print("hello");
3+
if (1 + 1 == 3) print("no?!?");
4+
}

‎pkg/front_end/test/coverage_merger_tests/if_dangling_uncovered.dart.commented.expect‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
void main() {
2+
print("hello");
3+
if (1 + 1 == 3)
4+
// Coverage-ignore(suite): Not run.
5+
print("no?!?");
6+
}

0 commit comments

Comments
 (0)