Skip to content

Commit 7755009

Browse files
netdpbnetdpb
authored
Support test names by comment (#148)
Partially fixes #141.
1 parent 388343b commit 7755009

File tree

6 files changed

+575
-390
lines changed

6 files changed

+575
-390
lines changed

‎conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestReport.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ private boolean isUnexpected(ReportedFact reportedFact) {
130130

131131
private static void writeFact(Formatter report, Fact fact, String status) {
132132
report.format(
133-
"%s: %s:%d %s%n", status, fact.getFile(), fact.getLineNumber(), fact.getFactText());
133+
"%s: %s:%s:%s%n", status, fact.getFile(), fact.getIdentifier(), fact.getFactText());
134134
}
135135

136136
/** A builder for {@link ConformanceTestReport}s. */
@@ -191,6 +191,7 @@ private Stream<ImmutableMap.Entry<ExpectedFact, ReportedFact>> matchFacts(
191191

192192
/** Builds the report. */
193193
ConformanceTestReport build() {
194+
expectedFactReader.checkErrors();
194195
return new ConformanceTestReport(
195196
files.build(),
196197
index(expectedFacts.build(), Fact::getFile),

‎conformance-test-framework/src/main/java/org/jspecify/conformance/ExpectedFact.java‎

Lines changed: 86 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,23 @@
1515
package org.jspecify.conformance;
1616

1717
import static com.google.common.base.MoreObjects.toStringHelper;
18+
import static com.google.common.base.Preconditions.checkArgument;
19+
import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
1820
import static java.util.stream.Collectors.joining;
1921

22+
import com.google.common.base.CharMatcher;
23+
import com.google.common.base.Joiner;
2024
import com.google.common.collect.ImmutableList;
2125
import java.nio.file.Path;
26+
import java.util.ArrayList;
2227
import java.util.HashMap;
2328
import java.util.List;
2429
import java.util.ListIterator;
2530
import java.util.Map;
2631
import java.util.Objects;
2732
import java.util.regex.Matcher;
2833
import java.util.regex.Pattern;
34+
import org.jspecify.annotations.Nullable;
2935

3036
/**
3137
* An assertion that the tool behaves in a way consistent with a specific fact about a line in the
@@ -47,11 +53,14 @@ public final class ExpectedFact extends Fact {
4753
Pattern.compile("test:irrelevant-annotation:\\S+"),
4854
Pattern.compile("test:sink-type:[^:]+:.*"));
4955

56+
private final @Nullable String testName;
5057
private final String factText;
5158
private final long factLineNumber;
5259

53-
ExpectedFact(Path file, long lineNumber, String factText, long factLineNumber) {
60+
ExpectedFact(
61+
Path file, long lineNumber, @Nullable String testName, String factText, long factLineNumber) {
5462
super(file, lineNumber);
63+
this.testName = testName;
5564
this.factText = factText;
5665
this.factLineNumber = factLineNumber;
5766
}
@@ -73,6 +82,11 @@ long getFactLineNumber() {
7382
return factLineNumber;
7483
}
7584

85+
@Override
86+
public String getIdentifier() {
87+
return testName == null ? super.getIdentifier() : testName;
88+
}
89+
7690
@Override
7791
public boolean equals(Object o) {
7892
if (this == o) {
@@ -84,19 +98,21 @@ public boolean equals(Object o) {
8498
ExpectedFact that = (ExpectedFact) o;
8599
return this.getFile().equals(that.getFile())
86100
&& this.getLineNumber() == that.getLineNumber()
101+
&& Objects.equals(this.testName, that.testName)
87102
&& this.factText.equals(that.factText)
88103
&& this.factLineNumber == that.factLineNumber;
89104
}
90105

91106
@Override
92107
public int hashCode() {
93-
return Objects.hash(getFile(), getLineNumber(), factText);
108+
return Objects.hash(getFile(), getLineNumber(), testName, factText);
94109
}
95110

96111
@Override
97112
public String toString() {
98113
return toStringHelper(this)
99114
.add("file", getFile())
115+
.add("testName", testName)
100116
.add("lineNumber", getLineNumber())
101117
.add("factText", factText)
102118
.add("factLineNumber", factLineNumber)
@@ -108,35 +124,96 @@ static class Reader {
108124

109125
private static final Pattern EXPECTATION_COMMENT =
110126
Pattern.compile(
111-
"\\s*// "
127+
"\\s*// ("
128+
+ "(test:name:(?<testName>.*))"
129+
+ "|"
112130
+ ("(?<fact>"
113131
+ FACT_PATTERNS.stream().map(Pattern::pattern).collect(joining("|"))
114132
+ ")")
115-
+ "\\s*");
133+
+ ")\\s*");
134+
135+
private static final CharMatcher ASCII_DIGIT = CharMatcher.inRange('0', '9');
136+
137+
private final Map<Long, String> facts = new HashMap<>();
138+
private final List<String> errors = new ArrayList<>();
139+
140+
private Path file;
141+
private @Nullable String testName;
142+
private long lineNumber;
116143

117144
/** Reads expected facts from lines in a file. */
118145
ImmutableList<ExpectedFact> readExpectedFacts(Path file, List<String> lines) {
146+
this.file = file;
119147
ImmutableList.Builder<ExpectedFact> expectedFacts = ImmutableList.builder();
120-
Map<Long, String> facts = new HashMap<>();
121148
ListIterator<String> i = lines.listIterator();
122149
while (i.hasNext()) {
123150
String line = i.next();
124-
long lineNumber = i.nextIndex();
151+
lineNumber = i.nextIndex();
125152
Matcher matcher = EXPECTATION_COMMENT.matcher(line);
126153
if (matcher.matches()) {
154+
setTestName(matcher.group("testName"));
127155
String fact = matcher.group("fact");
128156
if (fact != null) {
129157
facts.put(lineNumber, fact.trim());
130158
}
131159
} else {
160+
if (testName != null) {
161+
check(!facts.isEmpty(), "no expected facts for test named %s", testName);
162+
}
132163
facts.forEach(
133164
(factLineNumber, factText) ->
134-
expectedFacts.add(new ExpectedFact(file, lineNumber, factText, factLineNumber)));
165+
expectedFacts.add(
166+
new ExpectedFact(file, lineNumber, testName, factText, factLineNumber)));
135167
facts.clear();
168+
testName = null;
136169
}
137170
}
138-
// TODO(netdpb): Report an error if facts is not empty.
139-
return expectedFacts.build();
171+
return checkUniqueTestNames(expectedFacts.build());
172+
}
173+
174+
private void setTestName(@Nullable String testName) {
175+
if (testName == null) {
176+
return;
177+
}
178+
check(this.testName == null, "test name already set");
179+
check(facts.isEmpty(), "test name must come before assertions for a line");
180+
this.testName = checkTestName(testName.trim());
181+
}
182+
183+
private boolean check(boolean test, String format, Object... args) {
184+
if (!test) {
185+
errors.add(String.format(" %s:%d: %s", file, lineNumber, String.format(format, args)));
186+
}
187+
return test;
188+
}
189+
190+
private String checkTestName(String testName) {
191+
if (check(!testName.isEmpty(), "test name cannot be empty")) {
192+
check(!testName.contains(":"), "test name cannot contain a colon: %s", testName);
193+
check(!ASCII_DIGIT.matchesAllOf(testName), "test name cannot be an integer: %s", testName);
194+
}
195+
return testName;
196+
}
197+
198+
private ImmutableList<ExpectedFact> checkUniqueTestNames(
199+
ImmutableList<ExpectedFact> expectedFacts) {
200+
expectedFacts.stream()
201+
.filter(ef -> ef.testName != null)
202+
.collect(toImmutableSetMultimap(ef -> ef.testName, ExpectedFact::getLineNumber))
203+
.asMap()
204+
.forEach(
205+
(testName, lineNumbers) ->
206+
check(
207+
lineNumbers.size() == 1,
208+
"test name not unique: test '%s' appears on tests of lines %s",
209+
testName,
210+
lineNumbers));
211+
return expectedFacts;
212+
}
213+
214+
/** Throws if there were any errors encountered while reading expected facts. */
215+
void checkErrors() {
216+
checkArgument(errors.isEmpty(), "errors in test inputs:\n%s", Joiner.on('\n').join(errors));
140217
}
141218
}
142219
}

‎conformance-test-framework/src/main/java/org/jspecify/conformance/Fact.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,9 @@ final long getLineNumber() {
3939

4040
/** The text form of the fact. */
4141
protected abstract String getFactText();
42+
43+
/** Returns an object that helps to identify this fact within a file. */
44+
public String getIdentifier() {
45+
return String.valueOf(getLineNumber());
46+
}
4247
}

‎conformance-test-framework/src/test/java/org/jspecify/conformance/ExpectedFactTest.java‎

Lines changed: 108 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import static com.google.common.truth.Truth.assertThat;
1919
import static java.util.Arrays.asList;
20+
import static org.junit.Assert.assertThrows;
2021

2122
import com.google.common.collect.ImmutableList;
2223
import java.nio.file.Path;
@@ -50,12 +51,113 @@ public void readExpectedFacts() {
5051
"// test:irrelevant-annotation:@Nullable ",
5152
"yet another line under test"))
5253
.containsExactly(
53-
new ExpectedFact(FILE, 3, "jspecify_nullness_mismatch", 1),
54-
new ExpectedFact(FILE, 3, "jspecify_nullness_mismatch plus_other_stuff", 2),
55-
new ExpectedFact(FILE, 7, "test:cannot-convert:type1 to type2", 4),
56-
new ExpectedFact(FILE, 7, "test:expression-type:type2:expr2", 5),
57-
new ExpectedFact(FILE, 7, "test:sink-type:type1:expr1", 6),
58-
new ExpectedFact(FILE, 9, "test:irrelevant-annotation:@Nullable", 8))
54+
new ExpectedFact(FILE, 3, null, "jspecify_nullness_mismatch", 1),
55+
new ExpectedFact(FILE, 3, null, "jspecify_nullness_mismatch plus_other_stuff", 2),
56+
new ExpectedFact(FILE, 7, null, "test:cannot-convert:type1 to type2", 4),
57+
new ExpectedFact(FILE, 7, null, "test:expression-type:type2:expr2", 5),
58+
new ExpectedFact(FILE, 7, null, "test:sink-type:type1:expr1", 6),
59+
new ExpectedFact(FILE, 9, null, "test:irrelevant-annotation:@Nullable", 8))
5960
.inOrder();
61+
reader.checkErrors();
62+
}
63+
64+
@Test
65+
public void readExpectedFacts_name() {
66+
assertThat(
67+
readExpectedFacts(
68+
"// test:name: test 1 ",
69+
"// jspecify_nullness_mismatch ",
70+
"// jspecify_nullness_mismatch plus_other_stuff ",
71+
"line under test",
72+
"// test:name: test 2 ",
73+
"// test:cannot-convert:type1 to type2 ",
74+
"// test:expression-type:type2:expr2 ",
75+
"// test:sink-type:type1:expr1 ",
76+
"another line under test",
77+
"// test:name: test 3 ",
78+
"// test:irrelevant-annotation:@Nullable ",
79+
"yet another line under test"))
80+
.containsExactly(
81+
new ExpectedFact(FILE, 4, "test 1", "jspecify_nullness_mismatch", 2),
82+
new ExpectedFact(FILE, 4, "test 1", "jspecify_nullness_mismatch plus_other_stuff", 3),
83+
new ExpectedFact(FILE, 9, "test 2", "test:cannot-convert:type1 to type2", 6),
84+
new ExpectedFact(FILE, 9, "test 2", "test:expression-type:type2:expr2", 7),
85+
new ExpectedFact(FILE, 9, "test 2", "test:sink-type:type1:expr1", 8),
86+
new ExpectedFact(FILE, 12, "test 3", "test:irrelevant-annotation:@Nullable", 11))
87+
.inOrder();
88+
reader.checkErrors();
89+
}
90+
91+
@Test
92+
public void readExpectedFacts_name_empty_throws() {
93+
ImmutableList<ExpectedFact> unused = readExpectedFacts("// test:name: ");
94+
assertThat(assertThrows(IllegalArgumentException.class, () -> reader.checkErrors()))
95+
.hasMessageThat()
96+
.contains("test name cannot be empty");
97+
}
98+
99+
@Test
100+
public void readExpectedFacts_name_allDigits_throws() {
101+
ImmutableList<ExpectedFact> unused = readExpectedFacts("// test:name: 1234 ");
102+
assertThat(assertThrows(IllegalArgumentException.class, () -> reader.checkErrors()))
103+
.hasMessageThat()
104+
.contains("test name cannot be an integer");
105+
}
106+
107+
@Test
108+
public void readExpectedFacts_name_colon_throws() {
109+
ImmutableList<ExpectedFact> unused = readExpectedFacts("// test:name:has a : colon");
110+
assertThat(assertThrows(IllegalArgumentException.class, () -> reader.checkErrors()))
111+
.hasMessageThat()
112+
.contains("test name cannot contain a colon");
113+
}
114+
115+
@Test
116+
public void readExpectedFacts_name_notFirst_throws() {
117+
ImmutableList<ExpectedFact> unused =
118+
readExpectedFacts(
119+
"// test:expression-type:type:expr", //
120+
"// test:name:testName",
121+
"line under test");
122+
assertThat(assertThrows(IllegalArgumentException.class, () -> reader.checkErrors()))
123+
.hasMessageThat()
124+
.contains("test name must come before assertions for a line");
125+
}
126+
127+
@Test
128+
public void readExpectedFacts_name_noFact_throws() {
129+
ImmutableList<ExpectedFact> unused =
130+
readExpectedFacts(
131+
"// test:name:testName", //
132+
"line under test");
133+
assertThat(assertThrows(IllegalArgumentException.class, () -> reader.checkErrors()))
134+
.hasMessageThat()
135+
.contains("no expected facts");
136+
}
137+
138+
@Test
139+
public void readExpectedFacts_name_second_name() {
140+
ImmutableList<ExpectedFact> unused =
141+
readExpectedFacts(
142+
"// test:name:testName1", //
143+
"// test:name:testName2");
144+
assertThat(assertThrows(IllegalArgumentException.class, () -> reader.checkErrors()))
145+
.hasMessageThat()
146+
.contains("test name already set");
147+
}
148+
149+
@Test
150+
public void readExpectedFacts_name_notUnique() {
151+
ImmutableList<ExpectedFact> unused =
152+
readExpectedFacts(
153+
"// test:name:testName",
154+
" // test:expression-type:type1:expr1",
155+
"line 1 under test",
156+
"// test:name:testName",
157+
" // test:expression-type:type2:expr2",
158+
"line 2 under test");
159+
assertThat(assertThrows(IllegalArgumentException.class, () -> reader.checkErrors()))
160+
.hasMessageThat()
161+
.contains("test name not unique: test 'testName' appears on tests of lines [3, 6]");
60162
}
61163
}

‎tests/ConformanceTest-report.txt‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
# 12 pass; 7 fail; 19 total; 63.2% score
2-
PASS: Basic.java:28 test:expression-type:Object?:nullable
3-
PASS: Basic.java:28 test:sink-type:Object!:return
4-
PASS: Basic.java:28 test:cannot-convert:Object? to Object!
5-
PASS: Basic.java:34 test:expression-type:Object!:nonNull
6-
PASS: Basic.java:34 test:sink-type:Object?:return
7-
PASS: Basic.java:41 test:sink-type:Object?:nullableObject
8-
PASS: Basic.java:43 test:sink-type:String!:testSinkType#nonNullString
9-
FAIL: Basic.java:49 test:expression-type:List!<capture of ? extends String?>:nullableStrings
2+
PASS: Basic.java:28:test:expression-type:Object?:nullable
3+
PASS: Basic.java:28:test:sink-type:Object!:return
4+
PASS: Basic.java:28:test:cannot-convert:Object? to Object!
5+
PASS: Basic.java:34:test:expression-type:Object!:nonNull
6+
PASS: Basic.java:34:test:sink-type:Object?:return
7+
PASS: Basic.java:41:test:sink-type:Object?:nullableObject
8+
PASS: Basic.java:43:test:sink-type:String!:testSinkType#nonNullString
9+
FAIL: Basic.java:49:test:expression-type:List!<capture of ? extends String?>:nullableStrings
1010
PASS: Basic.java: no unexpected facts
11-
PASS: Irrelevant.java:28 test:irrelevant-annotation:Nullable
12-
PASS: Irrelevant.java:34 test:irrelevant-annotation:Nullable
13-
FAIL: Irrelevant.java:38 test:irrelevant-annotation:NonNull
14-
FAIL: Irrelevant.java:43 test:irrelevant-annotation:NonNull
15-
FAIL: Irrelevant.java:43 test:irrelevant-annotation:Nullable
16-
FAIL: Irrelevant.java:47 test:irrelevant-annotation:NullMarked
17-
FAIL: Irrelevant.java:49 test:irrelevant-annotation:NullUnmarked
11+
PASS: Irrelevant.java:28:test:irrelevant-annotation:Nullable
12+
PASS: Irrelevant.java:34:test:irrelevant-annotation:Nullable
13+
FAIL: Irrelevant.java:38:test:irrelevant-annotation:NonNull
14+
FAIL: Irrelevant.java:43:test:irrelevant-annotation:NonNull
15+
FAIL: Irrelevant.java:43:test:irrelevant-annotation:Nullable
16+
FAIL: Irrelevant.java:47:test:irrelevant-annotation:NullMarked
17+
FAIL: Irrelevant.java:49:test:irrelevant-annotation:NullUnmarked
1818
FAIL: Irrelevant.java: no unexpected facts
19-
PASS: UsesDep.java:24 test:cannot-convert:null? to Dep*
19+
PASS: UsesDep.java:24:test:cannot-convert:null? to Dep*
2020
PASS: UsesDep.java: no unexpected facts

0 commit comments

Comments
 (0)