Skip to content

Commit 517f2d0

Browse files
hmachmac
committed
Add optional results to InlineExpectationsTest
The idea behind optional results is that there may be instances where each line of source code has many results and you don't want to annotate all of them, but you still want to ensure that any annotations you do have are correct. This change makes that possible by exposing a new predicate `hasOptionalResult`, which has the same signature as `hasResult`. Results produced by `hasOptionalResult` will be matched against any annotations, but the lack of a matching annotation will not cause a failure. We will use this in the inline tests for the API edge getASubclass, because for each API path that uses getASubclass there is always a shorter path that does not use it, and thus we can't use the normal shortest-path matching approach that works for other API Graph tests.
1 parent d0a274c commit 517f2d0

File tree

5 files changed

+105
-20
lines changed

5 files changed

+105
-20
lines changed

‎cpp/ql/test/TestUtilities/InlineExpectationsTest.qll

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,15 @@ abstract class InlineExpectationsTest extends string {
123123
*/
124124
abstract predicate hasActualResult(Location location, string element, string tag, string value);
125125

126+
/**
127+
* Like `hasActualResult`, but returns results that do not require a matching annotation.
128+
* A failure will still arise if there is an annotation that does not match any results, but not vice versa.
129+
* Override this predicate to specify optional results.
130+
*/
131+
predicate hasOptionalResult(Location location, string element, string tag, string value) {
132+
none()
133+
}
134+
126135
final predicate hasFailureMessage(FailureLocatable element, string message) {
127136
exists(ActualResult actualResult |
128137
actualResult.getTest() = this and
@@ -134,7 +143,8 @@ abstract class InlineExpectationsTest extends string {
134143
)
135144
or
136145
not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
137-
message = "Unexpected result: " + actualResult.getExpectationText()
146+
message = "Unexpected result: " + actualResult.getExpectationText() and
147+
not actualResult.isOptional()
138148
)
139149
)
140150
or
@@ -243,9 +253,13 @@ private string expectationPattern() {
243253

244254
private newtype TFailureLocatable =
245255
TActualResult(
246-
InlineExpectationsTest test, Location location, string element, string tag, string value
256+
InlineExpectationsTest test, Location location, string element, string tag, string value,
257+
boolean optional
247258
) {
248-
test.hasActualResult(location, element, tag, value)
259+
test.hasActualResult(location, element, tag, value) and
260+
optional = false
261+
or
262+
test.hasOptionalResult(location, element, tag, value) and optional = true
249263
} or
250264
TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
251265
exists(TColumn column, string tags |
@@ -277,8 +291,9 @@ class ActualResult extends FailureLocatable, TActualResult {
277291
string element;
278292
string tag;
279293
string value;
294+
boolean optional;
280295

281-
ActualResult() { this = TActualResult(test, location, element, tag, value) }
296+
ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
282297

283298
override string toString() { result = element }
284299

@@ -289,6 +304,8 @@ class ActualResult extends FailureLocatable, TActualResult {
289304
override string getTag() { result = tag }
290305

291306
override string getValue() { result = value }
307+
308+
predicate isOptional() { optional = true }
292309
}
293310

294311
abstract private class Expectation extends FailureLocatable {

‎csharp/ql/test/TestUtilities/InlineExpectationsTest.qll

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,15 @@ abstract class InlineExpectationsTest extends string {
123123
*/
124124
abstract predicate hasActualResult(Location location, string element, string tag, string value);
125125

126+
/**
127+
* Like `hasActualResult`, but returns results that do not require a matching annotation.
128+
* A failure will still arise if there is an annotation that does not match any results, but not vice versa.
129+
* Override this predicate to specify optional results.
130+
*/
131+
predicate hasOptionalResult(Location location, string element, string tag, string value) {
132+
none()
133+
}
134+
126135
final predicate hasFailureMessage(FailureLocatable element, string message) {
127136
exists(ActualResult actualResult |
128137
actualResult.getTest() = this and
@@ -134,7 +143,8 @@ abstract class InlineExpectationsTest extends string {
134143
)
135144
or
136145
not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
137-
message = "Unexpected result: " + actualResult.getExpectationText()
146+
message = "Unexpected result: " + actualResult.getExpectationText() and
147+
not actualResult.isOptional()
138148
)
139149
)
140150
or
@@ -243,9 +253,13 @@ private string expectationPattern() {
243253

244254
private newtype TFailureLocatable =
245255
TActualResult(
246-
InlineExpectationsTest test, Location location, string element, string tag, string value
256+
InlineExpectationsTest test, Location location, string element, string tag, string value,
257+
boolean optional
247258
) {
248-
test.hasActualResult(location, element, tag, value)
259+
test.hasActualResult(location, element, tag, value) and
260+
optional = false
261+
or
262+
test.hasOptionalResult(location, element, tag, value) and optional = true
249263
} or
250264
TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
251265
exists(TColumn column, string tags |
@@ -277,8 +291,9 @@ class ActualResult extends FailureLocatable, TActualResult {
277291
string element;
278292
string tag;
279293
string value;
294+
boolean optional;
280295

281-
ActualResult() { this = TActualResult(test, location, element, tag, value) }
296+
ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
282297

283298
override string toString() { result = element }
284299

@@ -289,6 +304,8 @@ class ActualResult extends FailureLocatable, TActualResult {
289304
override string getTag() { result = tag }
290305

291306
override string getValue() { result = value }
307+
308+
predicate isOptional() { optional = true }
292309
}
293310

294311
abstract private class Expectation extends FailureLocatable {

‎java/ql/test/TestUtilities/InlineExpectationsTest.qll

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,15 @@ abstract class InlineExpectationsTest extends string {
123123
*/
124124
abstract predicate hasActualResult(Location location, string element, string tag, string value);
125125

126+
/**
127+
* Like `hasActualResult`, but returns results that do not require a matching annotation.
128+
* A failure will still arise if there is an annotation that does not match any results, but not vice versa.
129+
* Override this predicate to specify optional results.
130+
*/
131+
predicate hasOptionalResult(Location location, string element, string tag, string value) {
132+
none()
133+
}
134+
126135
final predicate hasFailureMessage(FailureLocatable element, string message) {
127136
exists(ActualResult actualResult |
128137
actualResult.getTest() = this and
@@ -134,7 +143,8 @@ abstract class InlineExpectationsTest extends string {
134143
)
135144
or
136145
not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
137-
message = "Unexpected result: " + actualResult.getExpectationText()
146+
message = "Unexpected result: " + actualResult.getExpectationText() and
147+
not actualResult.isOptional()
138148
)
139149
)
140150
or
@@ -243,9 +253,13 @@ private string expectationPattern() {
243253

244254
private newtype TFailureLocatable =
245255
TActualResult(
246-
InlineExpectationsTest test, Location location, string element, string tag, string value
256+
InlineExpectationsTest test, Location location, string element, string tag, string value,
257+
boolean optional
247258
) {
248-
test.hasActualResult(location, element, tag, value)
259+
test.hasActualResult(location, element, tag, value) and
260+
optional = false
261+
or
262+
test.hasOptionalResult(location, element, tag, value) and optional = true
249263
} or
250264
TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
251265
exists(TColumn column, string tags |
@@ -277,8 +291,9 @@ class ActualResult extends FailureLocatable, TActualResult {
277291
string element;
278292
string tag;
279293
string value;
294+
boolean optional;
280295

281-
ActualResult() { this = TActualResult(test, location, element, tag, value) }
296+
ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
282297

283298
override string toString() { result = element }
284299

@@ -289,6 +304,8 @@ class ActualResult extends FailureLocatable, TActualResult {
289304
override string getTag() { result = tag }
290305

291306
override string getValue() { result = value }
307+
308+
predicate isOptional() { optional = true }
292309
}
293310

294311
abstract private class Expectation extends FailureLocatable {

‎python/ql/test/TestUtilities/InlineExpectationsTest.qll

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,15 @@ abstract class InlineExpectationsTest extends string {
123123
*/
124124
abstract predicate hasActualResult(Location location, string element, string tag, string value);
125125

126+
/**
127+
* Like `hasActualResult`, but returns results that do not require a matching annotation.
128+
* A failure will still arise if there is an annotation that does not match any results, but not vice versa.
129+
* Override this predicate to specify optional results.
130+
*/
131+
predicate hasOptionalResult(Location location, string element, string tag, string value) {
132+
none()
133+
}
134+
126135
final predicate hasFailureMessage(FailureLocatable element, string message) {
127136
exists(ActualResult actualResult |
128137
actualResult.getTest() = this and
@@ -134,7 +143,8 @@ abstract class InlineExpectationsTest extends string {
134143
)
135144
or
136145
not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
137-
message = "Unexpected result: " + actualResult.getExpectationText()
146+
message = "Unexpected result: " + actualResult.getExpectationText() and
147+
not actualResult.isOptional()
138148
)
139149
)
140150
or
@@ -243,9 +253,13 @@ private string expectationPattern() {
243253

244254
private newtype TFailureLocatable =
245255
TActualResult(
246-
InlineExpectationsTest test, Location location, string element, string tag, string value
256+
InlineExpectationsTest test, Location location, string element, string tag, string value,
257+
boolean optional
247258
) {
248-
test.hasActualResult(location, element, tag, value)
259+
test.hasActualResult(location, element, tag, value) and
260+
optional = false
261+
or
262+
test.hasOptionalResult(location, element, tag, value) and optional = true
249263
} or
250264
TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
251265
exists(TColumn column, string tags |
@@ -277,8 +291,9 @@ class ActualResult extends FailureLocatable, TActualResult {
277291
string element;
278292
string tag;
279293
string value;
294+
boolean optional;
280295

281-
ActualResult() { this = TActualResult(test, location, element, tag, value) }
296+
ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
282297

283298
override string toString() { result = element }
284299

@@ -289,6 +304,8 @@ class ActualResult extends FailureLocatable, TActualResult {
289304
override string getTag() { result = tag }
290305

291306
override string getValue() { result = value }
307+
308+
predicate isOptional() { optional = true }
292309
}
293310

294311
abstract private class Expectation extends FailureLocatable {

‎ruby/ql/test/TestUtilities/InlineExpectationsTest.qll

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,15 @@ abstract class InlineExpectationsTest extends string {
123123
*/
124124
abstract predicate hasActualResult(Location location, string element, string tag, string value);
125125

126+
/**
127+
* Like `hasActualResult`, but returns results that do not require a matching annotation.
128+
* A failure will still arise if there is an annotation that does not match any results, but not vice versa.
129+
* Override this predicate to specify optional results.
130+
*/
131+
predicate hasOptionalResult(Location location, string element, string tag, string value) {
132+
none()
133+
}
134+
126135
final predicate hasFailureMessage(FailureLocatable element, string message) {
127136
exists(ActualResult actualResult |
128137
actualResult.getTest() = this and
@@ -134,7 +143,8 @@ abstract class InlineExpectationsTest extends string {
134143
)
135144
or
136145
not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
137-
message = "Unexpected result: " + actualResult.getExpectationText()
146+
message = "Unexpected result: " + actualResult.getExpectationText() and
147+
not actualResult.isOptional()
138148
)
139149
)
140150
or
@@ -243,9 +253,13 @@ private string expectationPattern() {
243253

244254
private newtype TFailureLocatable =
245255
TActualResult(
246-
InlineExpectationsTest test, Location location, string element, string tag, string value
256+
InlineExpectationsTest test, Location location, string element, string tag, string value,
257+
boolean optional
247258
) {
248-
test.hasActualResult(location, element, tag, value)
259+
test.hasActualResult(location, element, tag, value) and
260+
optional = false
261+
or
262+
test.hasOptionalResult(location, element, tag, value) and optional = true
249263
} or
250264
TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
251265
exists(TColumn column, string tags |
@@ -277,8 +291,9 @@ class ActualResult extends FailureLocatable, TActualResult {
277291
string element;
278292
string tag;
279293
string value;
294+
boolean optional;
280295

281-
ActualResult() { this = TActualResult(test, location, element, tag, value) }
296+
ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
282297

283298
override string toString() { result = element }
284299

@@ -289,6 +304,8 @@ class ActualResult extends FailureLocatable, TActualResult {
289304
override string getTag() { result = tag }
290305

291306
override string getValue() { result = value }
307+
308+
predicate isOptional() { optional = true }
292309
}
293310

294311
abstract private class Expectation extends FailureLocatable {

0 commit comments

Comments
 (0)