Skip to content

Commit 8bea205

Browse files
KyleKincerKyleKincer
committed
Ensure skipped tests not marked as passed
1 parent 9ea92ee commit 8bea205

File tree

6 files changed

+172
-103
lines changed

6 files changed

+172
-103
lines changed

‎docs/guide.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,20 @@ Function test_file_system_access($t : cs.Testing)
362362
- Whitespace around tags is automatically trimmed
363363
- Tests without explicit tags automatically receive the "unit" tag
364364

365+
### Skipping Tests
366+
367+
Mark a test with the `skip` tag to prevent it from running while still
368+
being reported in the overall test statistics:
369+
370+
```4d
371+
// #tags: unit, skip
372+
Function test_pending_feature($t : cs.Testing)
373+
$t.assert.fail($t; "This code is not ready")
374+
```
375+
376+
Skipped tests are counted in the totals and listed separately, but they
377+
do not affect the pass rate.
378+
365379
### Tag Filtering Commands
366380

367381
#### Include Tags (OR Logic)

‎testing/Project/Sources/Classes/TestRunner.4dm‎

Lines changed: 73 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,13 @@ Function _logHeader()
116116

117117
Function _collectSuiteResults($testSuite : cs:C1710._TestSuite)
118118
var $suiteResult : Object
119-
$suiteResult:=New object:C1471(\
120-
"name"; $testSuite.class.name; \
121-
"tests"; []; \
122-
"passed"; 0; \
123-
"failed"; 0\
124-
)
119+
$suiteResult:=New object:C1471(\
120+
"name"; $testSuite.class.name; \
121+
"tests"; []; \
122+
"passed"; 0; \
123+
"failed"; 0; \
124+
"skipped"; 0\
125+
)
125126

126127
var $testFunction : cs:C1710._TestFunction
127128
For each ($testFunction; $testSuite.testFunctions)
@@ -130,29 +131,37 @@ Function _collectSuiteResults($testSuite : cs:C1710._TestSuite)
130131

131132
This:C1470.results.totalTests+=1
132133

133-
If ($testResult.passed)
134-
This:C1470.results.passed+=1
135-
$suiteResult.passed+=1
136-
If (This:C1470.outputFormat="human")
137-
LOG EVENT:C667(Into system standard outputs:K38:9; " ✓ "+$testResult.name+" ("+String:C10($testResult.duration)+"ms)\r\n"; Information message:K38:1)
138-
End if
139-
Else
140-
This:C1470.results.failed+=1
141-
$suiteResult.failed+=1
142-
This:C1470.results.failedTests.push($testResult)
143-
If (This:C1470.outputFormat="human")
144-
var $errorDetails : Text
145-
$errorDetails:=""
146-
If ($testResult.runtimeErrors.length>0)
147-
$errorDetails:=" [Runtime Error: "+$testResult.runtimeErrors[0].text+"]"
148-
Else
149-
If ($testResult.logMessages.length>0)
150-
$errorDetails:=" ["+$testResult.logMessages[0]+"]"
151-
End if
152-
End if
153-
LOG EVENT:C667(Into system standard outputs:K38:9; " ✗ "+$testResult.name+" ("+String:C10($testResult.duration)+"ms)"+$errorDetails+"\r\n"; Error message:K38:3)
154-
End if
155-
End if
134+
If ($testResult.skipped)
135+
This:C1470.results.skipped+=1
136+
$suiteResult.skipped+=1
137+
If (This:C1470.outputFormat="human")
138+
LOG EVENT:C667(Into system standard outputs:K38:9; " - "+$testResult.name+" (skipped)\r\n"; Information message:K38:1)
139+
End if
140+
Else
141+
If ($testResult.passed)
142+
This:C1470.results.passed+=1
143+
$suiteResult.passed+=1
144+
If (This:C1470.outputFormat="human")
145+
LOG EVENT:C667(Into system standard outputs:K38:9; " ✓ "+$testResult.name+" ("+String:C10($testResult.duration)+"ms)\r\n"; Information message:K38:1)
146+
End if
147+
Else
148+
This:C1470.results.failed+=1
149+
$suiteResult.failed+=1
150+
This:C1470.results.failedTests.push($testResult)
151+
If (This:C1470.outputFormat="human")
152+
var $errorDetails : Text
153+
$errorDetails:=""
154+
If ($testResult.runtimeErrors.length>0)
155+
$errorDetails:=" [Runtime Error: "+$testResult.runtimeErrors[0].text+"]"
156+
Else
157+
If ($testResult.logMessages.length>0)
158+
$errorDetails:=" ["+$testResult.logMessages[0]+"]"
159+
End if
160+
End if
161+
LOG EVENT:C667(Into system standard outputs:K38:9; " ✗ "+$testResult.name+" ("+String:C10($testResult.duration)+"ms)"+$errorDetails+"\r\n"; Error message:K38:3)
162+
End if
163+
End if
164+
End if
156165

157166
$suiteResult.tests.push($testResult)
158167
End for each
@@ -168,18 +177,21 @@ Function _generateReport()
168177

169178
Function _generateHumanReport()
170179
var $passRate : Real
171-
If (This:C1470.results.totalTests>0)
172-
$passRate:=(This:C1470.results.passed/This:C1470.results.totalTests)*100
173-
Else
174-
$passRate:=0
175-
End if
180+
var $effectiveTotal : Integer
181+
$effectiveTotal:=This:C1470.results.totalTests-This:C1470.results.skipped
182+
If ($effectiveTotal>0)
183+
$passRate:=(This:C1470.results.passed/$effectiveTotal)*100
184+
Else
185+
$passRate:=0
186+
End if
176187

177188
LOG EVENT:C667(Into system standard outputs:K38:9; "\r\n"; Information message:K38:1)
178189
LOG EVENT:C667(Into system standard outputs:K38:9; "=== Test Results Summary ===\r\n"; Information message:K38:1)
179-
LOG EVENT:C667(Into system standard outputs:K38:9; "Total Tests: "+String:C10(This:C1470.results.totalTests)+"\r\n"; Information message:K38:1)
180-
LOG EVENT:C667(Into system standard outputs:K38:9; "Passed: "+String:C10(This:C1470.results.passed)+"\r\n"; Information message:K38:1)
181-
LOG EVENT:C667(Into system standard outputs:K38:9; "Failed: "+String:C10(This:C1470.results.failed)+"\r\n"; Information message:K38:1)
182-
LOG EVENT:C667(Into system standard outputs:K38:9; "Pass Rate: "+String:C10($passRate; "##0.0")+"%\r\n"; Information message:K38:1)
190+
LOG EVENT:C667(Into system standard outputs:K38:9; "Total Tests: "+String:C10(This:C1470.results.totalTests)+"\r\n"; Information message:K38:1)
191+
LOG EVENT:C667(Into system standard outputs:K38:9; "Passed: "+String:C10(This:C1470.results.passed)+"\r\n"; Information message:K38:1)
192+
LOG EVENT:C667(Into system standard outputs:K38:9; "Failed: "+String:C10(This:C1470.results.failed)+"\r\n"; Information message:K38:1)
193+
LOG EVENT:C667(Into system standard outputs:K38:9; "Skipped: "+String:C10(This:C1470.results.skipped)+"\r\n"; Information message:K38:1)
194+
LOG EVENT:C667(Into system standard outputs:K38:9; "Pass Rate: "+String:C10($passRate; "##0.0")+"%\r\n"; Information message:K38:1)
183195
LOG EVENT:C667(Into system standard outputs:K38:9; "Duration: "+String:C10(This:C1470.results.duration)+"ms\r\n"; Information message:K38:1)
184196

185197
If (This:C1470.results.failed>0)
@@ -206,12 +218,14 @@ Function _generateHumanReport()
206218
This:C1470._logFooter()
207219

208220
Function _generateJSONReport()
209-
var $passRate : Real
210-
If (This:C1470.results.totalTests>0)
211-
$passRate:=(This:C1470.results.passed/This:C1470.results.totalTests)*100
212-
Else
213-
$passRate:=0
214-
End if
221+
var $passRate : Real
222+
var $effectiveTotal : Integer
223+
$effectiveTotal:=This:C1470.results.totalTests-This:C1470.results.skipped
224+
If ($effectiveTotal>0)
225+
$passRate:=(This:C1470.results.passed/$effectiveTotal)*100
226+
Else
227+
$passRate:=0
228+
End if
215229

216230
var $jsonReport : Object
217231

@@ -222,14 +236,15 @@ Function _generateJSONReport()
222236
$jsonReport.status:=(This:C1470.results.failed=0) ? "success" : "failure"
223237
Else
224238
// Terse mode: minimal information
225-
$jsonReport:=New object:C1471(\
226-
"tests"; This:C1470.results.totalTests; \
227-
"passed"; This:C1470.results.passed; \
228-
"failed"; This:C1470.results.failed; \
229-
"rate"; Round:C94($passRate; 1); \
230-
"duration"; This:C1470.results.duration; \
231-
"status"; (This:C1470.results.failed=0) ? "ok" : "fail"\
232-
)
239+
$jsonReport:=New object:C1471(\
240+
"tests"; This:C1470.results.totalTests; \
241+
"passed"; This:C1470.results.passed; \
242+
"failed"; This:C1470.results.failed; \
243+
"skipped"; This:C1470.results.skipped; \
244+
"rate"; Round:C94($passRate; 1); \
245+
"duration"; This:C1470.results.duration; \
246+
"status"; (This:C1470.results.failed=0) ? "ok" : "fail"\
247+
)
233248

234249
// Only include failed tests if there are any
235250
If (This:C1470.results.failed>0)
@@ -258,11 +273,12 @@ Function _generateJSONReport()
258273
$suiteSummary:=[]
259274
var $suite : Object
260275
For each ($suite; This:C1470.results.suites)
261-
$suiteSummary.push(New object:C1471(\
262-
"name"; $suite.name; \
263-
"passed"; $suite.passed; \
264-
"failed"; $suite.failed\
265-
))
276+
$suiteSummary.push(New object:C1471(\
277+
"name"; $suite.name; \
278+
"passed"; $suite.passed; \
279+
"failed"; $suite.failed; \
280+
"skipped"; $suite.skipped\
281+
))
266282
End for each
267283
$jsonReport.suites:=$suiteSummary
268284
End if

‎testing/Project/Sources/Classes/_SkipTaggedTest.4dm‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
// Test class with a skipped test to verify skip functionality
2+
Class constructor()
3+
4+
// #tags: unit, skip
5+
Function test_should_be_skipped($t : cs:C1710.Testing)
6+
$t.assert.fail($t; "This test should be skipped and not executed")

‎testing/Project/Sources/Classes/_TestFunction.4dm‎

Lines changed: 44 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,40 @@ property t : cs:C1710.Testing
66
property startTime : Integer
77
property endTime : Integer
88
property runtimeErrors : Collection
9+
property skipped : Boolean
910
property tags : Collection // Collection of tag strings
1011
property useTransactions : Boolean // Whether to auto-manage transactions for this test
1112

1213
Class constructor($class : 4D:C1709.Class; $classInstance : 4D:C1709.Object; $function : 4D:C1709.Function; $name : Text; $classCode : Text)
1314
This:C1470.class:=$class
1415
This:C1470.classInstance:=$classInstance
1516
This:C1470.function:=$function
16-
This:C1470.functionName:=$name
17-
This:C1470.t:=cs:C1710.Testing.new()
18-
This:C1470.runtimeErrors:=[]
19-
This:C1470.tags:=This:C1470._parseTags($classCode || "")
20-
This:C1470.useTransactions:=This:C1470._shouldUseTransactions($classCode || "")
17+
This:C1470.functionName:=$name
18+
This:C1470.t:=cs:C1710.Testing.new()
19+
This:C1470.runtimeErrors:=[]
20+
This:C1470.skipped:=False:C215
21+
This:C1470.tags:=This:C1470._parseTags($classCode || "")
22+
This:C1470.useTransactions:=This:C1470._shouldUseTransactions($classCode || "")
2123

2224
Function run()
23-
This:C1470.startTime:=Milliseconds:C459
24-
25-
// Reset the testing context for this test
26-
This:C1470.t.resetForNewTest()
27-
28-
// Clear any existing test errors
29-
If (Storage:C1525.testErrors#Null:C1517)
30-
Use (Storage:C1525)
31-
Storage:C1525.testErrors.clear()
32-
End use
33-
End if
25+
This:C1470.startTime:=Milliseconds:C459
26+
27+
// Reset the testing context for this test
28+
This:C1470.t.resetForNewTest()
29+
30+
// Clear any existing test errors
31+
If (Storage:C1525.testErrors#Null:C1517)
32+
Use (Storage:C1525)
33+
Storage:C1525.testErrors.clear()
34+
End use
35+
End if
36+
37+
// Skip test early if tagged to skip
38+
If (This:C1470.shouldSkip())
39+
This:C1470.skipped:=True:C214
40+
This:C1470.endTime:=Milliseconds:C459
41+
return
42+
End if
3443

3544
// Start transaction if configured to use transactions
3645
var $transactionStarted : Boolean
@@ -77,22 +86,26 @@ Function run()
7786
End if
7887
End if
7988

80-
This:C1470.endTime:=Milliseconds:C459
81-
89+
This:C1470.endTime:=Milliseconds:C459
90+
8291
Function getResult() : Object
83-
var $duration : Integer
84-
$duration:=This:C1470.endTime-This:C1470.startTime
85-
86-
return New object:C1471(\
87-
"name"; This:C1470.functionName; \
88-
"passed"; Not:C34(This:C1470.t.failed); \
89-
"failed"; This:C1470.t.failed; \
90-
"duration"; $duration; \
91-
"suite"; This:C1470.class.name; \
92-
"runtimeErrors"; This:C1470.runtimeErrors; \
93-
"logMessages"; This:C1470.t.logMessages; \
94-
"tags"; This:C1470.tags\
95-
)
92+
var $duration : Integer
93+
$duration:=This:C1470.endTime-This:C1470.startTime
94+
95+
return New object:C1471(\
96+
"name"; This:C1470.functionName; \
97+
"passed"; Not:C34(This:C1470.t.failed) && Not:C34(This:C1470.skipped); \
98+
"failed"; This:C1470.t.failed; \
99+
"skipped"; This:C1470.skipped; \
100+
"duration"; $duration; \
101+
"suite"; This:C1470.class.name; \
102+
"runtimeErrors"; This:C1470.runtimeErrors; \
103+
"logMessages"; This:C1470.t.logMessages; \
104+
"tags"; This:C1470.tags\
105+
)
106+
107+
Function shouldSkip() : Boolean
108+
return (This:C1470.tags.indexOf("skip")>=0)
96109

97110
Function _parseTags($classCode : Text) : Collection
98111
// Parse tags from function comments in source code

‎testing/Project/Sources/Classes/_TestRunnerTest.4dm‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -321,8 +321,24 @@ Function test_error_handling_in_extracted_methods($t : cs:C1710.Testing)
321321
"InvalidTest"; New object:C1471("name"; "InvalidTest"); \
322322
"ValidTest"; New object:C1471("name"; "ValidTest"; "superclass"; New object:C1471("name"; "Object"))\
323323
)
324-
var $filteredClasses : Collection
325-
$filteredClasses:=$runner._filterTestClasses($malformedStore)
326-
// Should find ValidTest but skip InvalidTest (missing superclass)
327-
$t.assert.areEqual($t; 1; $filteredClasses.length; "Should handle classes without superclass gracefully")
328-
$t.assert.areEqual($t; "ValidTest"; $filteredClasses[0].name; "Should include ValidTest")
324+
var $filteredClasses : Collection
325+
$filteredClasses:=$runner._filterTestClasses($malformedStore)
326+
// Should find ValidTest but skip InvalidTest (missing superclass)
327+
$t.assert.areEqual($t; 1; $filteredClasses.length; "Should handle classes without superclass gracefully")
328+
$t.assert.areEqual($t; "ValidTest"; $filteredClasses[0].name; "Should include ValidTest")
329+
330+
Function test_skip_tag_counts_as_skipped($t : cs:C1710.Testing)
331+
332+
// Run TestRunner on a class that should be skipped
333+
var $runner : cs:C1710.TestRunner
334+
$runner:=cs:C1710.TestRunner.new()
335+
$runner.testPatterns:=["_SkipTaggedTest*"]
336+
$runner.run()
337+
338+
var $results : Object
339+
$results:=$runner.getResults()
340+
341+
$t.assert.areEqual($t; 1; $results.totalTests; "Total should count skipped test")
342+
$t.assert.areEqual($t; 1; $results.skipped; "Skipped test should be counted")
343+
$t.assert.areEqual($t; 0; $results.failed; "Skipped test should not fail")
344+
$t.assert.areEqual($t; 0; $results.passed; "Skipped test should not pass")

‎testing/Project/Sources/Classes/_TestSuite.4dm‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,20 @@ Class constructor($class : 4D:C1709.Class; $outputFormat : Text; $testPatterns :
1616
This:C1470.discoverTests()
1717

1818
Function run()
19-
This:C1470._callSetup()
20-
21-
var $testFunction : cs:C1710._TestFunction
22-
For each ($testFunction; This:C1470.testFunctions)
23-
This:C1470._callBeforeEach()
24-
$testFunction.run()
25-
This:C1470._callAfterEach()
26-
End for each
27-
28-
This:C1470._callTeardown()
19+
This:C1470._callSetup()
20+
21+
var $testFunction : cs:C1710._TestFunction
22+
For each ($testFunction; This:C1470.testFunctions)
23+
If ($testFunction.shouldSkip())
24+
$testFunction.run()
25+
Else
26+
This:C1470._callBeforeEach()
27+
$testFunction.run()
28+
This:C1470._callAfterEach()
29+
End if
30+
End for each
31+
32+
This:C1470._callTeardown()
2933

3034
Function discoverTests()
3135
var $testFunctions : Collection

0 commit comments

Comments
 (0)