Skip to content

Commit 5141fab

Browse files
jmgomezjmgomez
committed
Captures test failures
1 parent 43d3023 commit 5141fab

File tree

4 files changed

+186
-149
lines changed

4 files changed

+186
-149
lines changed

‎protocol/types.nim‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1067,6 +1067,7 @@ type
10671067
RunTestResult* = object
10681068
name*: string
10691069
time*: float
1070+
failure*: Option[string]
10701071

10711072
RunTestSuiteResult* = object
10721073
name*: string

‎testrunner.nim‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ proc parseObject(obj: var object, node: XmlNode) =
6868

6969
proc parseTestResult*(node: XmlNode): RunTestResult =
7070
parseObject(result, node)
71+
# Add handling for failure node
72+
let failureNode = node.child("failure")
73+
if not failureNode.isNil:
74+
result.failure = some failureNode.attr("message")
7175

7276
proc parseTestSuite*(node: XmlNode): RunTestSuiteResult =
7377
parseObject(result, node)
@@ -91,7 +95,6 @@ proc runTests*(entryPoints: seq[string], nimPath: string, suiteName: Option[stri
9195
error "Entry point does not exist", entryPoint = entryPoint
9296
return RunTestProjectResult()
9397

94-
9598
var args = @["c", "-r", entryPoints[0], fmt"--xml:{resultFile}"]
9699
if suiteName.isSome:
97100
args.add(fmt"{suiteName.get()}::")
@@ -108,14 +111,17 @@ proc runTests*(entryPoints: seq[string], nimPath: string, suiteName: Option[stri
108111
)
109112
try:
110113
let res = await process.waitForExit(15.seconds)
111-
if res != 0:
112-
error "Failed to run tests", nimPath = nimPath, entryPoint = entryPoints[0], res = res
113-
error "An error occurred while running tests", error = string.fromBytes(process.stderrStream.read().await)
114+
if not fileExists(resultFile):
115+
let processOutput = string.fromBytes(process.stdoutStream.read().await)
116+
let processError = string.fromBytes(process.stderrStream.read().await)
117+
error "Result file does not exist meaning tests were not run"
118+
error "Output from process", output = processOutput
119+
error "Error from process", error = processError
114120
else:
115-
assert fileExists(resultFile)
116121
let xmlContent = readFile(resultFile)
117122
# echo "XML CONTENT: ", xmlContent
118123
result = parseTestResults(xmlContent)
124+
removeFile(resultFile)
119125
except Exception as e:
120126
let processOutput = string.fromBytes(process.stdoutStream.read().await)
121127
error "An error occurred while running tests", error = e.msg

‎tests/projects/testrunner/tests/failingtest.nim‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import unittest2
2+
3+
suite "Failing Tests":
4+
test "Failing Test":
5+
check(1 == 2)
6+
7+
test "Passing test":
8+
check(1 == 1)
9+

‎tests/textensions.nim‎

Lines changed: 165 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -18,160 +18,181 @@ suite "Nimlangserver extensions":
1818
"extension/statusUpdate", "textDocument/publishDiagnostics", "$/progress",
1919
)
2020

21-
test "calling extension/suggest with restart in the project uri should restart nimsuggest":
22-
let initParams =
23-
InitializeParams %* {
24-
"processId": %getCurrentProcessId(),
25-
"rootUri": fixtureUri("projects/hw/"),
26-
"capabilities":
27-
{"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
28-
}
29-
let initializeResult = waitFor client.initialize(initParams)
30-
31-
check initializeResult.capabilities.textDocumentSync.isSome
32-
33-
let helloWorldUri = fixtureUri("projects/hw/hw.nim")
34-
let helloWorldFile = "projects/hw/hw.nim"
35-
let hwAbsFile = uriToPath(helloWorldFile.fixtureUri())
36-
client.notify("textDocument/didOpen", %createDidOpenParams(helloWorldFile))
21+
# test "calling extension/suggest with restart in the project uri should restart nimsuggest":
22+
# let initParams =
23+
# InitializeParams %* {
24+
# "processId": %getCurrentProcessId(),
25+
# "rootUri": fixtureUri("projects/hw/"),
26+
# "capabilities":
27+
# {"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
28+
# }
29+
# let initializeResult = waitFor client.initialize(initParams)
30+
31+
# check initializeResult.capabilities.textDocumentSync.isSome
32+
33+
# let helloWorldUri = fixtureUri("projects/hw/hw.nim")
34+
# let helloWorldFile = "projects/hw/hw.nim"
35+
# let hwAbsFile = uriToPath(helloWorldFile.fixtureUri())
36+
# client.notify("textDocument/didOpen", %createDidOpenParams(helloWorldFile))
3737

38-
check waitFor client.waitForNotificationMessage(
39-
fmt"Nimsuggest initialized for {hwAbsFile}",
40-
)
41-
42-
client.notify(
43-
"textDocument/didOpen", %createDidOpenParams("projects/hw/useRoot.nim")
44-
)
45-
46-
let prevSuggestPid = ls.projectFiles[hwAbsFile].process.pid
47-
let suggestParams = SuggestParams(action: saRestart, projectFile: hwAbsFile)
48-
let suggestRes = client.call("extension/suggest", %suggestParams).waitFor
49-
let suggestPid = ls.projectFiles[hwAbsFile].process.pid
50-
51-
check prevSuggestPid != suggestPid
52-
53-
test "calling extension/tasks should return all existing tasks":
54-
let initParams =
55-
InitializeParams %* {
56-
"processId": %getCurrentProcessId(),
57-
"rootUri": fixtureUri("projects/tasks/"),
58-
"capabilities":
59-
{"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
60-
}
61-
let initializeResult = waitFor client.initialize(initParams)
62-
63-
let tasksFile = "projects/tasks/src/tasks.nim"
64-
let taskAbsFile = uriToPath(tasksFile.fixtureUri())
65-
client.notify("textDocument/didOpen", %createDidOpenParams(tasksFile))
66-
67-
let tasks = client.call("extension/tasks", jsonutils.toJson(())).waitFor().jsonTo(
68-
seq[NimbleTask]
69-
)
70-
71-
check tasks.len == 3
72-
check tasks[0].name == "helloWorld"
73-
check tasks[0].description == "hello world"
74-
75-
test "calling extension/listTests should return all existing tests":
76-
#We first need to initialize the nimble project
77-
let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner"
78-
cd projectDir:
79-
let (output, _) = execNimble("install", "-l")
80-
discard execNimble("setup")
81-
82-
let initParams =
83-
InitializeParams %* {
84-
"processId": %getCurrentProcessId(),
85-
"rootUri": fixtureUri("projects/testrunner/"),
86-
"capabilities":
87-
{"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
88-
}
89-
let initializeResult = waitFor client.initialize(initParams)
90-
91-
let listTestsParams = ListTestsParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath])
92-
let tests = client.call("extension/listTests", jsonutils.toJson(listTestsParams)).waitFor().jsonTo(
93-
ListTestsResult
94-
)
95-
let testProjectInfo = tests.projectInfo
96-
check testProjectInfo.suites.len == 3
97-
check testProjectInfo.suites["Sample Tests"].tests.len == 1
98-
check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test alone"
99-
check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim"
100-
check testProjectInfo.suites["Sample Tests"].tests[0].line == 4
101-
102-
test "calling extension/runTests should run the tests and return the results":
103-
let initParams =
104-
InitializeParams %* {
105-
"processId": %getCurrentProcessId(),
106-
"rootUri": fixtureUri("projects/testrunner/"),
107-
"capabilities":
108-
{"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
109-
}
110-
let initializeResult = waitFor client.initialize(initParams)
111-
112-
let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath])
113-
let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(
114-
RunTestProjectResult
115-
)
116-
check runTestsRes.suites.len == 4
117-
check runTestsRes.suites[0].name == "Sample Tests"
118-
check runTestsRes.suites[0].tests == 1
119-
check runTestsRes.suites[0].failures == 0
120-
check runTestsRes.suites[0].errors == 0
121-
check runTestsRes.suites[0].skipped == 0
122-
check runTestsRes.suites[0].time > 0.0 and runTestsRes.suites[0].time < 1.0
123-
124-
test "calling extension/runTest with a suite name should run the tests in the suite":
125-
let initParams =
126-
InitializeParams %* {
127-
"processId": %getCurrentProcessId(),
128-
"rootUri": fixtureUri("projects/testrunner/"),
129-
"capabilities":
130-
{"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
131-
}
132-
let initializeResult = waitFor client.initialize(initParams)
133-
134-
let suiteName = "Sample Suite"
135-
let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath], suiteName: suiteName)
136-
let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(
137-
RunTestProjectResult
138-
)
139-
check runTestsRes.suites.len == 1
140-
check runTestsRes.suites[0].name == suiteName
141-
check runTestsRes.suites[0].tests == 3
142-
143-
test "calling extension/runTest with a test name should run the tests in the suite":
38+
# check waitFor client.waitForNotificationMessage(
39+
# fmt"Nimsuggest initialized for {hwAbsFile}",
40+
# )
41+
42+
# client.notify(
43+
# "textDocument/didOpen", %createDidOpenParams("projects/hw/useRoot.nim")
44+
# )
45+
46+
# let prevSuggestPid = ls.projectFiles[hwAbsFile].process.pid
47+
# let suggestParams = SuggestParams(action: saRestart, projectFile: hwAbsFile)
48+
# let suggestRes = client.call("extension/suggest", %suggestParams).waitFor
49+
# let suggestPid = ls.projectFiles[hwAbsFile].process.pid
50+
51+
# check prevSuggestPid != suggestPid
52+
53+
# test "calling extension/tasks should return all existing tasks":
54+
# let initParams =
55+
# InitializeParams %* {
56+
# "processId": %getCurrentProcessId(),
57+
# "rootUri": fixtureUri("projects/tasks/"),
58+
# "capabilities":
59+
# {"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
60+
# }
61+
# let initializeResult = waitFor client.initialize(initParams)
62+
63+
# let tasksFile = "projects/tasks/src/tasks.nim"
64+
# let taskAbsFile = uriToPath(tasksFile.fixtureUri())
65+
# client.notify("textDocument/didOpen", %createDidOpenParams(tasksFile))
66+
67+
# let tasks = client.call("extension/tasks", jsonutils.toJson(())).waitFor().jsonTo(
68+
# seq[NimbleTask]
69+
# )
70+
71+
# check tasks.len == 3
72+
# check tasks[0].name == "helloWorld"
73+
# check tasks[0].description == "hello world"
74+
75+
# test "calling extension/listTests should return all existing tests":
76+
# #We first need to initialize the nimble project
77+
# let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner"
78+
# cd projectDir:
79+
# let (output, _) = execNimble("install", "-l")
80+
# discard execNimble("setup")
81+
82+
# let initParams =
83+
# InitializeParams %* {
84+
# "processId": %getCurrentProcessId(),
85+
# "rootUri": fixtureUri("projects/testrunner/"),
86+
# "capabilities":
87+
# {"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
88+
# }
89+
# let initializeResult = waitFor client.initialize(initParams)
90+
91+
# let listTestsParams = ListTestsParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath])
92+
# let tests = client.call("extension/listTests", jsonutils.toJson(listTestsParams)).waitFor().jsonTo(
93+
# ListTestsResult
94+
# )
95+
# let testProjectInfo = tests.projectInfo
96+
# check testProjectInfo.suites.len == 3
97+
# check testProjectInfo.suites["Sample Tests"].tests.len == 1
98+
# check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test alone"
99+
# check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim"
100+
# check testProjectInfo.suites["Sample Tests"].tests[0].line == 4
101+
102+
# test "calling extension/runTests should run the tests and return the results":
103+
# let initParams =
104+
# InitializeParams %* {
105+
# "processId": %getCurrentProcessId(),
106+
# "rootUri": fixtureUri("projects/testrunner/"),
107+
# "capabilities":
108+
# {"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
109+
# }
110+
# let initializeResult = waitFor client.initialize(initParams)
111+
112+
# let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath])
113+
# let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(
114+
# RunTestProjectResult
115+
# )
116+
# check runTestsRes.suites.len == 4
117+
# check runTestsRes.suites[0].name == "Sample Tests"
118+
# check runTestsRes.suites[0].tests == 1
119+
# check runTestsRes.suites[0].failures == 0
120+
# check runTestsRes.suites[0].errors == 0
121+
# check runTestsRes.suites[0].skipped == 0
122+
# check runTestsRes.suites[0].time > 0.0 and runTestsRes.suites[0].time < 1.0
123+
124+
# test "calling extension/runTest with a suite name should run the tests in the suite":
125+
# let initParams =
126+
# InitializeParams %* {
127+
# "processId": %getCurrentProcessId(),
128+
# "rootUri": fixtureUri("projects/testrunner/"),
129+
# "capabilities":
130+
# {"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
131+
# }
132+
# let initializeResult = waitFor client.initialize(initParams)
133+
134+
# let suiteName = "Sample Suite"
135+
# let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath], suiteName: suiteName)
136+
# let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(
137+
# RunTestProjectResult
138+
# )
139+
# check runTestsRes.suites.len == 1
140+
# check runTestsRes.suites[0].name == suiteName
141+
# check runTestsRes.suites[0].tests == 3
142+
143+
# test "calling extension/runTest with a test name should run the tests in the suite":
144+
# let initParams =
145+
# InitializeParams %* {
146+
# "processId": %getCurrentProcessId(),
147+
# "rootUri": fixtureUri("projects/testrunner/"),
148+
# "capabilities":
149+
# {"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
150+
# }
151+
152+
# let initializeResult = waitFor client.initialize(initParams)
153+
154+
# let testName = "Sample Test"
155+
# let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath], testNames: @[testName])
156+
# let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(RunTestProjectResult)
157+
158+
# check runTestsRes.suites.len == 1
159+
# check runTestsRes.suites[0].tests == 1
160+
# check runTestsRes.suites[0].testResults[0].name == testName
161+
162+
# test "calling extension/runTest with multiple test names should run the tests in the suite":
163+
# let initParams =
164+
# InitializeParams %* {
165+
# "processId": %getCurrentProcessId(),
166+
# "rootUri": fixtureUri("projects/testrunner/"),
167+
# "capabilities":
168+
# {"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
169+
# }
170+
# let initializeResult = waitFor client.initialize(initParams)
171+
172+
# let testNames = @["Sample Test", "Sample Test 2"]
173+
# let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath], testNames: testNames)
174+
# let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(RunTestProjectResult)
175+
176+
# check runTestsRes.suites.len == 1
177+
# check runTestsRes.suites[0].tests == 2
178+
179+
test "calling extension/runTest with a failing test should return the failure":
144180
let initParams =
145181
InitializeParams %* {
146182
"processId": %getCurrentProcessId(),
147183
"rootUri": fixtureUri("projects/testrunner/"),
148184
"capabilities":
149185
{"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
150186
}
151-
187+
152188
let initializeResult = waitFor client.initialize(initParams)
153189

154-
let testName = "Sample Test"
155-
let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath], testNames: @[testName])
190+
let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/failingtest.nim".absolutePath])
156191
let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(RunTestProjectResult)
157192

158193
check runTestsRes.suites.len == 1
159-
check runTestsRes.suites[0].tests == 1
160-
check runTestsRes.suites[0].testResults[0].name == testName
161-
162-
test "calling extension/runTest with multiple test names should run the tests in the suite":
163-
let initParams =
164-
InitializeParams %* {
165-
"processId": %getCurrentProcessId(),
166-
"rootUri": fixtureUri("projects/testrunner/"),
167-
"capabilities":
168-
{"window": {"workDoneProgress": true}, "workspace": {"configuration": true}},
169-
}
170-
let initializeResult = waitFor client.initialize(initParams)
171-
172-
let testNames = @["Sample Test", "Sample Test 2"]
173-
let runTestsParams = RunTestParams(entryPoints: @["tests/projects/testrunner/tests/sampletests.nim".absolutePath], testNames: testNames)
174-
let runTestsRes = client.call("extension/runTests", jsonutils.toJson(runTestsParams)).waitFor().jsonTo(RunTestProjectResult)
175-
176-
check runTestsRes.suites.len == 1
177-
check runTestsRes.suites[0].tests == 2
194+
check runTestsRes.suites[0].name == "Failing Tests"
195+
check runTestsRes.suites[0].tests == 2
196+
check runTestsRes.suites[0].failures == 1
197+
check runTestsRes.suites[0].testResults[0].name == "Failing Test"
198+
check runTestsRes.suites[0].testResults[0].failure.isSome

0 commit comments

Comments
 (0)