Skip to content

Commit 638117d

Browse files
vchuravyvchuravy
committed
Support custom records
1 parent 26a0984 commit 638117d

File tree

3 files changed

+106
-12
lines changed

3 files changed

+106
-12
lines changed

‎Project.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "ParallelTestRunner"
22
uuid = "d3525ed8-44d0-4b2c-a655-542cee43accc"
33
authors = ["Valentin Churavy <[email protected]>"]
4-
version = "1.0.2"
4+
version = "1.1.0"
55

66
[deps]
77
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

‎src/ParallelTestRunner.jl‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
module ParallelTestRunner
22

33
export runtests, addworkers, addworker
4+
public extract_flag!
45

56
using Distributed
67
using Dates
@@ -100,7 +101,7 @@ struct TestIOContext
100101
rss_align::Int
101102
end
102103

103-
function test_IOContext(::Type{TestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int)
104+
function test_IOContext(::Type{<:AbstractTestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int)
104105
elapsed_align = textwidth("Time (s)")
105106
gc_align = textwidth("GC (s)")
106107
percent_align = textwidth("GC %")
@@ -115,7 +116,7 @@ function test_IOContext(::Type{TestRecord}, stdout::IO, stderr::IO, lock::Reentr
115116
)
116117
end
117118

118-
function print_header(::Type{TestRecord}, ctx::TestIOContext, testgroupheader, workerheader)
119+
function print_header(::Type{<:AbstractTestRecord}, ctx::TestIOContext, testgroupheader, workerheader)
119120
lock(ctx.lock)
120121
try
121122
printstyled(ctx.stdout, " "^(ctx.name_align + textwidth(testgroupheader) - 3), " │ ")
@@ -129,7 +130,7 @@ function print_header(::Type{TestRecord}, ctx::TestIOContext, testgroupheader, w
129130
end
130131
end
131132

132-
function print_test_started(::Type{TestRecord}, wrkr, test, ctx::TestIOContext)
133+
function print_test_started(::Type{<:AbstractTestRecord}, wrkr, test, ctx::TestIOContext)
133134
lock(ctx.lock)
134135
try
135136
printstyled(ctx.stdout, test, lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " "), " │", color = :white)
@@ -143,7 +144,7 @@ function print_test_started(::Type{TestRecord}, wrkr, test, ctx::TestIOContext)
143144
end
144145
end
145146

146-
function print_test_finished(record::TestRecord, wrkr, test, ctx::TestIOContext)
147+
function print_test_finished(record::AbstractTestRecord, wrkr, test, ctx::TestIOContext)
147148
lock(ctx.lock)
148149
try
149150
printstyled(ctx.stdout, test, color = :white)
@@ -158,7 +159,7 @@ function print_test_finished(record::TestRecord, wrkr, test, ctx::TestIOContext)
158159
alloc_str = @sprintf("%5.2f", record.bytes / 2^20)
159160
printstyled(ctx.stdout, lpad(alloc_str, ctx.alloc_align, " "), " │ ", color = :white)
160161

161-
rss_str = @sprintf("%5.2f", record.rss / 2^20)
162+
rss_str = @sprintf("%5.2f", memory_usage(record) / 2^20)
162163
printstyled(ctx.stdout, lpad(rss_str, ctx.rss_align, " "), " │\n", color = :white)
163164

164165
flush(ctx.stdout)
@@ -167,7 +168,7 @@ function print_test_finished(record::TestRecord, wrkr, test, ctx::TestIOContext)
167168
end
168169
end
169170

170-
function print_test_failed(record::TestRecord, wrkr, test, ctx::TestIOContext)
171+
function print_test_failed(record::AbstractTestRecord, wrkr, test, ctx::TestIOContext)
171172
lock(ctx.lock)
172173
try
173174
printstyled(ctx.stderr, test, color = :red)
@@ -193,7 +194,7 @@ function print_test_failed(record::TestRecord, wrkr, test, ctx::TestIOContext)
193194
end
194195
end
195196

196-
function print_test_crashed(::Type{TestRecord}, wrkr, test, ctx::TestIOContext)
197+
function print_test_crashed(::Type{<:AbstractTestRecord}, wrkr, test, ctx::TestIOContext)
197198
lock(ctx.lock)
198199
try
199200
printstyled(ctx.stderr, test, color = :red)
@@ -212,9 +213,9 @@ end
212213

213214
#
214215
# entry point
215-
#
216+
#
216217

217-
function runtest(::Type{TestRecord}, f, name, init_code, color)
218+
function runtest(::Type{TestRecord}, f, name, init_code, color, custom_args)
218219
function inner()
219220
# generate a temporary module to execute the tests in
220221
mod = @eval(Main, module $(gensym(name)) end)
@@ -466,7 +467,8 @@ Workers are automatically recycled when they exceed memory limits to prevent out
466467
issues during long test runs. The memory limit is set based on system architecture.
467468
"""
468469
function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = TestRecord,
469-
custom_tests::Dict{String, Expr}=Dict{String, Expr}(), init_code = :(),
470+
custom_tests::Dict{String, Expr}=Dict{String, Expr}(), init_code = :(),
471+
custom_record_init = :(), custom_args = (;),
470472
test_worker = Returns(nothing), stdout = Base.stdout, stderr = Base.stderr)
471473
#
472474
# set-up
@@ -789,8 +791,9 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
789791
put!(printer_channel, (:started, test, wrkr))
790792
result = try
791793
Distributed.remotecall_eval(Main, wrkr, :(import ParallelTestRunner))
794+
custom_record_init != :() && Distributed.remotecall_eval(Main, wrkr, custom_record_init)
792795
remotecall_fetch(runtest, wrkr, RecordType, test_runners[test], test,
793-
init_code, io_ctx.color)
796+
init_code, io_ctx.color, custom_args)
794797
catch ex
795798
if isa(ex, InterruptException)
796799
# the worker got interrupted, signal other tasks to stop

‎test/runtests.jl‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,97 @@ end
7575
@test contains(str, "SUCCESS")
7676
end
7777

78+
@testset "custom testrecord" begin
79+
custom_record_init = quote
80+
struct CustomTestRecord <: ParallelTestRunner.AbstractTestRecord
81+
# TODO: Would it be better to wrap "ParallelTestRunner.TestRecord "
82+
value::Any # AbstractTestSet or TestSetException
83+
output::String # captured stdout/stderr
84+
85+
# stats
86+
time::Float64
87+
bytes::UInt64
88+
gctime::Float64
89+
rss::UInt64
90+
end
91+
function ParallelTestRunner.memory_usage(rec::CustomTestRecord)
92+
return rec.rss
93+
end
94+
function ParallelTestRunner.test_IOContext(::Type{CustomTestRecord}, args...)
95+
return ParallelTestRunner.test_IOContext(ParallelTestRunner.TestRecord, args...)
96+
end
97+
function ParallelTestRunner.runtest(::Type{CustomTestRecord}, f, name, init_code, color, (; say_hello))
98+
function inner()
99+
# generate a temporary module to execute the tests in
100+
mod = @eval(Main, module $(gensym(name)) end)
101+
@eval(mod, import ParallelTestRunner: Test, Random)
102+
@eval(mod, using .Test, .Random)
103+
104+
Core.eval(mod, init_code)
105+
106+
data = @eval mod begin
107+
GC.gc(true)
108+
Random.seed!(1)
109+
110+
mktemp() do path, io
111+
stats = redirect_stdio(stdout=io, stderr=io) do
112+
@timed try
113+
if say_hello
114+
println("Hello from test '$name'")
115+
end
116+
@testset $name begin
117+
$f
118+
end
119+
catch err
120+
isa(err, Test.TestSetException) || rethrow()
121+
122+
# return the error to package it into a TestRecord
123+
err
124+
end
125+
end
126+
close(io)
127+
output = read(path, String)
128+
(; testset=stats.value, output, stats.time, stats.bytes, stats.gctime)
129+
130+
end
131+
end
132+
133+
# process results
134+
rss = Sys.maxrss()
135+
record = TestRecord(data..., rss)
136+
137+
GC.gc(true)
138+
return record
139+
end
140+
141+
@static if VERSION >= v"1.13.0-DEV.1044"
142+
@with Test.TESTSET_PRINT_ENABLE => false begin
143+
inner()
144+
end
145+
else
146+
old_print_setting = Test.TESTSET_PRINT_ENABLE[]
147+
Test.TESTSET_PRINT_ENABLE[] = false
148+
try
149+
inner()
150+
finally
151+
Test.TESTSET_PRINT_ENABLE[] = old_print_setting
152+
end
153+
end
154+
end
155+
end # quote
156+
eval(custom_record_init)
157+
158+
io = IOBuffer()
159+
160+
runtests(ParallelTestRunner, ["--verbose"]; custom_record_init, custom_args=(; say_hello=true), stdout=io, stderr=io)
161+
str = String(take!(io))
162+
163+
@test contains(str, r"basic .+ started at")
164+
@test contains(str, r"Hello from test 'basic'")
165+
@test contains(str, "SUCCESS")
166+
end
167+
168+
78169
@testset "failing test" begin
79170
custom_tests = Dict(
80171
"failing test" => quote

0 commit comments

Comments
 (0)