Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
StringViews = "354b36f9-a18e-4713-926e-db85100087ba"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
TestEnv = "1e6cf692-eddd-4d53-88a5-2d735e33781b"

Expand All @@ -19,6 +20,7 @@ Profile = "1"
Random = "1"
Serialization = "1"
Sockets = "1"
StringViews = "1"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can't (don't want) any non-stdlib deps... how much benefit does StringViews give us? Our options are (i think): drop it, reimplement a minimal version, vendor it

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only thing this gives is the ability to run regexes against a view of a Vector{UInt8} (and comparing them to Strings, but that is easy to do manually)

Test = "1"
TestEnv = "1.8"
julia = "1.8"
Expand Down
5 changes: 3 additions & 2 deletions src/ReTestItems.jl
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ include("junit_xml.jl")
include("testcontext.jl")
include("log_capture.jl")
include("filtering.jl")
include("include_test_file.jl")

function __init__()
if ccall(:jl_generating_output, Cint, ()) == 0 # not precompiling
Expand Down Expand Up @@ -865,7 +866,7 @@ end

# Parses and evals files found by the `walkdir_task`. During macro expansion of `@testitem`
# test items are push!d onto the FileNode stored in task local storage as `:__RE_TEST_ITEMS__`.
function include_task(walkdir_channel, setup_channel, project_root, ti_filter)
function include_task(walkdir_channel, setup_channel, project_root, ti_filter::TestItemFilter)
try
testitem_names = Set{String}() # to enforce that names in the same file are unique
task_local_storage(:__RE_TEST_RUNNING__, true) do
Expand All @@ -874,7 +875,7 @@ function include_task(walkdir_channel, setup_channel, project_root, ti_filter)
for (file_path, file_node) in walkdir_channel
@debugv 1 "Including test items from file `$(file_path)`"
task_local_storage(:__RE_TEST_ITEMS__, (file_node, empty!(testitem_names))) do
Base.include(ti_filter, Main, file_path)
include_test_file(ti_filter, file_path)
end
end
end
Expand Down
20 changes: 12 additions & 8 deletions src/filtering.jl
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,7 @@ function filter_testitem(f, expr)
@assert expr.head == :macrocall
macro_name = expr.args[1]
if macro_name === Symbol("@testitem")
# `@testitem` have have at least: macro_name, line_number, name, body
length(expr.args) < 4 && return expr
name = try_get_name(expr)
name === nothing && return expr
tags = try_get_tags(expr)
tags === nothing && return expr
ti = TestItemMetadata(name, tags)
return f(ti) ? expr : nothing
return __filter_ti(f, expr)
elseif macro_name === Symbol("@testsetup")
return expr
elseif macro_name === ___RAI_MACRO_NAME_DONT_USE # TODO: drop this branch when we can
Expand Down Expand Up @@ -98,6 +91,17 @@ function try_get_tags(expr::Expr)
return tags
end

function __filter_ti(f, expr)
# `@testitem` have have at least: macro_name, line_number, name, body
length(expr.args) < 4 && return expr
name = try_get_name(expr)
name === nothing && return expr
tags = try_get_tags(expr)
tags === nothing && return expr
ti = TestItemMetadata(name, tags)
return f(ti) ? expr : nothing
end

# Macro used by RAI (corporate sponsor of this package)
# TODO: drop support for this when RAI codebase is fully switched to ReTestItems.jl
const ___RAI_MACRO_NAME_DONT_USE = Symbol("@test_rel")
Expand Down
95 changes: 95 additions & 0 deletions src/include_test_file.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
using Base.JuliaSyntax: JuliaSyntax, ParseError, ParseStream, @K_str
using Base.JuliaSyntax: any_error, build_tree, first_byte, kind, parse!, peek_full_token, peek_token
using StringViews

function include_test_file(ti_filter::TestItemFilter, path::String)
bytes = read(path)
stream = ParseStream(bytes)
line_starts = Int32[Int32(i) for (i, x) in enumerate(bytes) if x == 0x0a] # :(
tls = task_local_storage()
tls[:SOURCE_PATH] = path # This is also done by Base.include
try
@inbounds while true
JuliaSyntax.bump_trivia(stream, skip_newlines=true)
t = peek_token(stream, 1)
k = kind(t)
k == K"EndMarker" && break
line = _source_line_index(line_starts, first_byte(stream))
if k == K"@"
tf = peek_full_token(stream, 2)
v = @view(bytes[tf.first_byte:tf.last_byte])
if v == b"testitem" ; _eval_test_item_stream(stream, path, line, ti_filter, bytes)
elseif v == b"testsetup"; _eval_test_setup_stream(stream, path, line)
elseif v == b"test_rel" ; _eval_test_rel_stream(stream, path, line, ti_filter, bytes)
else
error("Test files must only include `@testitem` and `@testsetup` calls, got an `@$(StringView(v))` at $(path):$(line)") # TODO
end
else
error("Test files must only include `@testitem` and `@testsetup` calls, got a $t at $(path):$(line)") # TODO
end
empty!(stream) # TODO: SourceFile created on a reset stream always start line number at 1
end
finally
delete!(tls, :SOURCE_PATH)
end
end

@inline function _source_line_index(line_starts, bytes_pos)
lineidx = searchsortedfirst(line_starts, bytes_pos)
return (lineidx < lastindex(line_starts)) ? lineidx : lineidx-1
end

# unconditionally eval
function _eval_test_setup_stream(stream, path, line)
parse!(stream; rule=:statement)
ast = build_tree(Expr, stream; filename=path, first_line=line)
Core.eval(Main, ast)
return nothing
end

# test_rel -> apply ti_filter on the parsed ast
function _eval_test_rel_stream(stream, path, line, ti_filter::TestItemFilter, bytes)
parse!(stream; rule=:statement)
if !(ti_filter.name isa Nothing)
@inbounds for (i, token) in enumerate(stream.tokens)
if kind(token) == K"Identifier"
fbyte = JuliaSyntax.token_first_byte(stream, i)
lbyte = JuliaSyntax.token_last_byte(stream, i)
if @view(bytes[fbyte:lbyte]) == b"name"
fbyte = JuliaSyntax.token_first_byte(stream, i + 3)
lbyte = JuliaSyntax.token_last_byte(stream, i + 3)
name = StringView(@view(bytes[fbyte:lbyte]))
_contains(name, ti_filter.name) && break
return nothing
end
end
end
end
ast = build_tree(Expr, stream; filename=path, first_line=line)
any_error(stream) && throw(ParseError(stream, filename=path))
filtered = __filter_rai(ti_filter, ast)::Union{Nothing, Expr}
filtered === nothing || Core.eval(Main, filtered::Expr)
return nothing
end

# like above, but tries to avoid parsing the ast if it sees from the name identifier token
# it won't pass the filter
function _eval_test_item_stream(stream, path, line, ti_filter::TestItemFilter, bytes)
if !(ti_filter.name isa Nothing)
name_t = peek_full_token(stream, 4) # 3 was '\"'
name = @inbounds StringView(@view(bytes[name_t.first_byte:name_t.last_byte]))
parse!(stream; rule=:statement)
_contains(name, ti_filter.name) || return nothing
else
parse!(stream; rule=:statement)
end

ast = build_tree(Expr, stream; filename=path, first_line=line)
any_error(stream) && throw(ParseError(stream, filename=path))
filtered = __filter_ti(ti_filter, ast)::Union{Nothing, Expr}
filtered === nothing || Core.eval(Main, filtered::Expr)
return nothing
end

@inline _contains(s::AbstractString, pattern::Regex) = occursin(pattern, s)
@inline _contains(s::AbstractString, pattern::AbstractString) = s == pattern
6 changes: 3 additions & 3 deletions test/integrationtests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -816,7 +816,7 @@ end
end

@testset "error on code outside `@testitem`/`@testsetup`" begin
err_msg = "Test files must only include `@testitem` and `@testsetup` calls."
err_msg = "Test files must only include `@testitem` and `@testsetup` calls"
filter_func(ti) = false
@test_throws err_msg runtests(joinpath(TEST_FILES_DIR, "_misuse_file1_test.jl"))
@test_throws err_msg runtests(joinpath(TEST_FILES_DIR, "_misuse_file2_test.jl"))
Expand Down Expand Up @@ -868,9 +868,9 @@ end
file = joinpath(TEST_FILES_DIR, "_duplicate_names_test.jl")
relfpath = relpath(file, pkgdir(ReTestItems))
expected_msg = if Base.Sys.iswindows()
Regex("Duplicate test item name `dup` in file")
"Duplicate test item name `dup` in file"
else
Regex("Duplicate test item name `dup` in file `$(relfpath)` at line 4")
"Duplicate test item name `dup` in file `$(relfpath)` at line 4"
end
@test_throws expected_msg runtests(file; nworkers=0)
@test_throws expected_msg runtests(file; nworkers=1)
Expand Down
54 changes: 29 additions & 25 deletions test/packages/DontPass.jl/Manifest.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.8.2"
julia_version = "1.10.10"
manifest_format = "2.0"
project_hash = "31dabfddf1e36ecf2dc5cff56cdddc361b29f16f"

Expand Down Expand Up @@ -33,55 +33,54 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
version = "0.6.3"
version = "0.6.4"

[[deps.LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
version = "7.84.0+0"
version = "8.4.0+0"

[[deps.LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[deps.LibGit2_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
version = "1.6.4+0"

[[deps.LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
version = "1.10.2+0"
version = "1.11.0+1"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[deps.LoggingExtras]]
deps = ["Dates", "Logging"]
git-tree-sha1 = "cedb76b37bc5a6c702ade66be44f831fa23c681e"
uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36"
version = "1.0.0"

[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[deps.MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.28.0+0"
version = "2.28.2+1"

[[deps.MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
version = "2022.2.1"
version = "2023.1.10"

[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
version = "1.2.0"

[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
version = "1.8.0"
version = "1.10.0"

[[deps.Printf]]
deps = ["Unicode"]
Expand All @@ -92,14 +91,14 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[deps.Random]]
deps = ["SHA", "Serialization"]
deps = ["SHA"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[deps.ReTestItems]]
deps = ["Dates", "Logging", "LoggingExtras", "Pkg", "Serialization", "Sockets", "Test", "TestEnv"]
deps = ["Dates", "Logging", "Pkg", "Serialization", "Sockets", "StringViews", "Test", "TestEnv"]
path = "../../.."
uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
version = "0.1.0"
version = "1.34.0"

[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
Expand All @@ -111,25 +110,30 @@ uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[deps.StringViews]]
git-tree-sha1 = "d3c7a06c80622b8404b1105886c732abcb25cc2b"
uuid = "354b36f9-a18e-4713-926e-db85100087ba"
version = "1.3.5"

[[deps.TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
version = "1.0.0"
version = "1.0.3"

[[deps.Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
version = "1.10.1"
version = "1.10.0"

[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[deps.TestEnv]]
deps = ["Pkg"]
git-tree-sha1 = "b5a483bcc8c9f0df569101df166601e5e699f346"
git-tree-sha1 = "aec63ef091b11d67040b8b35d2dcdfdc4567c4d0"
uuid = "1e6cf692-eddd-4d53-88a5-2d735e33781b"
version = "1.9.3"
version = "1.102.2"

[[deps.UUIDs]]
deps = ["Random", "SHA"]
Expand All @@ -141,14 +145,14 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
[[deps.Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
version = "1.2.12+3"
version = "1.2.13+1"

[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
version = "1.48.0+0"
version = "1.52.0+1"

[[deps.p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
version = "17.4.0+0"
version = "17.4.0+2"
Loading
Loading