diff --git a/CHANGELOG.md b/CHANGELOG.md index 81940e4c18..45c42ac79a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Added `Remotes.Forgejo` for specifying a `Remote` hosted on a Forgejo instance (such as codeberg.org). ([#2857]) +* Doctests now default to the `parser_for_module` of the module that the docstring appears in, allowing modules that set their syntax version via `Base.Experimental.@set_syntax_version` to have their doctests parsed with the correct syntax automatically. Also added support for `DocTestSyntax` metadata and per-block `syntax=` attributes to explicitly specify a syntax version. ([#2874]) ### Changed diff --git a/src/DocMeta.jl b/src/DocMeta.jl index 8b2580de8a..4c9efb3935 100644 --- a/src/DocMeta.jl +++ b/src/DocMeta.jl @@ -13,6 +13,7 @@ module — a special variable is created in each module that has documentation m * `DocTestSetup`: contains the doctest setup code for doctests in the module. * `DocTestTeardown`: contains the doctest teardown code for doctests in the module. +* `DocTestSyntax`: specifies the Julia syntax version for parsing doctests in the module (requires Julia 1.14+). """ module DocMeta import ..Documenter @@ -30,7 +31,8 @@ const METATYPE = Dict{Symbol, Any} "Dictionary of all valid metadata keys and their types." const VALIDMETA = Dict{Symbol, Type}( :DocTestSetup => Union{Expr, Symbol}, - :DocTestTeardown => Union{Expr, Symbol} + :DocTestTeardown => Union{Expr, Symbol}, + :DocTestSyntax => VersionNumber, ) """ diff --git a/src/doctests.jl b/src/doctests.jl index f3ae7fa3ad..48cbbfbf76 100644 --- a/src/doctests.jl +++ b/src/doctests.jl @@ -8,7 +8,8 @@ struct DocTestContext file::String doc::Documenter.Document meta::Dict{Symbol, Any} - DocTestContext(file::String, doc::Documenter.Document) = new(file, doc, copy(doc.user.meta)) + mod::Union{Module, Nothing} + DocTestContext(file::String, doc::Documenter.Document, mod::Union{Module, Nothing} = nothing) = new(file, doc, copy(doc.user.meta), mod) end """ @@ -69,7 +70,7 @@ function _doctest(docstr::Docs.DocStr, mod::Module, doc::Documenter.Document) """ docstr.data rethrow(err) end - ctx = DocTestContext(docstr.data[:path], doc) + ctx = DocTestContext(docstr.data[:path], doc, mod) merge!(ctx.meta, DocMeta.getdocmeta(mod)) ctx.meta[:CurrentFile] = get(docstr.data, :path, nothing) return _doctest(ctx, mdast) @@ -170,10 +171,25 @@ function _doctest(ctx::DocTestContext, block::MarkdownAST.CodeBlock) return false end end + # Extract syntax version: per-block syntax= takes precedence over global DocTestSyntax + syntax_version = get(d, :syntax, get(ctx.meta, :DocTestSyntax, nothing)) + + # Check if syntax versioning is supported (requires Julia 1.14+) + if syntax_version !== nothing && !isdefined(Base, :VersionedParse) + if syntax_version >= v"1.14" + file = ctx.meta[:CurrentFile] + lines = Documenter.find_block_in_file(block.code, file) + @warn "Skipping doctest in $(Documenter.locrepr(file, lines)): syntax version $syntax_version requires Julia 1.14 or later (running on Julia $VERSION)" + return true # Skip this doctest but don't treat as error + end + # For syntax versions < 1.14, we can proceed normally since the syntax should be compatible + syntax_version = nothing + end + if occursin(r"^julia> "m, block.code) - eval_repl(block, sandbox, ctx.meta, ctx.doc, ctx.file) + eval_repl(block, sandbox, ctx.meta, ctx.doc, ctx.file; syntax_version = syntax_version, mod = ctx.mod) elseif occursin(r"^# output$"m, block.code) - eval_script(block, sandbox, ctx.meta, ctx.doc, ctx.file) + eval_script(block, sandbox, ctx.meta, ctx.doc, ctx.file; syntax_version = syntax_version, mod = ctx.mod) elseif occursin(r"^# output\s+$"m, block.code) file = ctx.meta[:CurrentFile] lines = Documenter.find_block_in_file(block.code, file) @@ -246,13 +262,13 @@ mutable struct Result end -function eval_repl(block::MarkdownAST.CodeBlock, sandbox, meta::Dict, doc::Documenter.Document, page) +function eval_repl(block::MarkdownAST.CodeBlock, sandbox, meta::Dict, doc::Documenter.Document, page; syntax_version = nothing, mod = nothing) file = meta[:CurrentFile] src_lines = Documenter.find_block_in_file(block.code, file) (prefix, split) = repl_splitter(block.code, doc, file, src_lines) for (raw_input, input, output) in split result = Result(block, raw_input, input, output, file) - for (ex, str) in Documenter.parseblock(input, doc, page; keywords = false, raise = false) + for (ex, str) in Documenter.parseblock(input, doc, page; keywords = false, raise = false, syntax_version = syntax_version, mod = mod) # Input containing a semi-colon gets suppressed in the final output. @debug "Evaluating REPL line from doctest at $(Documenter.locrepr(result.file, src_lines))" unparsed_string = str parsed_expression = ex result.hide = REPL.ends_with_semicolon(str) @@ -276,7 +292,7 @@ function eval_repl(block::MarkdownAST.CodeBlock, sandbox, meta::Dict, doc::Docum return end -function eval_script(block::MarkdownAST.CodeBlock, sandbox, meta::Dict, doc::Documenter.Document, page) +function eval_script(block::MarkdownAST.CodeBlock, sandbox, meta::Dict, doc::Documenter.Document, page; syntax_version = nothing, mod = nothing) # TODO: decide whether to keep `# output` syntax for this. It's a bit ugly. # Maybe use double blank lines, i.e. # @@ -286,7 +302,7 @@ function eval_script(block::MarkdownAST.CodeBlock, sandbox, meta::Dict, doc::Doc input = rstrip(input, '\n') output = lstrip(output, '\n') result = Result(block, input, output, meta[:CurrentFile]) - for (ex, str) in Documenter.parseblock(input, doc, page; keywords = false, raise = false) + for (ex, str) in Documenter.parseblock(input, doc, page; keywords = false, raise = false, syntax_version = syntax_version, mod = mod) c = IOCapture.capture(rethrow = InterruptException) do Core.eval(sandbox, ex) end diff --git a/src/utilities/utilities.jl b/src/utilities/utilities.jl index 925b1d851f..c31ae0508a 100644 --- a/src/utilities/utilities.jl +++ b/src/utilities/utilities.jl @@ -139,10 +139,17 @@ returns this expression normally and it must be handled appropriately by the cal The `linenumbernode` can be passed as a `LineNumberNode` to give information about filename and starting line number of the block (requires Julia 1.6 or higher). + +The `syntax_version` can be passed as a `VersionNumber` to parse the code using a specific +Julia syntax version. This requires Julia 1.14 or higher. + +The `mod` can be passed as a `Module` to use that module's parser (via `Meta.parser_for_module`). +If not specified, the default parser is used. When both `syntax_version` and `mod` are specified, +`syntax_version` takes precedence. """ function parseblock( code::AbstractString, doc, file; skip = 0, keywords = true, raise = true, - linenumbernode = nothing, lines = nothing + linenumbernode = nothing, lines = nothing, syntax_version = nothing, mod = nothing ) # Drop `skip` leading lines from the code block. Needed for deprecated `{docs}` syntax. code = string(code, '\n') @@ -150,6 +157,16 @@ function parseblock( endofstr = lastindex(code) results = [] cursor = 1 + # Determine which parser to use: + # 1. If syntax_version is specified and VersionedParse exists, use versioned parser + # 2. Otherwise, use the module's parser via parser_for_module (defaults to Core._parse) + _parse = if syntax_version !== nothing && isdefined(Base, :VersionedParse) + Base.VersionedParse(syntax_version) + elseif isdefined(Meta, :parser_for_module) + Meta.parser_for_module(mod) + else + nothing + end while cursor < endofstr # Check for keywords first since they will throw parse errors if we `parse` them. line = match(r"^(.*)\r?\n"m, SubString(code, cursor)).match @@ -159,7 +176,11 @@ function parseblock( (QuoteNode(keyword), cursor + lastindex(line)) else try - Meta.parse(code, cursor; raise = raise) + if _parse !== nothing + Meta.parse(code, cursor; raise = raise, _parse = _parse) + else + Meta.parse(code, cursor; raise = raise) + end catch err @docerror(doc, :parse_error, "failed to parse code block in $(locrepr(file, lines))", exception = err) return [] @@ -172,7 +193,8 @@ function parseblock( cursor = ncursor end if linenumbernode isa LineNumberNode - exs = Meta.parseall(code; filename = linenumbernode.file).args + parseall_kwargs = _parse !== nothing ? (; filename = linenumbernode.file, _parse = _parse) : (; filename = linenumbernode.file) + exs = Meta.parseall(code; parseall_kwargs...).args if isempty(results) && length(exs) == 1 && exs[1] isa LineNumberNode # block was empty or consisted of just comments empty!(exs) diff --git a/test/doctests/src/SyntaxVersioning.jl b/test/doctests/src/SyntaxVersioning.jl new file mode 100644 index 0000000000..b31a359c59 --- /dev/null +++ b/test/doctests/src/SyntaxVersioning.jl @@ -0,0 +1,63 @@ +# Test module that sets its syntax version to 1.14 +# Doctests in this module should automatically use 1.14 syntax + +module SyntaxVersioning + +# Set this module's syntax version to 1.14 +Base.Experimental.@set_syntax_version v"1.14" + +""" +Verify that the syntax version is set correctly. + +```jldoctest +julia> (Base.Experimental.@VERSION).syntax +v"1.14.0" +``` +""" +function check_syntax_version end + +""" +This doctest uses labeled break syntax which requires Julia 1.14. +Since the module has its syntax version set, this should work without +needing a `syntax=` annotation. + +```jldoctest +julia> result = @label myblock begin + for i in 1:10 + if i > 5 + break myblock i * 2 + end + end + 0 + end +12 +``` +""" +function labeled_break_example end + +""" +This doctest uses labeled continue syntax which requires Julia 1.14. + +```jldoctest +julia> output = Int[] +Int64[] + +julia> @label outer for i in 1:3 + for j in 1:3 + if j == 2 + continue outer + end + push!(output, i * 10 + j) + end + end + +julia> output +3-element Vector{Int64}: + 11 + 21 + 31 +``` +""" +function labeled_continue_example end + +end # module diff --git a/test/doctests/src/SyntaxVersioning13.jl b/test/doctests/src/SyntaxVersioning13.jl new file mode 100644 index 0000000000..a5b9b48dd8 --- /dev/null +++ b/test/doctests/src/SyntaxVersioning13.jl @@ -0,0 +1,18 @@ +# Negative test: module with syntax version set to 1.13 +# Verifies that the parser actually respects the older syntax version + +module SyntaxVersioning13 + +Base.Experimental.@set_syntax_version v"1.13" + +""" +Verify that the syntax version is 1.13, not 1.14. + +```jldoctest +julia> (Base.Experimental.@VERSION).syntax == v"1.13" +true +``` +""" +function check_syntax_version end + +end # module diff --git a/test/doctests/src/syntax_versioning.md b/test/doctests/src/syntax_versioning.md new file mode 100644 index 0000000000..0d1437ce6c --- /dev/null +++ b/test/doctests/src/syntax_versioning.md @@ -0,0 +1,36 @@ +# Syntax Versioning Tests + +This page tests the `syntax=` attribute for doctests. + +## Per-block syntax version + +This doctest uses the `syntax=` attribute to specify Julia 1.14 syntax: + +```jldoctest; syntax = v"1.14" +julia> result = @label myblock begin + for i in 1:5 + if i > 3 + break myblock i * 10 + end + end + 0 + end +40 +``` + +## Verifying syntax version with @VERSION + +```jldoctest; syntax = v"1.14" +julia> (Base.Experimental.@VERSION).syntax == v"1.14" +true +``` + +## Negative test: syntax version 1.13 + +This verifies that setting `syntax = v"1.13"` actually uses the 1.13 parser, +not the default one. + +```jldoctest; syntax = v"1.13" +julia> (Base.Experimental.@VERSION).syntax == v"1.13" +true +``` diff --git a/test/doctests/syntax_versioning.jl b/test/doctests/syntax_versioning.jl new file mode 100644 index 0000000000..ea8de443dd --- /dev/null +++ b/test/doctests/syntax_versioning.jl @@ -0,0 +1,64 @@ +# Tests for syntax versioning support in doctests +# +# These tests verify that modules with syntax version set via +# Base.Experimental.@set_syntax_version have their doctests parsed +# with the correct syntax version. + +module SyntaxVersioningTests +using Test +using Documenter +import IOCapture + +function run_doctest(; modules = Module[], mdfiles = String[]) + builds_directory = mktempdir() + srcdir = joinpath(builds_directory, "src") + mkpath(srcdir) + touch(joinpath(srcdir, "index.md")) + for mdfile in mdfiles + cp(joinpath(@__DIR__, "src", mdfile), joinpath(srcdir, mdfile)) + end + c = IOCapture.capture(rethrow = InterruptException) do + withenv("JULIA_DEBUG" => "") do + makedocs( + sitename = " ", + format = Documenter.HTML(edit_link = nothing), + root = builds_directory, + modules = modules, + pages = isempty(mdfiles) ? Documenter.Page[] : mdfiles, + remotes = nothing, + checkdocs = :none, + doctest = true, + warnonly = false, + ) + end + end + if c.error + @error "Doctest failed" output = c.output + end + return @test !c.error +end + +# Only run these tests on Julia 1.14+ where syntax versioning is available +@testset "Syntax Versioning" begin + if !isdefined(Base, :VersionedParse) + @info "Skipping syntax versioning tests: requires Julia 1.14+" + @test_skip false + else + include("src/SyntaxVersioning.jl") + include("src/SyntaxVersioning13.jl") + + @testset "Module with @set_syntax_version v\"1.14\"" begin + run_doctest(modules = [SyntaxVersioning]) + end + + @testset "Module with @set_syntax_version v\"1.13\" (negative)" begin + run_doctest(modules = [SyntaxVersioning13]) + end + + @testset "Markdown with syntax= attribute" begin + run_doctest(mdfiles = ["syntax_versioning.md"]) + end + end +end + +end # module diff --git a/test/runtests.jl b/test/runtests.jl index adb18d9613..89093b2a1c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -67,6 +67,7 @@ end include("doctests/doctestapi.jl") include("doctests/doctests.jl") include("doctests/fix/tests.jl") + include("doctests/syntax_versioning.jl") # DOM Tests. include("dom.jl")