From aa464088a99e6867b1cc987f1ccb0b4ba63c764d Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 11:50:44 +0200 Subject: [PATCH 01/14] implement source range mechanism --- src/server.jl | 56 +++++++++++++++++++++++++++++++++++++++------------ src/socket.jl | 32 ++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/server.jl b/src/server.jl index d0084d4..6fce6df 100644 --- a/src/server.jl +++ b/src/server.jl @@ -66,6 +66,19 @@ mutable struct File end end +struct SourceRange + file::Union{String,Nothing} + lines::UnitRange{Int} + source_line::Union{Nothing,Int} # first line in source +end + +function SourceRange(file, lines, source_lines::UnitRange) + if length(lines) != length(source_lines) + error("Mismatching lengths of lines $lines ($(length(lines))) and source_lines $source_lines ($(length(source_lines)))") + end + SourceRange(file, lines, source_lines.start) +end + function _has_juliaup() try success(`juliaup --version`) && success(`julia --version`) @@ -339,13 +352,14 @@ function evaluate!( options::Union{String,Dict{String,Any}} = Dict{String,Any}(), chunk_callback = (i, n, c) -> nothing, markdown::Union{String,Nothing} = nothing, + source_ranges::Union{Nothing,Vector} = nothing, ) _check_output_dst(output) options = _parsed_options(options) path = abspath(f.path) if isfile(path) - source_code_hash, raw_chunks, file_frontmatter = raw_text_chunks(f, markdown) + source_code_hash, raw_chunks, file_frontmatter = raw_text_chunks(f, markdown; source_ranges) merged_options = _extract_relevant_options(file_frontmatter, options) # A change of parameter values must invalidate the source code hash. @@ -583,9 +597,9 @@ write_json(::Nothing, data) = data Return a vector of raw markdown and code chunks from `file` ready for evaluation by `evaluate_raw_cells!`. """ -raw_text_chunks(file::File, ::Nothing) = raw_text_chunks(file.path) -raw_text_chunks(file::File, markdown::String) = - raw_markdown_chunks_from_string(file.path, markdown) +raw_text_chunks(file::File, ::Nothing; source_ranges = nothing) = raw_text_chunks(file.path) +raw_text_chunks(file::File, markdown::String; source_ranges = nothing) = + raw_markdown_chunks_from_string(file.path, markdown; source_ranges) function raw_text_chunks(path::String) endswith(path, ".qmd") && return raw_markdown_chunks(path) @@ -607,7 +621,7 @@ raw_markdown_chunks(path::String) = struct Unset end -function raw_markdown_chunks_from_string(path::String, markdown::String) +function raw_markdown_chunks_from_string(path::String, markdown::String; source_ranges = nothing) raw_chunks = [] source_code_hash = hash(VERSION) pars = Parser() @@ -616,6 +630,22 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) source_code_hash = hash(file_fromtmatter, source_code_hash) source_lines = collect(eachline(IOBuffer(markdown))) terminal_line = 1 + + function source_file_and_line(terminal_line::Int) + if source_ranges === nothing + return (; file = path, line = terminal_line) + else + for source_range in source_ranges + if terminal_line in source_range.lines + file::String = something(source_range.file, "unknown") + source_line::Int = terminal_line - source_range.lines.start + source_range.source_line + return (; file, line = source_line) + end + end + error("Terminal line $terminal_line was not included in source ranges, last range was $(last(source_ranges))") + end + end + code_cells = false for (node, enter) in ast if enter && @@ -625,7 +655,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) md = join(source_lines[terminal_line:(line-1)], "\n") push!( raw_chunks, - (type = :markdown, source = md, file = path, line = terminal_line), + (; type = :markdown, source = md, source_file_and_line(terminal_line)...), ) if contains(md, r"`{(?:julia|python|r)} ") source_code_hash = hash(md, source_code_hash) @@ -636,7 +666,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) # this option could in the future also include a vector of line numbers, which knitr supports. # all other options seem to be quarto-rendering related, like where to put figure captions etc. source = node.literal - cell_options = extract_cell_options(source; file = path, line = line) + cell_options = extract_cell_options(source; source_file_and_line(line)...) evaluate = get(cell_options, "eval", Unset()) if !(evaluate isa Union{Bool,Unset}) error( @@ -649,12 +679,11 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) is_r_toplevel(node) ? :r : error("Unhandled code block language") push!( raw_chunks, - ( + (; type = :code, language = language, source, - file = path, - line, + source_file_and_line(line)..., evaluate, cell_options, ), @@ -666,7 +695,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) md = join(source_lines[terminal_line:end], "\n") push!( raw_chunks, - (type = :markdown, source = md, file = path, line = terminal_line), + (; type = :markdown, source = md, source_file_and_line(terminal_line)...), ) if contains(md, r"`{(?:julia|python|r)} ") source_code_hash = hash(md, source_code_hash) @@ -690,7 +719,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) if raw_chunks[end].type == :code push!( raw_chunks, - (type = :markdown, source = "", file = path, line = terminal_line), + (; type = :markdown, source = "", source_file_and_line(terminal_line)...), ) end @@ -1499,6 +1528,7 @@ function run!( showprogress::Bool = true, options::Union{String,Dict{String,Any}} = Dict{String,Any}(), chunk_callback = (i, n, c) -> nothing, + source_ranges::Union{Nothing,Vector} = nothing, ) try borrow_file!(server, path; options, optionally_create = true) do file @@ -1519,7 +1549,7 @@ function run!( result_task = Threads.@spawn begin try - evaluate!(file, output; showprogress, options, markdown, chunk_callback) + evaluate!(file, output; showprogress, options, markdown, chunk_callback, source_ranges) finally put!(file.run_decision_channel, :evaluate_finished) end diff --git a/src/socket.jl b/src/socket.jl index 88b5d97..9c3428c 100644 --- a/src/socket.jl +++ b/src/socket.jl @@ -345,9 +345,6 @@ function _handle_response_internal( # Running: if type == "run" - options = _get_options(request.content) - markdown = _get_markdown(options) - function chunk_callback(i, n, chunk) _write_json( socket, @@ -362,6 +359,10 @@ function _handle_response_internal( end result = try + options = _get_options(request.content) + markdown = _get_markdown(options) + source_ranges = _get_source_ranges(request.content) + (; notebook = run!( notebooks, @@ -370,6 +371,7 @@ function _handle_response_internal( markdown, showprogress, chunk_callback, + source_ranges, ) ) catch error @@ -464,6 +466,30 @@ _get_file(content::String) = content _get_options(content::Dict) = get(Dict{String,Any}, content, "options") _get_options(::String) = Dict{String,Any}() +function _get_source_ranges(content::Dict) + ranges = get(content, "sourceRanges", nothing) + ranges === nothing && return nothing + return map(ranges) do range + file = get(range, "file", nothing) + _lines::Vector{Int} = range["lines"] + @assert length(_lines) == 2 + lines = _lines[1]:_lines[2] + _source_lines::Union{Nothing,Vector{Int}} = get(range, "sourceLines", nothing) + source_lines = if _source_lines === nothing + 1:length(lines) # source lines are only missing in degenerate cases like additional newlines anyway so this doesn't really matter + else + @assert length(_source_lines) == 2 + _source_lines[1]:_source_lines[2] + end + SourceRange( + file, + lines, + source_lines, + ) + end +end +_get_source_ranges(::String) = nothing + function _get_nested(d::Dict, keys...) _d = d for key in keys From 5fce6ad6632f5847c560bb16b148a300e59889ea Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 12:53:26 +0200 Subject: [PATCH 02/14] don't look up additional line that doesn't exist --- src/server.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.jl b/src/server.jl index 6fce6df..6f99019 100644 --- a/src/server.jl +++ b/src/server.jl @@ -719,7 +719,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String; source_ if raw_chunks[end].type == :code push!( raw_chunks, - (; type = :markdown, source = "", source_file_and_line(terminal_line)...), + (; type = :markdown, source = "", file = path, line = terminal_line), ) end From 76be2f7614ccf9e680bd5cec6b4d8a14f65bc7a6 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 12:53:37 +0200 Subject: [PATCH 03/14] add test --- test/examples/sourceranges/to_include.qmd | 3 + test/examples/sourceranges/with_include.qmd | 9 +++ test/testsets/socket_server/client.js | 21 ++++--- test/testsets/socket_server/socket_server.jl | 66 +++++++++++++++----- 4 files changed, 76 insertions(+), 23 deletions(-) create mode 100644 test/examples/sourceranges/to_include.qmd create mode 100644 test/examples/sourceranges/with_include.qmd diff --git a/test/examples/sourceranges/to_include.qmd b/test/examples/sourceranges/to_include.qmd new file mode 100644 index 0000000..60f0d85 --- /dev/null +++ b/test/examples/sourceranges/to_include.qmd @@ -0,0 +1,3 @@ +```{julia} +print("$(@__FILE__):$(@__LINE__)") +``` \ No newline at end of file diff --git a/test/examples/sourceranges/with_include.qmd b/test/examples/sourceranges/with_include.qmd new file mode 100644 index 0000000..271ece9 --- /dev/null +++ b/test/examples/sourceranges/with_include.qmd @@ -0,0 +1,9 @@ +--- +engine: julia +--- + +{{< include to_include.qmd >}} + +```{julia} +print("$(@__FILE__):$(@__LINE__)") +``` \ No newline at end of file diff --git a/test/testsets/socket_server/client.js b/test/testsets/socket_server/client.js index 3f06ad6..0b4d3b8 100644 --- a/test/testsets/socket_server/client.js +++ b/test/testsets/socket_server/client.js @@ -22,27 +22,30 @@ function handle() { const isready = () => toJSON({ type: 'isready', content: '' }); const status = () => toJSON({ type: 'status', content: '' }); - const notebook = (arg) => { - if (path.isAbsolute(arg)) { - return arg + const content = (arg) => { + if (typeof arg === 'string') { + if (path.isAbsolute(arg)) { + return arg + } + throw new Error('No notebook with absolute path specified.'); } - throw new Error('No notebook with absolute path specified.'); + return arg; } const type = process.argv[4]; - const arg = process.argv[5]; + const arg = process.argv.length >= 6 ? JSON.parse(process.argv[5]) : undefined; switch (type) { case 'run': - return run(notebook(arg)); + return run(content(arg)); case 'close': - return close(notebook(arg)); + return close(content(arg)); case 'forceclose': - return forceclose(notebook(arg)); + return forceclose(content(arg)); case 'stop': return stop(); case 'isopen': - return isopen(notebook(arg)); + return isopen(content(arg)); case 'isready': return isready(); case 'status': diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index 5e024f8..c3f5e31 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -12,17 +12,17 @@ include("../../utilities/prelude.jl") @test json(`$node $client $(server.port) $(server.key) isready`) - d1 = json(`$node $client $(server.port) $(server.key) isopen $(cell_types)`) + d1 = json(`$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`) @test d1 == false - d2 = json(`$node $client $(server.port) $(server.key) run $(cell_types)`) + d2 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) @test length(d2["notebook"]["cells"]) == 9 - d3 = json(`$node $client $(server.port) $(server.key) isopen $(cell_types)`) + d3 = json(`$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`) @test d3 == true t_before_run = Dates.now() - d4 = json(`$node $client $(server.port) $(server.key) run $(cell_types)`) + d4 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) t_after_run = Dates.now() @test d2 == d4 @@ -31,20 +31,20 @@ include("../../utilities/prelude.jl") @test occursin("workers active: 1", d5) @test occursin(abspath(cell_types), d5) - d6 = json(`$node $client $(server.port) $(server.key) close $(cell_types)`) + d6 = json(`$node $client $(server.port) $(server.key) close $(JSON3.write(cell_types))`) @test d6["status"] == true - d7 = json(`$node $client $(server.port) $(server.key) isopen $(cell_types)`) + d7 = json(`$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`) @test d7 == false - d8 = json(`$node $client $(server.port) $(server.key) run $(cell_types)`) + d8 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) @test d2 == d8 # test that certain commands on notebooks fail while those notebooks are already running sleep_10 = abspath("../../examples/sleep_10.qmd") sleep_task = Threads.@spawn json( - `$node $client $(server.port) $(server.key) run $(sleep_10)`, + `$node $client $(server.port) $(server.key) run $(JSON3.write(sleep_10))`, ) # wait until server lock locks due to the `run` command above @@ -59,10 +59,10 @@ include("../../utilities/prelude.jl") # both of these tasks should then try to access the worker that is busy and fail because # the lock is already held d9_task = Threads.@spawn json( - `$node $client $(server.port) $(server.key) run $(sleep_10)`, + `$node $client $(server.port) $(server.key) run $(JSON3.write(sleep_10))`, ) d10_task = Threads.@spawn json( - `$node $client $(server.port) $(server.key) close $(sleep_10)`, + `$node $client $(server.port) $(server.key) close $(JSON3.write(sleep_10))`, ) d9 = fetch(d9_task) @@ -90,7 +90,7 @@ end sleep_10 = abspath("../../examples/sleep_10.qmd") sleep_task = Threads.@spawn json( - `$node $client $(server.port) $(server.key) run $(sleep_10)`, + `$node $client $(server.port) $(server.key) run $(JSON3.write(sleep_10))`, ) # wait until server lock locks due to the `run` command above @@ -103,7 +103,7 @@ end end # force-closing should kill the worker even if it's running - d1 = json(`$node $client $(server.port) $(server.key) forceclose $(sleep_10)`) + d1 = json(`$node $client $(server.port) $(server.key) forceclose $(JSON3.write(sleep_10))`) @test d1 == Dict{String,Any}("status" => true) d2 = fetch(sleep_task) @@ -113,9 +113,9 @@ end cell_types = abspath("../../examples/cell_types.qmd") - d3 = json(`$node $client $(server.port) $(server.key) run $(cell_types)`) + d3 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) - d4 = json(`$node $client $(server.port) $(server.key) forceclose $(cell_types)`) + d4 = json(`$node $client $(server.port) $(server.key) forceclose $(JSON3.write(cell_types))`) @test d4 == Dict{String,Any}("status" => true) run(`$node $client $(server.port) $(server.key) stop`) @@ -123,3 +123,41 @@ end wait(server) end end + +@testset "source ranges" begin + cd(@__DIR__) do + with_include = abspath("../../examples/sourceranges/with_include.qmd") + to_include = abspath("../../examples/sourceranges/to_include.qmd") + with_include_md = read(with_include, String) + to_include_md = read(to_include, String) + with_include_A, with_include_B = split(with_include_md, "{{< include to_include.qmd >}}\n") + lines_A = length(split(with_include_A, "\n")) + lines_B = length(split(with_include_B, "\n")) + lines_to_include = length(split(to_include_md, "\n")) + full = join([with_include_A, to_include_md, with_include_B], "\n") + + ends = cumsum([lines_A, lines_to_include, lines_B]) + + source_ranges = [ + (; file = with_include, lines = [1, ends[1]], sourceLines = [1, lines_A]), + (; file = to_include, lines = [ends[1]+1, ends[2]], sourceLines = [1, lines_to_include]), + (; file = with_include, lines = [ends[2]+1, ends[3]], sourceLines = [lines_A+1, lines_A+lines_B]), + ] + + node = NodeJS_18_jll.node() + client = joinpath(@__DIR__, "client.js") + server = QuartoNotebookRunner.serve(; showprogress = false) + sleep(1) + json(cmd) = JSON3.read(read(cmd, String), Any) + + options = (; target = (; markdown = (; value = full))) + content = (; file = with_include, sourceRanges = source_ranges, options) + result = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(content))`) + + line_in_with_include = findfirst(contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), collect(eachline(with_include))) + line_in_to_include = findfirst(contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), collect(eachline(to_include))) + + @test result["notebook"]["cells"][2]["outputs"][1]["text"] == "$(to_include):$line_in_to_include" + @test result["notebook"]["cells"][4]["outputs"][1]["text"] == "$(with_include):$line_in_with_include" + end +end \ No newline at end of file From 32cb8c41abb964a40a608fe79cbba73aaa9feb7a Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 12:58:05 +0200 Subject: [PATCH 04/14] formatting --- src/server.jl | 30 ++++++-- src/socket.jl | 6 +- test/testsets/socket_server/socket_server.jl | 80 +++++++++++++++----- 3 files changed, 86 insertions(+), 30 deletions(-) diff --git a/src/server.jl b/src/server.jl index 6f99019..37814c1 100644 --- a/src/server.jl +++ b/src/server.jl @@ -74,7 +74,9 @@ end function SourceRange(file, lines, source_lines::UnitRange) if length(lines) != length(source_lines) - error("Mismatching lengths of lines $lines ($(length(lines))) and source_lines $source_lines ($(length(source_lines)))") + error( + "Mismatching lengths of lines $lines ($(length(lines))) and source_lines $source_lines ($(length(source_lines)))", + ) end SourceRange(file, lines, source_lines.start) end @@ -359,7 +361,8 @@ function evaluate!( options = _parsed_options(options) path = abspath(f.path) if isfile(path) - source_code_hash, raw_chunks, file_frontmatter = raw_text_chunks(f, markdown; source_ranges) + source_code_hash, raw_chunks, file_frontmatter = + raw_text_chunks(f, markdown; source_ranges) merged_options = _extract_relevant_options(file_frontmatter, options) # A change of parameter values must invalidate the source code hash. @@ -621,7 +624,11 @@ raw_markdown_chunks(path::String) = struct Unset end -function raw_markdown_chunks_from_string(path::String, markdown::String; source_ranges = nothing) +function raw_markdown_chunks_from_string( + path::String, + markdown::String; + source_ranges = nothing, +) raw_chunks = [] source_code_hash = hash(VERSION) pars = Parser() @@ -638,11 +645,14 @@ function raw_markdown_chunks_from_string(path::String, markdown::String; source_ for source_range in source_ranges if terminal_line in source_range.lines file::String = something(source_range.file, "unknown") - source_line::Int = terminal_line - source_range.lines.start + source_range.source_line + source_line::Int = + terminal_line - source_range.lines.start + source_range.source_line return (; file, line = source_line) end end - error("Terminal line $terminal_line was not included in source ranges, last range was $(last(source_ranges))") + error( + "Terminal line $terminal_line was not included in source ranges, last range was $(last(source_ranges))", + ) end end @@ -1549,7 +1559,15 @@ function run!( result_task = Threads.@spawn begin try - evaluate!(file, output; showprogress, options, markdown, chunk_callback, source_ranges) + evaluate!( + file, + output; + showprogress, + options, + markdown, + chunk_callback, + source_ranges, + ) finally put!(file.run_decision_channel, :evaluate_finished) end diff --git a/src/socket.jl b/src/socket.jl index 9c3428c..47313dd 100644 --- a/src/socket.jl +++ b/src/socket.jl @@ -481,11 +481,7 @@ function _get_source_ranges(content::Dict) @assert length(_source_lines) == 2 _source_lines[1]:_source_lines[2] end - SourceRange( - file, - lines, - source_lines, - ) + SourceRange(file, lines, source_lines) end end _get_source_ranges(::String) = nothing diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index c3f5e31..9789469 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -12,17 +12,25 @@ include("../../utilities/prelude.jl") @test json(`$node $client $(server.port) $(server.key) isready`) - d1 = json(`$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`) + d1 = json( + `$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`, + ) @test d1 == false - d2 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) + d2 = json( + `$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`, + ) @test length(d2["notebook"]["cells"]) == 9 - d3 = json(`$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`) + d3 = json( + `$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`, + ) @test d3 == true t_before_run = Dates.now() - d4 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) + d4 = json( + `$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`, + ) t_after_run = Dates.now() @test d2 == d4 @@ -31,13 +39,19 @@ include("../../utilities/prelude.jl") @test occursin("workers active: 1", d5) @test occursin(abspath(cell_types), d5) - d6 = json(`$node $client $(server.port) $(server.key) close $(JSON3.write(cell_types))`) + d6 = json( + `$node $client $(server.port) $(server.key) close $(JSON3.write(cell_types))`, + ) @test d6["status"] == true - d7 = json(`$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`) + d7 = json( + `$node $client $(server.port) $(server.key) isopen $(JSON3.write(cell_types))`, + ) @test d7 == false - d8 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) + d8 = json( + `$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`, + ) @test d2 == d8 # test that certain commands on notebooks fail while those notebooks are already running @@ -103,7 +117,9 @@ end end # force-closing should kill the worker even if it's running - d1 = json(`$node $client $(server.port) $(server.key) forceclose $(JSON3.write(sleep_10))`) + d1 = json( + `$node $client $(server.port) $(server.key) forceclose $(JSON3.write(sleep_10))`, + ) @test d1 == Dict{String,Any}("status" => true) d2 = fetch(sleep_task) @@ -113,9 +129,13 @@ end cell_types = abspath("../../examples/cell_types.qmd") - d3 = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`) + d3 = json( + `$node $client $(server.port) $(server.key) run $(JSON3.write(cell_types))`, + ) - d4 = json(`$node $client $(server.port) $(server.key) forceclose $(JSON3.write(cell_types))`) + d4 = json( + `$node $client $(server.port) $(server.key) forceclose $(JSON3.write(cell_types))`, + ) @test d4 == Dict{String,Any}("status" => true) run(`$node $client $(server.port) $(server.key) stop`) @@ -128,20 +148,33 @@ end cd(@__DIR__) do with_include = abspath("../../examples/sourceranges/with_include.qmd") to_include = abspath("../../examples/sourceranges/to_include.qmd") + with_include_md = read(with_include, String) to_include_md = read(to_include, String) - with_include_A, with_include_B = split(with_include_md, "{{< include to_include.qmd >}}\n") + + with_include_A, with_include_B = + split(with_include_md, "{{< include to_include.qmd >}}\n") + lines_A = length(split(with_include_A, "\n")) lines_B = length(split(with_include_B, "\n")) lines_to_include = length(split(to_include_md, "\n")) + full = join([with_include_A, to_include_md, with_include_B], "\n") ends = cumsum([lines_A, lines_to_include, lines_B]) source_ranges = [ (; file = with_include, lines = [1, ends[1]], sourceLines = [1, lines_A]), - (; file = to_include, lines = [ends[1]+1, ends[2]], sourceLines = [1, lines_to_include]), - (; file = with_include, lines = [ends[2]+1, ends[3]], sourceLines = [lines_A+1, lines_A+lines_B]), + (; + file = to_include, + lines = [ends[1] + 1, ends[2]], + sourceLines = [1, lines_to_include], + ), + (; + file = with_include, + lines = [ends[2] + 1, ends[3]], + sourceLines = [lines_A + 1, lines_A + lines_B], + ), ] node = NodeJS_18_jll.node() @@ -152,12 +185,21 @@ end options = (; target = (; markdown = (; value = full))) content = (; file = with_include, sourceRanges = source_ranges, options) - result = json(`$node $client $(server.port) $(server.key) run $(JSON3.write(content))`) + result = + json(`$node $client $(server.port) $(server.key) run $(JSON3.write(content))`) - line_in_with_include = findfirst(contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), collect(eachline(with_include))) - line_in_to_include = findfirst(contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), collect(eachline(to_include))) + line_in_with_include = findfirst( + contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), + collect(eachline(with_include)), + ) + line_in_to_include = findfirst( + contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), + collect(eachline(to_include)), + ) - @test result["notebook"]["cells"][2]["outputs"][1]["text"] == "$(to_include):$line_in_to_include" - @test result["notebook"]["cells"][4]["outputs"][1]["text"] == "$(with_include):$line_in_with_include" + @test result["notebook"]["cells"][2]["outputs"][1]["text"] == + "$(to_include):$line_in_to_include" + @test result["notebook"]["cells"][4]["outputs"][1]["text"] == + "$(with_include):$line_in_with_include" end -end \ No newline at end of file +end From b30972cbd67b4b578e6975a69779325c974fd2f8 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 13:02:17 +0200 Subject: [PATCH 05/14] explain test --- test/testsets/socket_server/socket_server.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index 9789469..a1e26ce 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -197,6 +197,7 @@ end collect(eachline(to_include)), ) + # check that the FILE/LINE printouts reflect the original files and line numbers @test result["notebook"]["cells"][2]["outputs"][1]["text"] == "$(to_include):$line_in_to_include" @test result["notebook"]["cells"][4]["outputs"][1]["text"] == From bea25bf36336d734882e1bd8b600d68d7494ec41 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 13:13:55 +0200 Subject: [PATCH 06/14] add changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad35839..9b17255 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added the ability to read a `sourceRanges` attribute from quarto to correctly point to included files in stacktraces [#339]. - Support new requirement from `quarto` to population environment variables explicitly in the notebook process [#306]. - Contributing guidelines with AI assistance policy and AGENTS.md for AI coding assistants [#335]. @@ -468,3 +469,4 @@ caching is enabled. Delete this folder to clear the cache. [#259] [#305]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/305 [#306]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/306 [#335]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/335 +[#339]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/339 From e71dd06ad426add101b39b20a1308b03d7d9c4ec Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 13:46:52 +0200 Subject: [PATCH 07/14] try fixing windows line splitting --- test/testsets/socket_server/socket_server.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index a1e26ce..f32a77e 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -153,7 +153,7 @@ end to_include_md = read(to_include, String) with_include_A, with_include_B = - split(with_include_md, "{{< include to_include.qmd >}}\n") + split(with_include_md, "{{< include to_include.qmd >}}\r?\n") lines_A = length(split(with_include_A, "\n")) lines_B = length(split(with_include_B, "\n")) From cd1aed4cf3830a2aaaf27c90a8e09ef22c99cc1e Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 14:04:03 +0200 Subject: [PATCH 08/14] wasn't a regex --- test/testsets/socket_server/socket_server.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index f32a77e..1a03f6b 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -153,7 +153,7 @@ end to_include_md = read(to_include, String) with_include_A, with_include_B = - split(with_include_md, "{{< include to_include.qmd >}}\r?\n") + split(with_include_md, r"{{< include to_include\.qmd >}}\r?\n") lines_A = length(split(with_include_A, "\n")) lines_B = length(split(with_include_B, "\n")) From 96a4f13de7eb5bded69fc7a3a4b17e3e5a0c73b3 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 15:15:09 +0200 Subject: [PATCH 09/14] use constant time lookup instead --- src/server.jl | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/server.jl b/src/server.jl index 37814c1..75978b8 100644 --- a/src/server.jl +++ b/src/server.jl @@ -624,6 +624,21 @@ raw_markdown_chunks(path::String) = struct Unset end +function compute_line_file_lookup(nlines, path, source_ranges) + lookup = fill((; file = "unknown", line = 0), nlines) + for source_range in source_ranges + file::String = something(source_range.file, "unknown") + for line in source_range.lines + source_line = line - source_range.lines.start + source_range.source_line + lookup[line] = (; file, line = source_line) + end + end + return lookup +end +function compute_line_file_lookup(nlines, path, source_ranges::Nothing) + return [(; file = path, line) for line in 1:nlines] +end + function raw_markdown_chunks_from_string( path::String, markdown::String; @@ -638,23 +653,7 @@ function raw_markdown_chunks_from_string( source_lines = collect(eachline(IOBuffer(markdown))) terminal_line = 1 - function source_file_and_line(terminal_line::Int) - if source_ranges === nothing - return (; file = path, line = terminal_line) - else - for source_range in source_ranges - if terminal_line in source_range.lines - file::String = something(source_range.file, "unknown") - source_line::Int = - terminal_line - source_range.lines.start + source_range.source_line - return (; file, line = source_line) - end - end - error( - "Terminal line $terminal_line was not included in source ranges, last range was $(last(source_ranges))", - ) - end - end + line_file_lookup = compute_line_file_lookup(length(source_lines), path, source_ranges) code_cells = false for (node, enter) in ast @@ -665,7 +664,7 @@ function raw_markdown_chunks_from_string( md = join(source_lines[terminal_line:(line-1)], "\n") push!( raw_chunks, - (; type = :markdown, source = md, source_file_and_line(terminal_line)...), + (; type = :markdown, source = md, line_file_lookup[terminal_line]...), ) if contains(md, r"`{(?:julia|python|r)} ") source_code_hash = hash(md, source_code_hash) @@ -676,7 +675,7 @@ function raw_markdown_chunks_from_string( # this option could in the future also include a vector of line numbers, which knitr supports. # all other options seem to be quarto-rendering related, like where to put figure captions etc. source = node.literal - cell_options = extract_cell_options(source; source_file_and_line(line)...) + cell_options = extract_cell_options(source; line_file_lookup[line]...) evaluate = get(cell_options, "eval", Unset()) if !(evaluate isa Union{Bool,Unset}) error( @@ -693,7 +692,7 @@ function raw_markdown_chunks_from_string( type = :code, language = language, source, - source_file_and_line(line)..., + line_file_lookup[line]..., evaluate, cell_options, ), @@ -705,7 +704,7 @@ function raw_markdown_chunks_from_string( md = join(source_lines[terminal_line:end], "\n") push!( raw_chunks, - (; type = :markdown, source = md, source_file_and_line(terminal_line)...), + (; type = :markdown, source = md, line_file_lookup[terminal_line]...), ) if contains(md, r"`{(?:julia|python|r)} ") source_code_hash = hash(md, source_code_hash) From e797d2257545ef1a68f26b766ddd90193a5bb8cd Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 15:21:21 +0200 Subject: [PATCH 10/14] add test for error condition --- src/server.jl | 2 +- test/testsets/socket_server/socket_server.jl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/server.jl b/src/server.jl index 75978b8..46adb05 100644 --- a/src/server.jl +++ b/src/server.jl @@ -636,7 +636,7 @@ function compute_line_file_lookup(nlines, path, source_ranges) return lookup end function compute_line_file_lookup(nlines, path, source_ranges::Nothing) - return [(; file = path, line) for line in 1:nlines] + return [(; file = path, line) for line = 1:nlines] end function raw_markdown_chunks_from_string( diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index 1a03f6b..51a00e1 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -202,5 +202,14 @@ end "$(to_include):$line_in_to_include" @test result["notebook"]["cells"][4]["outputs"][1]["text"] == "$(with_include):$line_in_with_include" + + # modify one of the source line boundaries so it mismatches + source_ranges[1].sourceLines[2] -= 1 + result = + json(`$node $client $(server.port) $(server.key) run $(JSON3.write(content))`) + @test contains( + result["juliaError"], + "Mismatching lengths of lines 1:5 (5) and source_lines 1:4", + ) end end From c5c09cf627caa7a7de4541d52ce7036dd6b1dcd7 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Sep 2025 16:37:06 +0200 Subject: [PATCH 11/14] test no source ranges as well to cover the last line --- test/testsets/socket_server/socket_server.jl | 31 ++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index 51a00e1..1cb0a63 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -184,9 +184,29 @@ end json(cmd) = JSON3.read(read(cmd, String), Any) options = (; target = (; markdown = (; value = full))) - content = (; file = with_include, sourceRanges = source_ranges, options) - result = - json(`$node $client $(server.port) $(server.key) run $(JSON3.write(content))`) + + full_lines = split(full, "\n") + + content_without_ranges = (; file = with_include, options) + result = json( + `$node $client $(server.port) $(server.key) run $(JSON3.write(content_without_ranges))`, + ) + + first_line_with_print = + findfirst(contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), full_lines) + last_line_with_print = + findlast(contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), full_lines) + + # check that the FILE/LINE printouts reflect only the concatenated (root) file + @test result["notebook"]["cells"][2]["outputs"][1]["text"] == + "$(with_include):$first_line_with_print" + @test result["notebook"]["cells"][4]["outputs"][1]["text"] == + "$(with_include):$last_line_with_print" + + content_with_ranges = (; file = with_include, sourceRanges = source_ranges, options) + result = json( + `$node $client $(server.port) $(server.key) run $(JSON3.write(content_with_ranges))`, + ) line_in_with_include = findfirst( contains(raw"""print("$(@__FILE__):$(@__LINE__)")"""), @@ -205,8 +225,9 @@ end # modify one of the source line boundaries so it mismatches source_ranges[1].sourceLines[2] -= 1 - result = - json(`$node $client $(server.port) $(server.key) run $(JSON3.write(content))`) + result = json( + `$node $client $(server.port) $(server.key) run $(JSON3.write(content_with_ranges))`, + ) @test contains( result["juliaError"], "Mismatching lengths of lines 1:5 (5) and source_lines 1:4", From 90d2aa2fdd9ddde5e2975640c0ca1ca58621dc0a Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 24 Sep 2025 11:18:06 +0200 Subject: [PATCH 12/14] hit one more edge case --- test/testsets/socket_server/socket_server.jl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/testsets/socket_server/socket_server.jl b/test/testsets/socket_server/socket_server.jl index 1cb0a63..3441938 100644 --- a/test/testsets/socket_server/socket_server.jl +++ b/test/testsets/socket_server/socket_server.jl @@ -159,9 +159,13 @@ end lines_B = length(split(with_include_B, "\n")) lines_to_include = length(split(to_include_md, "\n")) - full = join([with_include_A, to_include_md, with_include_B], "\n") + # this mocks the current behavior of quarto where it sometimes inserts newlines + # without tracking info after includes + empty_line = "" - ends = cumsum([lines_A, lines_to_include, lines_B]) + full = join([with_include_A, to_include_md, empty_line, with_include_B], "\n") + + ends = cumsum([lines_A, lines_to_include, 1, lines_B]) source_ranges = [ (; file = with_include, lines = [1, ends[1]], sourceLines = [1, lines_A]), @@ -171,8 +175,12 @@ end sourceLines = [1, lines_to_include], ), (; - file = with_include, + # the empty lines that quarto sometimes add lack file and sourceLines lines = [ends[2] + 1, ends[3]], + ), + (; + file = with_include, + lines = [ends[3] + 1, ends[4]], sourceLines = [lines_A + 1, lines_A + lines_B], ), ] From f012ba25c913c866feb0e5282c3183681209724f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 24 Sep 2025 12:46:52 +0200 Subject: [PATCH 13/14] handle quarto empty line bug --- src/server.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server.jl b/src/server.jl index 46adb05..4af9909 100644 --- a/src/server.jl +++ b/src/server.jl @@ -625,7 +625,8 @@ raw_markdown_chunks(path::String) = struct Unset end function compute_line_file_lookup(nlines, path, source_ranges) - lookup = fill((; file = "unknown", line = 0), nlines) + nlines_ranges = maximum(r -> r.lines.stop, source_ranges) # number of lines reported might be different from the markdown string due to quarto bugs + lookup = fill((; file = "unknown", line = 0), nlines_ranges) for source_range in source_ranges file::String = something(source_range.file, "unknown") for line in source_range.lines From 5d622ba88b8b1374af6158b6e66e105b611eb097 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 26 Sep 2025 14:32:56 +0200 Subject: [PATCH 14/14] replace `.start` with `first()` --- src/server.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.jl b/src/server.jl index 4af9909..3ed5c11 100644 --- a/src/server.jl +++ b/src/server.jl @@ -78,7 +78,7 @@ function SourceRange(file, lines, source_lines::UnitRange) "Mismatching lengths of lines $lines ($(length(lines))) and source_lines $source_lines ($(length(source_lines)))", ) end - SourceRange(file, lines, source_lines.start) + SourceRange(file, lines, first(source_lines)) end function _has_juliaup() @@ -630,7 +630,7 @@ function compute_line_file_lookup(nlines, path, source_ranges) for source_range in source_ranges file::String = something(source_range.file, "unknown") for line in source_range.lines - source_line = line - source_range.lines.start + source_range.source_line + source_line = line - first(source_range.lines) + source_range.source_line lookup[line] = (; file, line = source_line) end end