Source ranges (#339)

* implement source range mechanism

* don't look up additional line that doesn't exist

* add test

* formatting

* explain test

* add changelog entry

* try fixing windows line splitting

* wasn't a regex

* use constant time lookup instead

* add test for error condition

* test no source ranges as well to cover the last line

* hit one more edge case

* handle quarto empty line bug

* replace `.start` with `first()`

Author: jkrumbiegel

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added the ability to read a `sourceRanges` attribute from quarto to correctly point to included files in stacktraces [#339].
 - Support new requirement from `quarto` to population environment variables
   explicitly in the notebook process [#306].
 - Contributing guidelines with AI assistance policy and AGENTS.md for AI coding assistants [#335].
@@ -468,3 +469,4 @@ caching is enabled. Delete this folder to clear the cache. [#259]
 [#305]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/305
 [#306]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/306
 [#335]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/335
+[#339]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/339

src/server.jl

@@ -66,6 +66,21 @@ mutable struct File
     end
 end
 
+struct SourceRange
+    file::Union{String,Nothing}
+    lines::UnitRange{Int}
+    source_line::Union{Nothing,Int} # first line in source
+end
+
+function SourceRange(file, lines, source_lines::UnitRange)
+    if length(lines) != length(source_lines)
+        error(
+            "Mismatching lengths of lines $lines ($(length(lines))) and source_lines $source_lines ($(length(source_lines)))",
+        )
+    end
+    SourceRange(file, lines, first(source_lines))
+end
+
 function _has_juliaup()
     try
         success(`juliaup --version`) && success(`julia --version`)
@@ -339,13 +354,15 @@ function evaluate!(
     options::Union{String,Dict{String,Any}} = Dict{String,Any}(),
     chunk_callback = (i, n, c) -> nothing,
     markdown::Union{String,Nothing} = nothing,
+    source_ranges::Union{Nothing,Vector} = nothing,
 )
     _check_output_dst(output)
 
     options = _parsed_options(options)
     path = abspath(f.path)
     if isfile(path)
-        source_code_hash, raw_chunks, file_frontmatter = raw_text_chunks(f, markdown)
+        source_code_hash, raw_chunks, file_frontmatter =
+            raw_text_chunks(f, markdown; source_ranges)
         merged_options = _extract_relevant_options(file_frontmatter, options)
 
         # A change of parameter values must invalidate the source code hash.
@@ -583,9 +600,9 @@ write_json(::Nothing, data) = data
 Return a vector of raw markdown and code chunks from `file` ready for evaluation
 by `evaluate_raw_cells!`.
 """
-raw_text_chunks(file::File, ::Nothing) = raw_text_chunks(file.path)
-raw_text_chunks(file::File, markdown::String) =
-    raw_markdown_chunks_from_string(file.path, markdown)
+raw_text_chunks(file::File, ::Nothing; source_ranges = nothing) = raw_text_chunks(file.path)
+raw_text_chunks(file::File, markdown::String; source_ranges = nothing) =
+    raw_markdown_chunks_from_string(file.path, markdown; source_ranges)
 
 function raw_text_chunks(path::String)
     endswith(path, ".qmd") && return raw_markdown_chunks(path)
@@ -607,7 +624,27 @@ raw_markdown_chunks(path::String) =
 
 struct Unset end
 
-function raw_markdown_chunks_from_string(path::String, markdown::String)
+function compute_line_file_lookup(nlines, path, source_ranges)
+    nlines_ranges = maximum(r -> r.lines.stop, source_ranges) # number of lines reported might be different from the markdown string due to quarto bugs
+    lookup = fill((; file = "unknown", line = 0), nlines_ranges)
+    for source_range in source_ranges
+        file::String = something(source_range.file, "unknown")
+        for line in source_range.lines
+            source_line = line - first(source_range.lines) + source_range.source_line
+            lookup[line] = (; file, line = source_line)
+        end
+    end
+    return lookup
+end
+function compute_line_file_lookup(nlines, path, source_ranges::Nothing)
+    return [(; file = path, line) for line = 1:nlines]
+end
+
+function raw_markdown_chunks_from_string(
+    path::String,
+    markdown::String;
+    source_ranges = nothing,
+)
     raw_chunks = []
     source_code_hash = hash(VERSION)
     pars = Parser()
@@ -616,6 +653,9 @@ function raw_markdown_chunks_from_string(path::String, markdown::String)
     source_code_hash = hash(file_fromtmatter, source_code_hash)
     source_lines = collect(eachline(IOBuffer(markdown)))
     terminal_line = 1
+
+    line_file_lookup = compute_line_file_lookup(length(source_lines), path, source_ranges)
+
     code_cells = false
     for (node, enter) in ast
         if enter &&
@@ -625,7 +665,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String)
             md = join(source_lines[terminal_line:(line-1)], "\n")
             push!(
                 raw_chunks,
-                (type = :markdown, source = md, file = path, line = terminal_line),
+                (; type = :markdown, source = md, line_file_lookup[terminal_line]...),
             )
             if contains(md, r"`{(?:julia|python|r)} ")
                 source_code_hash = hash(md, source_code_hash)
@@ -636,7 +676,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String)
             # this option could in the future also include a vector of line numbers, which knitr supports.
             # all other options seem to be quarto-rendering related, like where to put figure captions etc.
             source = node.literal
-            cell_options = extract_cell_options(source; file = path, line = line)
+            cell_options = extract_cell_options(source; line_file_lookup[line]...)
             evaluate = get(cell_options, "eval", Unset())
             if !(evaluate isa Union{Bool,Unset})
                 error(
@@ -649,12 +689,11 @@ function raw_markdown_chunks_from_string(path::String, markdown::String)
                 is_r_toplevel(node) ? :r : error("Unhandled code block language")
             push!(
                 raw_chunks,
-                (
+                (;
                     type = :code,
                     language = language,
                     source,
-                    file = path,
-                    line,
+                    line_file_lookup[line]...,
                     evaluate,
                     cell_options,
                 ),
@@ -666,7 +705,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String)
         md = join(source_lines[terminal_line:end], "\n")
         push!(
             raw_chunks,
-            (type = :markdown, source = md, file = path, line = terminal_line),
+            (; type = :markdown, source = md, line_file_lookup[terminal_line]...),
         )
         if contains(md, r"`{(?:julia|python|r)} ")
             source_code_hash = hash(md, source_code_hash)
@@ -690,7 +729,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String)
     if raw_chunks[end].type == :code
         push!(
             raw_chunks,
-            (type = :markdown, source = "", file = path, line = terminal_line),
+            (; type = :markdown, source = "", file = path, line = terminal_line),
         )
     end
 
@@ -1499,6 +1538,7 @@ function run!(
     showprogress::Bool = true,
     options::Union{String,Dict{String,Any}} = Dict{String,Any}(),
     chunk_callback = (i, n, c) -> nothing,
+    source_ranges::Union{Nothing,Vector} = nothing,
 )
     try
         borrow_file!(server, path; options, optionally_create = true) do file
@@ -1519,7 +1559,15 @@ function run!(
 
             result_task = Threads.@spawn begin
                 try
-                    evaluate!(file, output; showprogress, options, markdown, chunk_callback)
+                    evaluate!(
+                        file,
+                        output;
+                        showprogress,
+                        options,
+                        markdown,
+                        chunk_callback,
+                        source_ranges,
+                    )
                 finally
                     put!(file.run_decision_channel, :evaluate_finished)
                 end

src/socket.jl

@@ -345,9 +345,6 @@ function _handle_response_internal(
     # Running:
 
     if type == "run"
-        options = _get_options(request.content)
-        markdown = _get_markdown(options)
-
         function chunk_callback(i, n, chunk)
             _write_json(
                 socket,
@@ -362,6 +359,10 @@ function _handle_response_internal(
         end
 
         result = try
+            options = _get_options(request.content)
+            markdown = _get_markdown(options)
+            source_ranges = _get_source_ranges(request.content)
+
             (;
                 notebook = run!(
                     notebooks,
@@ -370,6 +371,7 @@ function _handle_response_internal(
                     markdown,
                     showprogress,
                     chunk_callback,
+                    source_ranges,
                 )
             )
         catch error
@@ -464,6 +466,26 @@ _get_file(content::String) = content
 _get_options(content::Dict) = get(Dict{String,Any}, content, "options")
 _get_options(::String) = Dict{String,Any}()
 
+function _get_source_ranges(content::Dict)
+    ranges = get(content, "sourceRanges", nothing)
+    ranges === nothing && return nothing
+    return map(ranges) do range
+        file = get(range, "file", nothing)
+        _lines::Vector{Int} = range["lines"]
+        @assert length(_lines) == 2
+        lines = _lines[1]:_lines[2]
+        _source_lines::Union{Nothing,Vector{Int}} = get(range, "sourceLines", nothing)
+        source_lines = if _source_lines === nothing
+            1:length(lines) # source lines are only missing in degenerate cases like additional newlines anyway so this doesn't really matter
+        else
+            @assert length(_source_lines) == 2
+            _source_lines[1]:_source_lines[2]
+        end
+        SourceRange(file, lines, source_lines)
+    end
+end
+_get_source_ranges(::String) = nothing
+
 function _get_nested(d::Dict, keys...)
     _d = d
     for key in keys

test/examples/sourceranges/to_include.qmd

@@ -0,0 +1,3 @@
+```{julia}
+print("$(@__FILE__):$(@__LINE__)")
+```

test/examples/sourceranges/with_include.qmd

@@ -0,0 +1,9 @@
+---
+engine: julia
+---
+
+{{< include to_include.qmd >}}
+
+```{julia}
+print("$(@__FILE__):$(@__LINE__)")
+```

test/testsets/socket_server/client.js

@@ -22,27 +22,30 @@ function handle() {
     const isready = () => toJSON({ type: 'isready', content: '' });
     const status = () => toJSON({ type: 'status', content: '' });
 
-    const notebook = (arg) => {
-        if (path.isAbsolute(arg)) {
-            return arg
+    const content = (arg) => {
+        if (typeof arg === 'string') {
+            if (path.isAbsolute(arg)) {
+                return arg
+            }
+            throw new Error('No notebook with absolute path specified.');
         }
-        throw new Error('No notebook with absolute path specified.');
+        return arg;
     }
 
     const type = process.argv[4];
-    const arg = process.argv[5];
+    const arg = process.argv.length >= 6 ? JSON.parse(process.argv[5]) : undefined;
 
     switch (type) {
         case 'run':
-            return run(notebook(arg));
+            return run(content(arg));
         case 'close':
-            return close(notebook(arg));
+            return close(content(arg));
         case 'forceclose':
-            return forceclose(notebook(arg));
+            return forceclose(content(arg));
         case 'stop':
             return stop();
         case 'isopen':
-            return isopen(notebook(arg));
+            return isopen(content(arg));
         case 'isready':
             return isready();
         case 'status':