From 11779384fc1f9c393b9b466e7b9f3296539ddf61 Mon Sep 17 00:00:00 2001 From: Casey Duckering Date: Sun, 19 Jul 2020 01:22:12 -0500 Subject: [PATCH 1/5] Add support for cell macros and registering automatic macros --- src/execute_request.jl | 62 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index d5548806..810db607 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -16,6 +16,65 @@ import REPL: helpmode # use a global array to accumulate "payloads" for the execute_reply message const execute_payloads = Dict[] + +const cell_macros = [] + +function run_cell_code(code) + do_auto_macros = true + + # Check for cell macros + cell_macro_calls = [] + line_no = 1 + if startswith(code, "@@") + lines = split(code, '\n') + for line in lines + startswith(line, "@@") || break + mac_call = Meta.parse(line[2:end]) + @assert Meta.isexpr(mac_call, :macrocall) + if mac_call.args[1] == Symbol("@nothing") + do_auto_macros = false + continue + end + push!(cell_macro_calls, mac_call) + line_no += 1 + end + code = join((@view lines[line_no:end]), '\n') + end + + # Apply macros to ast + mod = current_module[] + file = "In[$n]" + line_node = LineNumberNode(line_no, file) + ast = Meta.parse("begin\n$(code)\nend") + for mac_call in Iterators.reverse(cell_macro_calls) + push!(mac_call.args, ast) + ast = macroexpand(mod, mac_call) + end + if do_auto_macros + if SOFTSCOPE[] + ast = _apply_macro(var"@softscope", ast, mod) + end + for mac in Iterators.reverse(cell_macros) + ast = _apply_macro(mac, ast, mod) + end + end + + # Make top level + if ast isa Expr && Meta.isexpr(ast.head, :block) + ast.head = :toplevel + end + + # Run + Base.eval(mod, ast) +end + +function _apply_macro(mac, ast, mod) + macro_expr = :(@macro_placehoder $ast) + @assert Meta.isexpr(macro_expr, :macrocall) + macro_expr.args[1] = mac + macroexpand(mod, macro_expr) +end + function execute_request(socket, msg) code = msg.content["code"] @vprintln("EXECUTING ", code) @@ -65,8 +124,7 @@ function execute_request(socket, msg) else #run the code! occursin(magics_regex, code) && match(magics_regex, code).offset == 1 ? magics_help(code) : - SOFTSCOPE[] ? softscope_include_string(current_module[], code, "In[$n]") : - include_string(current_module[], code, "In[$n]") + run_cell_code(code) end if silent From 68d4e10321a37a112241d4443c31875547543b2b Mon Sep 17 00:00:00 2001 From: Casey Duckering Date: Mon, 20 Jul 2020 02:27:29 -0500 Subject: [PATCH 2/5] Bug fixes --- src/execute_request.jl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index 810db607..2ec086a1 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -16,7 +16,7 @@ import REPL: helpmode # use a global array to accumulate "payloads" for the execute_reply message const execute_payloads = Dict[] - +# An array of macros to run on the contents of each cell const cell_macros = [] function run_cell_code(code) @@ -31,11 +31,11 @@ function run_cell_code(code) startswith(line, "@@") || break mac_call = Meta.parse(line[2:end]) @assert Meta.isexpr(mac_call, :macrocall) - if mac_call.args[1] == Symbol("@nothing") + if mac_call.args[1] == Symbol("@noauto") do_auto_macros = false - continue + else + push!(cell_macro_calls, mac_call) end - push!(cell_macro_calls, mac_call) line_no += 1 end code = join((@view lines[line_no:end]), '\n') @@ -46,9 +46,13 @@ function run_cell_code(code) file = "In[$n]" line_node = LineNumberNode(line_no, file) ast = Meta.parse("begin\n$(code)\nend") + # Make block top level + if Meta.isexpr(ast, :block) + ast.head = :toplevel + end for mac_call in Iterators.reverse(cell_macro_calls) push!(mac_call.args, ast) - ast = macroexpand(mod, mac_call) + ast = macroexpand(mod, mac_call, recursive=false) end if do_auto_macros if SOFTSCOPE[] @@ -59,11 +63,6 @@ function run_cell_code(code) end end - # Make top level - if ast isa Expr && Meta.isexpr(ast.head, :block) - ast.head = :toplevel - end - # Run Base.eval(mod, ast) end @@ -72,7 +71,7 @@ function _apply_macro(mac, ast, mod) macro_expr = :(@macro_placehoder $ast) @assert Meta.isexpr(macro_expr, :macrocall) macro_expr.args[1] = mac - macroexpand(mod, macro_expr) + macroexpand(mod, macro_expr, recursive=false) end function execute_request(socket, msg) From 179562c791fbdce5c95907b3ba3bed999e8cbd18 Mon Sep 17 00:00:00 2001 From: Casey Duckering Date: Mon, 20 Jul 2020 03:18:34 -0500 Subject: [PATCH 3/5] Fix string index --- src/execute_request.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index 2ec086a1..344e736d 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -29,7 +29,7 @@ function run_cell_code(code) lines = split(code, '\n') for line in lines startswith(line, "@@") || break - mac_call = Meta.parse(line[2:end]) + mac_call = Meta.parse(line[nextind("abc", 0, 2):end]) @assert Meta.isexpr(mac_call, :macrocall) if mac_call.args[1] == Symbol("@noauto") do_auto_macros = false From 1d5d973112f772bd4c155f134e4fa0ad4385f6fd Mon Sep 17 00:00:00 2001 From: Casey Duckering Date: Mon, 20 Jul 2020 03:48:49 -0500 Subject: [PATCH 4/5] Add completion for cell macros --- src/execute_request.jl | 5 +++++ src/handlers.jl | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index 344e736d..f06c62b3 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -28,6 +28,11 @@ function run_cell_code(code) if startswith(code, "@@") lines = split(code, '\n') for line in lines + line = strip(line) + if length(line) <= 0 + line_no += 1 + continue + end startswith(line, "@@") || break mac_call = Meta.parse(line[nextind("abc", 0, 2):end]) @assert Meta.isexpr(mac_call, :macrocall) diff --git a/src/handlers.jl b/src/handlers.jl index a0e25a56..a2e0f5a4 100644 --- a/src/handlers.jl +++ b/src/handlers.jl @@ -7,10 +7,22 @@ using .CommManager parseok(s) = !Meta.isexpr(Meta.parse(s, raise=false), :error) function find_parsestart(code, cursorpos) s = firstindex(code) + in_macro_header = true while s < cursorpos + s_next = nextind(code, s) + if in_macro_header && s_next <= cursorpos && code[s:s_next] == "@@" + # Skip first '@' to pretend it's a normal macro for completion + s, s_next = s_next, nextind(code, s_next) + else + in_macro_header = false + end parseok(code[s:cursorpos]) && return s - s = nextind(code, s) - while s < cursorpos && code[s] ∉ ('\n','\r') + # Go to next line + s = s_next + while s < cursorpos && code[s] ∉ "\n\r" + s = nextind(code, s) + end + while s < cursorpos && code[s] ∈ "\n\r \t" s = nextind(code, s) end end From 23fae651b669f7ed2a7452f7c3a6c36231fd7e1e Mon Sep 17 00:00:00 2001 From: Casey Duckering Date: Mon, 20 Jul 2020 17:12:02 -0500 Subject: [PATCH 5/5] Fix cell filename in stack traces --- src/execute_request.jl | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index f06c62b3..ae39092d 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -49,12 +49,17 @@ function run_cell_code(code) # Apply macros to ast mod = current_module[] file = "In[$n]" - line_node = LineNumberNode(line_no, file) - ast = Meta.parse("begin\n$(code)\nend") - # Make block top level - if Meta.isexpr(ast, :block) - ast.head = :toplevel - end + + ast = parse_file(code, file, false) + # More compatible? + #ast = Meta.parse("begin\n$(code)\nend", raise=false) + ## Make block top level + #if Meta.isexpr(ast, :block) + # ast.head = :toplevel + # line_node = LineNumberNode(line_no, file) + # push!(ast.args, 1, line_node) + #end + for mac_call in Iterators.reverse(cell_macro_calls) push!(mac_call.args, ast) ast = macroexpand(mod, mac_call, recursive=false) @@ -79,6 +84,29 @@ function _apply_macro(mac, ast, mod) macroexpand(mod, macro_expr, recursive=false) end +""" + parse_file(content, fname; raise=true, depwarn=true) + +Parse the entire string as a file, reading multiple expressions. Equivalent to +`Meta.parse()` but for more than one expression. +""" +function parse_file(content::AbstractString, fname::AbstractString, + raise::Bool=true, depwarn::Bool=true) + # returns (expr, end_pos). expr is () in case of parse error. + bcontent = String(content) + bfname = String(fname) + # For now, assume all parser warnings are depwarns + ex = Meta.with_logger(depwarn ? Meta.current_logger() : Meta.NullLogger()) do + ccall(:jl_parse_all, Any, + (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t), + bcontent, sizeof(bcontent), fname, sizeof(fname)) + end + if raise && Meta.isexpr(ex, :error) + throw(ParseError(ex.args[1])) + end + ex +end + function execute_request(socket, msg) code = msg.content["code"] @vprintln("EXECUTING ", code)