Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions base/client.jl
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,8 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool)
nothing
end

function _parse_input_line_core(s::String, filename::String)
ex = Meta.parseall(s, filename=filename)
function _parse_input_line_core(s::String, filename::String, mod::Union{Module, Nothing})
ex = Meta.parseall(s; filename, mod)
if ex isa Expr && ex.head === :toplevel
if isempty(ex.args)
return nothing
Expand All @@ -189,18 +189,18 @@ function _parse_input_line_core(s::String, filename::String)
return ex
end

function parse_input_line(s::String; filename::String="none", depwarn=true)
function parse_input_line(s::String; filename::String="none", depwarn=true, mod::Union{Module, Nothing}=nothing)
# For now, assume all parser warnings are depwarns
ex = if depwarn
_parse_input_line_core(s, filename)
_parse_input_line_core(s, filename, mod)
else
with_logger(NullLogger()) do
_parse_input_line_core(s, filename)
_parse_input_line_core(s, filename, mod)
end
end
return ex
end
parse_input_line(s::AbstractString) = parse_input_line(String(s))
parse_input_line(s::AbstractString; kwargs...) = parse_input_line(String(s); kwargs...)

# detect the reason which caused an :incomplete expression
# from the error message
Expand Down Expand Up @@ -443,7 +443,7 @@ function run_fallback_repl(interactive::Bool)
let input = stdin
if isa(input, File) || isa(input, IOStream)
# for files, we can slurp in the whole thing at once
ex = parse_input_line(read(input, String))
ex = parse_input_line(read(input, String); mod=Main)
if Meta.isexpr(ex, :toplevel)
# if we get back a list of statements, eval them sequentially
# as if we had parsed them sequentially
Expand All @@ -466,7 +466,7 @@ function run_fallback_repl(interactive::Bool)
ex = nothing
while !eof(input)
line *= readline(input, keep=true)
ex = parse_input_line(line)
ex = parse_input_line(line; mod=Main)
if !(isa(ex, Expr) && ex.head === :incomplete)
break
end
Expand Down
44 changes: 44 additions & 0 deletions base/experimental.jl
Original file line number Diff line number Diff line change
Expand Up @@ -746,4 +746,48 @@ macro reexport(ex)
return esc(calls)
end

struct VersionedParse
ver::VersionNumber
end

function (vp::VersionedParse)(code, filename::String, lineno::Int, offset::Int, options::Symbol)
if !isdefined(Base, :JuliaSyntax)
if vp.ver === VERSION
return Core._parse
end
error("JuliaSyntax module is required for syntax version $(vp.ver), but it is not loaded.")
end
Base.JuliaSyntax.core_parser_hook(code, filename, lineno, offset, options; syntax_version=vp.ver)
end

struct VersionedLower
ver::VersionNumber
end

function (vp::VersionedLower)(@nospecialize(code), mod::Module,
file="none", line=0, world=typemax(Csize_t), warn=false)
if !isdefined(Base, :JuliaLowering)
if vp.ver === VERSION
return Core._parse
end
error("JuliaLowering module is required for syntax version $(vp.ver), but it is not loaded.")
end
Base.JuliaLowering.core_lowering_hook(code, filename, lineno, offset, options; syntax_version=vp.ver)
end

function set_syntax_version(m::Module, ver::VersionNumber)
if !Base.is_root_module(m)
error("set_syntax_version can only be called on root modules")
end
parser = VersionedParse(ver)
lowerer = VersionedLower(ver)
Core.declare_const(m, :_internal_julia_parse, parser)
Core.declare_const(m, :_internal_julia_lower, lowerer)
nothing
end

macro set_syntax_version(ver)
Expr(:call, set_syntax_version, __module__, esc(ver))
end

end # module
30 changes: 20 additions & 10 deletions base/meta.jl
Original file line number Diff line number Diff line change
Expand Up @@ -304,12 +304,22 @@ end

ParseError(msg::AbstractString) = ParseError(msg, nothing)

# N.B.: Should match definition in src/ast.c:jl_parse
function parser_for_module(mod::Union{Module, Nothing})
mod === nothing && return Core._parse
mod = Base.moduleroot(mod)
isdefined(mod, :_internal_julia_parse) ?
getglobal(mod, :_internal_julia_parse) :
Core._parse
end

function _parse_string(text::AbstractString, filename::AbstractString,
lineno::Integer, index::Integer, options)
lineno::Integer, index::Integer, options,
_parse=parser_for_module(nothing))
if index < 1 || index > ncodeunits(text) + 1
throw(BoundsError(text, index))
end
ex, offset::Int = Core._parse(text, filename, lineno, index-1, options)
ex, offset::Int = _parse(text, filename, lineno, index-1, options)
ex, offset+1
end

Expand Down Expand Up @@ -346,8 +356,8 @@ julia> Meta.parse("(α, β) = 3, 5", 11, greedy=false)
```
"""
function parse(str::AbstractString, pos::Integer;
filename="none", greedy::Bool=true, raise::Bool=true, depwarn::Bool=true)
ex, pos = _parse_string(str, String(filename), 1, pos, greedy ? :statement : :atom)
filename="none", greedy::Bool=true, raise::Bool=true, depwarn::Bool=true, mod = nothing)
ex, pos = _parse_string(str, String(filename), 1, pos, greedy ? :statement : :atom, parser_for_module(mod))
if raise && isexpr(ex, :error)
err = ex.args[1]
if err isa String
Expand Down Expand Up @@ -386,8 +396,8 @@ julia> Meta.parse("x = ")
```
"""
function parse(str::AbstractString;
filename="none", raise::Bool=true, depwarn::Bool=true)
ex, pos = parse(str, 1; filename, greedy=true, raise, depwarn)
filename="none", raise::Bool=true, depwarn::Bool=true, mod = nothing)
ex, pos = parse(str, 1; filename, greedy=true, raise, depwarn, mod = mod)
if isexpr(ex, :error)
return ex
end
Expand All @@ -398,12 +408,12 @@ function parse(str::AbstractString;
return ex
end

function parseatom(text::AbstractString, pos::Integer; filename="none", lineno=1)
return _parse_string(text, String(filename), lineno, pos, :atom)
function parseatom(text::AbstractString, pos::Integer; filename="none", lineno=1, mod = nothing)
return _parse_string(text, String(filename), lineno, pos, :atom, parser_for_module(mod))
end

function parseall(text::AbstractString; filename="none", lineno=1)
ex,_ = _parse_string(text, String(filename), lineno, 1, :all)
function parseall(text::AbstractString; filename="none", lineno=1, mod = nothing)
ex,_ = _parse_string(text, String(filename), lineno, 1, :all, parser_for_module(mod))
return ex
end

Expand Down
34 changes: 21 additions & 13 deletions src/ast.c
Original file line number Diff line number Diff line change
Expand Up @@ -1224,15 +1224,19 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule,
const char *filename, int line, size_t world, bool_t warn)
{
jl_value_t *core_lower = NULL;
if (jl_core_module)
core_lower = jl_get_global_value(jl_core_module, jl_symbol("_lower"), jl_current_task->world_age);
if (!core_lower || core_lower == jl_nothing) {
jl_value_t *julia_lower = NULL;
if (inmodule) {
jl_module_t *this_root_module = jl_module_root(inmodule);
julia_lower = jl_get_global(this_root_module, jl_symbol("_internal_julia_lower"));
}
if ((!julia_lower || julia_lower == jl_nothing) && jl_core_module)
julia_lower = jl_get_global_value(jl_core_module, jl_symbol("_lower"), jl_current_task->world_age);
if (!julia_lower || julia_lower == jl_nothing) {
return jl_fl_lower(expr, inmodule, filename, line, world, warn);
}
jl_value_t **args;
JL_GC_PUSHARGS(args, 7);
args[0] = core_lower;
args[0] = julia_lower;
args[1] = expr;
args[2] = (jl_value_t*)inmodule;
args[3] = jl_cstr_to_string(filename);
Expand Down Expand Up @@ -1288,20 +1292,24 @@ jl_code_info_t *jl_inner_ctor_body(jl_array_t *fieldkinds, jl_module_t *inmodule
// `text` is passed as a pointer to allow raw non-String buffers to be used
// without copying.
jl_value_t *jl_parse(const char *text, size_t text_len, jl_value_t *filename,
size_t lineno, size_t offset, jl_value_t *options)
size_t lineno, size_t offset, jl_value_t *options, jl_module_t *inmodule)
{
jl_value_t *core_parse = NULL;
if (jl_core_module) {
core_parse = jl_get_global(jl_core_module, jl_symbol("_parse"));
jl_value_t *parser = NULL;
if (inmodule) {
inmodule = jl_module_root(inmodule);
parser = jl_get_global(inmodule, jl_symbol("_internal_julia_parse"));
}
if ((!parser || parser == jl_nothing) && jl_core_module) {
parser = jl_get_global(jl_core_module, jl_symbol("_parse"));
}
if (!core_parse || core_parse == jl_nothing) {
if (!parser || parser == jl_nothing) {
// In bootstrap, directly call the builtin parser.
jl_value_t *result = jl_fl_parse(text, text_len, filename, lineno, offset, options);
return result;
}
jl_value_t **args;
JL_GC_PUSHARGS(args, 6);
args[0] = core_parse;
args[0] = parser;
args[1] = (jl_value_t*)jl_alloc_svec(2);
jl_svecset(args[1], 0, jl_box_uint8pointer((uint8_t*)text));
jl_svecset(args[1], 1, jl_box_long(text_len));
Expand Down Expand Up @@ -1330,7 +1338,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len,
{
jl_value_t *fname = jl_pchar_to_string(filename, filename_len);
JL_GC_PUSH1(&fname);
jl_value_t *p = jl_parse(text, text_len, fname, lineno, 0, (jl_value_t*)jl_all_sym);
jl_value_t *p = jl_parse(text, text_len, fname, lineno, 0, (jl_value_t*)jl_all_sym, NULL);
JL_GC_POP();
return jl_svecref(p, 0);
}
Expand All @@ -1343,7 +1351,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
jl_value_t *fname = jl_cstr_to_string("none");
JL_GC_PUSH1(&fname);
jl_value_t *result = jl_parse(text, text_len, fname, 1, offset,
(jl_value_t*)(greedy ? jl_statement_sym : jl_atom_sym));
(jl_value_t*)(greedy ? jl_statement_sym : jl_atom_sym), NULL);
JL_GC_POP();
return result;
}
Expand Down
3 changes: 2 additions & 1 deletion src/julia_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -932,6 +932,7 @@ STATIC_INLINE size_t module_usings_max(jl_module_t *m) JL_NOTSAFEPOINT {
}

JL_DLLEXPORT jl_sym_t *jl_module_name(jl_module_t *m) JL_NOTSAFEPOINT;
jl_module_t *jl_module_root(jl_module_t *m);
void jl_add_scanned_method(jl_module_t *m, jl_method_t *meth);
jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e, size_t world);
JL_DLLEXPORT jl_value_t *jl_eval_globalref(jl_globalref_t *g, size_t world);
Expand Down Expand Up @@ -1366,7 +1367,7 @@ jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs
JL_DLLEXPORT int jl_has_meta(jl_array_t *body, jl_sym_t *sym) JL_NOTSAFEPOINT;

JL_DLLEXPORT jl_value_t *jl_parse(const char *text, size_t text_len, jl_value_t *filename,
size_t lineno, size_t offset, jl_value_t *options);
size_t lineno, size_t offset, jl_value_t *options, jl_module_t *inmodule);
jl_code_info_t *jl_inner_ctor_body(jl_array_t *fieldkinds, jl_module_t *inmodule, const char *file, int line);
jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line);
void jl_ctor_def(jl_value_t *ty, jl_value_t *functionloc);
Expand Down
2 changes: 0 additions & 2 deletions src/timing.c
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
#define DISABLE_FREQUENT_EVENTS
#endif

jl_module_t *jl_module_root(jl_module_t *m);

#ifdef __cplusplus
extern "C" {
#endif
Expand Down
2 changes: 1 addition & 1 deletion src/toplevel.c
Original file line number Diff line number Diff line change
Expand Up @@ -830,7 +830,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text,
JL_GC_PUSH3(&ast, &result, &expression);

ast = jl_svecref(jl_parse(jl_string_data(text), jl_string_len(text),
filename, 1, 0, (jl_value_t*)jl_all_sym), 0);
filename, 1, 0, (jl_value_t*)jl_all_sym, module), 0);
if (!jl_is_expr(ast) || ((jl_expr_t*)ast)->head != jl_toplevel_sym) {
jl_errorf("jl_parse_all() must generate a top level expression");
}
Expand Down
15 changes: 8 additions & 7 deletions stdlib/REPL/src/REPL.jl
Original file line number Diff line number Diff line change
Expand Up @@ -740,7 +740,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef)
rethrow()
end
end
ast = Base.parse_input_line(line)
ast = Base.parse_input_line(line; mod=Base.active_module(repl))
(isa(ast,Expr) && ast.head === :incomplete) || break
end
if !isempty(line)
Expand Down Expand Up @@ -814,7 +814,8 @@ REPLCompletionProvider() = REPLCompletionProvider(LineEdit.Modifiers())
mutable struct ShellCompletionProvider <: CompletionProvider end
struct LatexCompletions <: CompletionProvider end

Base.active_module((; mistate)::LineEditREPL) = mistate === nothing ? Main : mistate.active_module
Base.active_module(mistate::MIState) = mistate.active_module
Base.active_module((; mistate)::LineEditREPL) = mistate === nothing ? Main : Base.active_module(mistate)
Base.active_module(::AbstractREPL) = Main
Base.active_module(d::REPLDisplay) = Base.active_module(d.repl)

Expand Down Expand Up @@ -1117,7 +1118,7 @@ end
LineEdit.reset_state(hist::REPLHistoryProvider) = history_reset_state(hist)

function return_callback(s)
ast = Base.parse_input_line(takestring!(copy(LineEdit.buffer(s))), depwarn=false)
ast = Base.parse_input_line(takestring!(copy(LineEdit.buffer(s))); mod=Base.active_module(s), depwarn=false)
return !(isa(ast, Expr) && ast.head === :incomplete)
end

Expand Down Expand Up @@ -1286,7 +1287,7 @@ function setup_interface(
repl = repl,
complete = replc,
# When we're done transform the entered line into a call to helpmode function
on_done = respond(line::String->helpmode(outstream(repl), line, repl.mistate.active_module),
on_done = respond(line::String->helpmode(outstream(repl), line, Base.active_module(repl)),
repl, julia_prompt, pass_empty=true, suppress_on_semicolon=false))


Expand Down Expand Up @@ -1367,7 +1368,7 @@ function setup_interface(
help_mode.hist = hp
dummy_pkg_mode.hist = hp

julia_prompt.on_done = respond(x->Base.parse_input_line(x,filename=repl_filename(repl,hp)), repl, julia_prompt)
julia_prompt.on_done = respond(x->Base.parse_input_line(x; filename=repl_filename(repl,hp), mod=Base.active_module(repl)), repl, julia_prompt)

shell_prompt_len = length(SHELL_PROMPT)
help_prompt_len = length(HELP_PROMPT)
Expand Down Expand Up @@ -1531,7 +1532,7 @@ function setup_interface(
dump_tail = false
nl_pos = findfirst('\n', input[oldpos:end])
if s.current_mode == julia_prompt
ast, pos = Meta.parse(input, oldpos, raise=false, depwarn=false)
ast, pos = Meta.parse(input, oldpos, raise=false, depwarn=false, mod=Base.active_module(s))
if (isa(ast, Expr) && (ast.head === :error || ast.head === :incomplete)) ||
(pos > ncodeunits(input) && !endswith(input, '\n'))
# remaining text is incomplete (an error, or parser ran to the end but didn't stop with a newline):
Expand Down Expand Up @@ -1787,7 +1788,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef)
end
line = readline(repl.stream, keep=true)
if !isempty(line)
ast = Base.parse_input_line(line)
ast = Base.parse_input_line(line; mod=Base.active_module(repl))
if have_color
print(repl.stream, Base.color_normal)
end
Expand Down
1 change: 1 addition & 0 deletions stdlib/REPL/src/REPLCompletions.jl
Original file line number Diff line number Diff line change
Expand Up @@ -993,6 +993,7 @@ end

function completions(string::String, pos::Int, context_module::Module=Main, shift::Bool=true, hint::Bool=false)
# filename needs to be string so macro can be evaluated
# TODO: JuliaSyntax version API here
node = parseall(CursorNode, string, ignore_errors=true, keep_parens=true, filename="none")
cur = @something seek_pos(node, pos) node

Expand Down