Skip to content

Revert to the use of StringMacroName/CmdMacroName kinds #583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 12, 2025
Merged

Revert to the use of StringMacroName/CmdMacroName kinds #583

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
209ca63
Revert to the use of StringMacroName/CmdMacroName kinds
c42f Aug 11, 2025
b7a7ef2
fixup! Revert to the use of StringMacroName/CmdMacroName kinds
c42f Aug 11, 2025
d5cd3d7
fixup! Revert to the use of StringMacroName/CmdMacroName kinds
c42f Aug 11, 2025
2ef16a0
fixup! fixup! Revert to the use of StringMacroName/CmdMacroName kinds
c42f Aug 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 15 additions & 22 deletions src/integration/expr.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,8 @@ function node_to_expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt
val isa UInt128 ? Symbol("@uint128_str") :
Symbol("@big_str")
return Expr(:macrocall, GlobalRef(Core, macname), nothing, str)
elseif is_identifier(k)
return lower_identifier_name(val, k)
else
return val
end
Expand Down Expand Up @@ -294,27 +296,18 @@ function node_to_expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt
nodehead, source)
end

function adjust_macro_name!(retexpr::Union{Expr, Symbol}, k::Kind)
if !(retexpr isa Symbol)
function adjust_macro_name!(retexpr::Union{Expr, Symbol})
if retexpr isa Symbol
return lower_identifier_name(retexpr, K"macro_name")
else
retexpr::Expr
# can happen for incomplete or errors
(length(retexpr.args) < 2 || retexpr.head != :(.)) && return retexpr
arg2 = retexpr.args[2]
isa(arg2, QuoteNode) || return retexpr
retexpr.args[2] = QuoteNode(adjust_macro_name!(arg2.value, k))
return retexpr
end
if k == K"macro_name"
if retexpr === Symbol(".")
return Symbol("@__dot__")
else
return Symbol("@$retexpr")
if length(retexpr.args) == 2 && retexpr.head == :(.)
arg2 = retexpr.args[2]
if isa(arg2, QuoteNode) && arg2.value isa Symbol
retexpr.args[2] = QuoteNode(lower_identifier_name(arg2.value, K"macro_name"))
end
end
elseif k == K"macro_name_cmd"
return Symbol("@$(retexpr)_cmd")
else
@assert k == K"macro_name_str"
return Symbol("@$(retexpr)_str")
return retexpr
end
end

Expand All @@ -334,8 +327,8 @@ end
# However, errors can add additional errors tokens which we represent
# as e.g. `Expr(:var, ..., Expr(:error))`.
return retexpr.args[1]
elseif k in KSet"macro_name macro_name_cmd macro_name_str"
return adjust_macro_name!(retexpr.args[1], k)
elseif k == K"macro_name"
return adjust_macro_name!(retexpr.args[1])
elseif k == K"?"
retexpr.head = :if
elseif k == K"op=" && length(args) == 3
Expand All @@ -355,7 +348,7 @@ end
elseif k == K"macrocall"
if length(args) >= 2
a2 = args[2]
if @isexpr(a2, :macrocall) && kind(firstchildhead) == K"macro_name_cmd"
if @isexpr(a2, :macrocall) && kind(firstchildhead) == K"CmdMacroName"
# Fix up for custom cmd macros like foo`x`
args[2] = a2.args[3]
end
Expand Down
10 changes: 7 additions & 3 deletions src/julia/kinds.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,10 @@ register_kinds!(JuliaSyntax, 0, [
"BEGIN_IDENTIFIERS"
"Identifier"
"Placeholder" # Used for empty catch variables, and all-underscore identifiers in lowering
# String and command macro names are modeled as a special kind of
# identifier as they need to be mangled before lookup.
"StringMacroName"
"CmdMacroName"
"END_IDENTIFIERS"

"BEGIN_KEYWORDS"
Expand Down Expand Up @@ -1039,10 +1043,7 @@ register_kinds!(JuliaSyntax, 0, [
"iteration"
"comprehension"
"typed_comprehension"
# Macro names
"macro_name"
"macro_name_cmd"
"macro_name_str"
# Container for a single statement/atom plus any trivia and errors
"wrapper"
"END_SYNTAX_KINDS"
Expand Down Expand Up @@ -1106,6 +1107,9 @@ const _nonunique_kind_names = Set([
K"String"
K"Char"
K"CmdString"

K"StringMacroName"
K"CmdMacroName"
])

"""
Expand Down
32 changes: 31 additions & 1 deletion src/julia/literal_parsing.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ function parse_julia_literal(txtbuf::Vector{UInt8}, head::SyntaxHead, srcrange)
parse_int_literal(val_str)
elseif k in KSet"BinInt OctInt HexInt"
parse_uint_literal(val_str, k)
elseif k == K"Identifier" || k == K"Placeholder"
elseif is_identifier(k)
if has_flags(head, RAW_STRING_FLAG)
io = IOBuffer()
unescape_raw_string(io, txtbuf, first(srcrange), last(srcrange)+1, false)
Expand All @@ -442,3 +442,33 @@ function parse_julia_literal(txtbuf::Vector{UInt8}, head::SyntaxHead, srcrange)
end
end

"""
lower_identifier_name(name, kind)

Lower a Julia identifier `name` of given `kind` to the name used by the Julia
runtime. (In particular, this handles the name mangling of macros.)

This is a lowering (rather than parsing) step, but is needed for `Expr`
conversion and is also used for pretty printing.
"""
function lower_identifier_name(name::AbstractString, k::Kind)
# Replicate eager lowering done by the flisp parser
if k == K"macro_name"
name == "." ? "@__dot__" : "@$name"
elseif k == K"StringMacroName"
"@$(name)_str"
elseif k == K"CmdMacroName"
"@$(name)_cmd"
else
name
end
end

function lower_identifier_name(name::Symbol, k::Kind)
if k == K"Identifier"
name # avoid unnecessary conversion
else
Symbol(lower_identifier_name(string(name), k))
end
end

38 changes: 20 additions & 18 deletions src/julia/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1519,7 +1519,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
macro_atname_range = nothing
# $A.@x ==> (macrocall (. ($ A) (macro_name x)))
maybe_strmac = true
last_identifier_orig_kind = peek_behind(ps).orig_kind
last_identifier_pos = peek_behind_pos(ps)
last_identifier_orig_kind = peek_behind(ps, last_identifier_pos).orig_kind
while true
maybe_strmac_1 = false
t = peek_token(ps)
Expand Down Expand Up @@ -1577,7 +1578,6 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
# f (a) ==> (call f (error-t) a)
processing_macro_name = maybe_parsed_macro_name(
ps, processing_macro_name, mark)
processing_macro_name = false
bump_disallowed_space(ps)
bump(ps, TRIVIA_FLAG)
opts = parse_call_arglist(ps, K")")
Expand Down Expand Up @@ -1714,7 +1714,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
bump(ps, TRIVIA_FLAG)
end
parse_macro_name(ps)
last_identifier_orig_kind = peek_behind(ps).orig_kind
last_identifier_pos = peek_behind_pos(ps)
last_identifier_orig_kind = peek_behind(ps, last_identifier_pos).orig_kind
!is_macrocall && emit(ps, m, K"macro_name")
macro_atname_range = (m, position(ps))
is_macrocall = true
Expand Down Expand Up @@ -1747,7 +1748,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
emit(ps, macro_name_mark, K"macro_name")
misplaced_atsym_mark = (aterror_mark, position(ps))
end
last_identifier_orig_kind = peek_behind(ps).orig_kind
last_identifier_pos = peek_behind_pos(ps)
last_identifier_orig_kind = peek_behind(ps, last_identifier_pos).orig_kind
maybe_strmac_1 = true
emit(ps, mark, K".")
end
Expand Down Expand Up @@ -1784,29 +1786,29 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
origk = last_identifier_orig_kind;
origk == K"Identifier" || is_contextual_keyword(origk) || is_word_operator(origk))
# Custom string and command literals
# x"str" ==> (macrocall (macro_name_str x) (string-r "str"))
# x`str` ==> (macrocall (macro_name_cmd x) (cmdstring-r "str"))
# x"" ==> (macrocall (macro_name_str x) (string-r ""))
# x`` ==> (macrocall (macro_name_cmd x) (cmdstring-r ""))
# x"str" ==> (macrocall @x_str (string-r "str"))
# x`str` ==> (macrocall @x_cmd (cmdstring-r "str"))
# x"" ==> (macrocall @x_str (string-r ""))
# x`` ==> (macrocall @x_cmd (cmdstring-r ""))
# Triple quoted processing for custom strings
# r"""\nx""" ==> (macrocall (macro_name_str r) (string-s-r "x"))
# r"""\n x\n y""" ==> (macrocall (macro_name_str r) (string-s-r "x\n" "y"))
# r"""\n x\\n y""" ==> (macrocall (macro_name_str r) (string-s-r "x\\\n" "y"))
# r"""\nx""" ==> (macrocall @r_str (string-s-r "x"))
# r"""\n x\n y""" ==> (macrocall @r_str (string-s-r "x\n" "y"))
# r"""\n x\\n y""" ==> (macrocall @r_str (string-s-r "x\\\n" "y"))
#
# Use a special token kind for string and cmd macro names so the
# names can be expanded later as necessary.
outk = is_string_delim(k) ? K"macro_name_str" : K"macro_name_cmd"
emit(ps, mark, outk)
name_kind = is_string_delim(k) ? K"StringMacroName" : K"CmdMacroName"
reset_node!(ps, last_identifier_pos, kind=name_kind)
parse_string(ps, true)
t = peek_token(ps)
k = kind(t)
if !preceding_whitespace(t) && is_string_macro_suffix(k)
# Macro suffixes can include keywords and numbers
# x"s"y ==> (macrocall (macro_name_str x) (string-r "s") "y")
# x"s"end ==> (macrocall (macro_name_str x) (string-r "s") "end")
# x"s"in ==> (macrocall (macro_name_str x) (string-r "s") "in")
# x"s"2 ==> (macrocall (macro_name_str x) (string-r "s") 2)
# x"s"10.0 ==> (macrocall (macro_name_str x) (string-r "s") 10.0)
# x"s"y ==> (macrocall @x_str (string-r "s") "y")
# x"s"end ==> (macrocall @x_str (string-r "s") "end")
# x"s"in ==> (macrocall @x_str (string-r "s") "in")
# x"s"2 ==> (macrocall @x_str (string-r "s") 2)
# x"s"10.0 ==> (macrocall @x_str (string-r "s") 10.0)
suffix_kind = (k == K"Identifier" || is_keyword(k) ||
is_word_operator(k)) ? K"String" : k
bump(ps, remap_kind=suffix_kind)
Expand Down
9 changes: 7 additions & 2 deletions src/porcelain/syntax_tree.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ sourcefile(node::AbstractSyntaxNode) = node.source

function leaf_string(ex)
if !is_leaf(ex)
throw(ArgumentError("_value_string should be used for leaf nodes only"))
throw(ArgumentError("leaf_string should be used for leaf nodes only"))
end
k = kind(ex)
value = ex.val
Expand Down Expand Up @@ -243,7 +243,12 @@ function _show_syntax_node_sexpr(io, node::AbstractSyntaxNode, show_kind)
if is_error(node)
print(io, "(", untokenize(head(node)), ")")
else
print(io, leaf_string(node))
str = leaf_string(node)
k = kind(node)
if is_identifier(k) && !show_kind
str = lower_identifier_name(str, k)
end
print(io, str)
if show_kind
print(io, "::", kind(node))
end
Expand Down
1 change: 0 additions & 1 deletion test/expr.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,6 @@

# var""
@test parsestmt("@var\"#\" a") == Expr(:macrocall, Symbol("@#"), LineNumberNode(1), :a)
@test parsestmt("@var\"\\\"\" a") == Expr(:macrocall, Symbol("@\""), LineNumberNode(1), :a)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be kept. This is JuliaLang/julia#58885

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, thanks. My mistake in doing the partial-revert I'll do a closer self-review of the test changes.

@test parsestmt("A.@var\"#\" a") == Expr(:macrocall, Expr(:., :A, QuoteNode(Symbol("@#"))), LineNumberNode(1), :a)

# Square brackets
Expand Down
48 changes: 26 additions & 22 deletions test/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,28 +463,30 @@ tests = [
"S{a,b}" => "(curly S a b)"
"T{y for x = xs; a}" => "(curly T (generator y (iteration (in x xs))) (parameters a))"
# String macros
"x\"str\"" => """(macrocall (macro_name_str x) (string-r "str"))"""
"x`str`" => """(macrocall (macro_name_cmd x) (cmdstring-r "str"))"""
"x\"\"" => """(macrocall (macro_name_str x) (string-r ""))"""
"x``" => """(macrocall (macro_name_cmd x) (cmdstring-r ""))"""
"in\"str\"" => """(macrocall (macro_name_str in) (string-r "str"))"""
"outer\"str\"" => """(macrocall (macro_name_str outer) (string-r "str"))"""
"x\"str\"" => """(macrocall @x_str (string-r "str"))"""
"x`str`" => """(macrocall @x_cmd (cmdstring-r "str"))"""
"x\"\"" => """(macrocall @x_str (string-r ""))"""
"x``" => """(macrocall @x_cmd (cmdstring-r ""))"""
"in\"str\"" => """(macrocall @in_str (string-r "str"))"""
"outer\"str\"" => """(macrocall @outer_str (string-r "str"))"""
"A.x\"str\"" => """(macrocall (. A @x_str) (string-r "str"))"""
"A.x`str`" => """(macrocall (. A @x_cmd) (cmdstring-r "str"))"""
# Triple quoted processing for custom strings
"r\"\"\"\nx\"\"\"" => raw"""(macrocall (macro_name_str r) (string-s-r "x"))"""
"r\"\"\"\n x\n y\"\"\"" => raw"""(macrocall (macro_name_str r) (string-s-r "x\n" "y"))"""
"r\"\"\"\n x\\\n y\"\"\"" => raw"""(macrocall (macro_name_str r) (string-s-r "x\\\n" "y"))"""
"r\"\"\"\nx\"\"\"" => raw"""(macrocall @r_str (string-s-r "x"))"""
"r\"\"\"\n x\n y\"\"\"" => raw"""(macrocall @r_str (string-s-r "x\n" "y"))"""
"r\"\"\"\n x\\\n y\"\"\"" => raw"""(macrocall @r_str (string-s-r "x\\\n" "y"))"""
# Macro suffixes can include keywords and numbers
"x\"s\"y" => """(macrocall (macro_name_str x) (string-r "s") "y")"""
"x\"s\"end" => """(macrocall (macro_name_str x) (string-r "s") "end")"""
"x\"s\"in" => """(macrocall (macro_name_str x) (string-r "s") "in")"""
"x\"s\"2" => """(macrocall (macro_name_str x) (string-r "s") 2)"""
"x\"s\"10.0" => """(macrocall (macro_name_str x) (string-r "s") 10.0)"""
"x\"s\"y" => """(macrocall @x_str (string-r "s") "y")"""
"x\"s\"end" => """(macrocall @x_str (string-r "s") "end")"""
"x\"s\"in" => """(macrocall @x_str (string-r "s") "in")"""
"x\"s\"2" => """(macrocall @x_str (string-r "s") 2)"""
"x\"s\"10.0" => """(macrocall @x_str (string-r "s") 10.0)"""
# Cmd macro suffixes
"x`s`y" => """(macrocall (macro_name_cmd x) (cmdstring-r "s") "y")"""
"x`s`end" => """(macrocall (macro_name_cmd x) (cmdstring-r "s") "end")"""
"x`s`in" => """(macrocall (macro_name_cmd x) (cmdstring-r "s") "in")"""
"x`s`2" => """(macrocall (macro_name_cmd x) (cmdstring-r "s") 2)"""
"x`s`10.0" => """(macrocall (macro_name_cmd x) (cmdstring-r "s") 10.0)"""
"x`s`y" => """(macrocall @x_cmd (cmdstring-r "s") "y")"""
"x`s`end" => """(macrocall @x_cmd (cmdstring-r "s") "end")"""
"x`s`in" => """(macrocall @x_cmd (cmdstring-r "s") "in")"""
"x`s`2" => """(macrocall @x_cmd (cmdstring-r "s") 2)"""
"x`s`10.0" => """(macrocall @x_cmd (cmdstring-r "s") 10.0)"""
],
JuliaSyntax.parse_resword => [
# In normal_context
Expand Down Expand Up @@ -1167,6 +1169,9 @@ parsestmt_with_kind_tests = [
"a >>= b" => "(op= a::Identifier >>::Identifier b::Identifier)"
":+=" => "(quote-: +=::op=)"
":.+=" => "(quote-: (. +=::op=))"
# str/cmd macro name kinds
"x\"str\"" => """(macrocall x::StringMacroName (string-r "str"::String))"""
"x`str`" => """(macrocall x::CmdMacroName (cmdstring-r "str"::CmdString))"""
]

@testset "parser `Kind` remapping" begin
Expand Down Expand Up @@ -1195,13 +1200,12 @@ end
# ɛµ normalizes to εμ
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5()") == "(call \u03B5\u03BC)"
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "@\u025B\u00B5") == "(macrocall (macro_name \u03B5\u03BC))"
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5\"\"") == "(macrocall (macro_name_str \u03B5\u03BC) (string-r \"\"))"
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5``") == "(macrocall (macro_name_cmd \u03B5\u03BC) (cmdstring-r \"\"))"
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5\"\"") == "(macrocall @\u03B5\u03BC_str (string-r \"\"))"
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5``") == "(macrocall @\u03B5\u03BC_cmd (cmdstring-r \"\"))"
# · and · normalize to ⋅
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a \u00B7 b") == "(call-i a \u22C5 b)"
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a \u0387 b") == "(call-i a \u22C5 b)"
# − ('\u2212') normalizes to - ('\u002d')
@test parse_to_sexpr_str(JuliaSyntax.parse_expr, "a \u2212 b") == "(call-i a - b)"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unrelated?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, no idea what happened here, yikes

@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a \u2212= b") == "(op= a - b)"
@test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a .\u2212= b") == "(.op= a - b)"
end
Expand Down
Loading