Skip to content

Commit f3e6efd

Browse files
tecosaurtecosaur
committed
Use the parse tree
1 parent 8db89bd commit f3e6efd

File tree

1 file changed

+152
-96
lines changed

1 file changed

+152
-96
lines changed

‎src/JuliaSyntaxHighlighting.jl

Lines changed: 152 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
module JuliaSyntaxHighlighting
22

33
import Base: JuliaSyntax, AnnotatedString, annotate!
4-
import Base.JuliaSyntax: var"@K_str", Kind, Tokenize, tokenize
5-
import .Tokenize: kind, untokenize
4+
import Base.JuliaSyntax: var"@K_str", Kind, GreenNode, parseall, kind, flags
65
using StyledStrings: Face, addface!
76

87
public highlight, highlight!
@@ -59,96 +58,148 @@ const HIGHLIGHT_FACES = [
5958

6059
__init__() = foreach(addface!, HIGHLIGHT_FACES)
6160

62-
function _hl_annotations(content::AbstractString, tokens)
63-
highlighted = Vector{Tuple{UnitRange{Int}, Pair{Symbol, Any}}}()
64-
lastk, last2k = K"None", K"None"
65-
lastf, last2f = :none, :none
66-
function paren_type(k)
67-
if k == K"("; 1, :paren
68-
elseif k == K")"; -1, :paren
69-
elseif k == K"["; 1, :bracket
70-
elseif k == K"]"; -1, :bracket
71-
elseif k == K"{"; 1, :curly
72-
elseif k == K"}"; -1, :curly
73-
else 0, :none
74-
end
61+
function paren_type(k::Kind)
62+
if k == K"("; 1, :paren
63+
elseif k == K")"; -1, :paren
64+
elseif k == K"["; 1, :bracket
65+
elseif k == K"]"; -1, :bracket
66+
elseif k == K"{"; 1, :curly
67+
elseif k == K"}"; -1, :curly
68+
else 0, :none
7569
end
76-
depthcounters = (paren = Ref(0), bracket = Ref(0), curly = Ref(0))
77-
for (; head::JuliaSyntax.SyntaxHead, range::UnitRange{UInt32}) in tokens
78-
range = first(range):thisind(content, last(range))
79-
kind = head.kind
80-
face = if kind == K"Identifier"
81-
if lastk == K":" && !JuliaSyntax.is_number(last2k) &&
82-
last2k ∉ (K"Identifier", K")", K"]", K"end", K"'")
83-
highlighted[end] = (highlighted[end][1], :face => :julia_symbol)
84-
:julia_symbol
85-
elseif lastk == K"::"
86-
:julia_type
87-
elseif lastk ∈ (K".", K"{") && last2f == :julia_type
88-
:julia_type
89-
elseif view(content, range) in SINGLETON_IDENTIFIERS
90-
:julia_singleton_identifier
91-
elseif view(content, range) == "NaN"
92-
:julia_number
93-
else
94-
:julia_identifier
95-
end
96-
elseif kind == K"@"; :julia_macro
97-
elseif kind == K"MacroName"; :julia_macro
98-
elseif kind == K"StringMacroName"; :julia_macro
99-
elseif kind == K"CmdMacroName"; :julia_macro
100-
elseif kind == K"::"; :julia_type
101-
elseif kind == K"Comment"; :julia_comment
102-
elseif kind == K"String"; :julia_string
103-
elseif JuliaSyntax.is_string_delim(kind); :julia_string_delim
104-
elseif kind == K"CmdString"; :julia_cmdstring
105-
elseif kind == K"`" || kind == K"```"; :julia_cmdstring
106-
elseif kind == K"Char"
107-
lastk == K"'" &&
108-
(highlighted[end] = (highlighted[end][1], :face => :julia_char_delim))
109-
:julia_char
110-
elseif kind == K"'" && lastk == K"Char"; :julia_char_delim
111-
elseif kind == K"true" || kind == K"false"; :julia_bool
112-
elseif JuliaSyntax.is_number(kind); :julia_number
113-
elseif JuliaSyntax.is_prec_assignment(kind); :julia_assignment
114-
elseif JuliaSyntax.is_prec_comparison(kind); :julia_comparator
115-
elseif JuliaSyntax.is_operator(kind); :julia_operator
116-
elseif JuliaSyntax.is_keyword(kind); :julia_keyword
117-
elseif JuliaSyntax.is_error(kind); :julia_error
118-
elseif ((depthchange, ptype) = paren_type(kind)) |> last != :none
119-
if kind == K"(" && lastk == K"Identifier"
120-
highlighted[end] = (highlighted[end][1], :face => :julia_funcall)
121-
end
122-
depthref = getfield(depthcounters, ptype)[]
123-
pdepth = if depthchange > 0
124-
getfield(depthcounters, ptype)[] += depthchange
125-
else
126-
depth0 = getfield(depthcounters, ptype)[]
127-
getfield(depthcounters, ptype)[] += depthchange
128-
depth0
129-
end
130-
if pdepth <= 0 && UNMATCHED_DELIMITERS_ENABLED[]
131-
:julia_unpaired_parenthetical
132-
elseif !RAINBOW_DELIMITERS_ENABLED[]
133-
:julia_parenthetical
134-
else
135-
displaydepth = mod1(pdepth, MAX_PAREN_HIGHLIGHT_DEPTH)
136-
Symbol("julia_rainbow_$(ptype)_$(displaydepth)")
137-
end
70+
end
71+
72+
struct ParenDepthCounter
73+
paren::Ref{UInt}
74+
bracket::Ref{UInt}
75+
curly::Ref{UInt}
76+
end
77+
78+
ParenDepthCounter() =
79+
ParenDepthCounter(Ref(zero(UInt)), Ref(zero(UInt)), Ref(zero(UInt)))
80+
81+
struct GreenLineage
82+
node::GreenNode
83+
parent::Union{Nothing, GreenLineage}
84+
end
85+
86+
struct HighlightContext{S <: AbstractString}
87+
content::S
88+
offset::Int
89+
lnode::GreenNode
90+
llnode::GreenNode
91+
pdepths::ParenDepthCounter
92+
end
93+
94+
function _hl_annotations(content::AbstractString, ast::GreenNode)
95+
highlights = Vector{Tuple{UnitRange{Int}, Pair{Symbol, Any}}}()
96+
ctx = HighlightContext(content, 0, ast, ast, ParenDepthCounter())
97+
_hl_annotations!(highlights, GreenLineage(ast, nothing), ctx)
98+
highlights
99+
end
100+
101+
function _hl_annotations!(highlights::Vector{Tuple{UnitRange{Int}, Pair{Symbol, Any}}},
102+
lineage::GreenLineage, ctx::HighlightContext)
103+
(; node, parent) = lineage
104+
(; content, offset, lnode, llnode, pdepths) = ctx
105+
region = firstindex(content)+offset:node.span+offset
106+
nkind = node.head.kind
107+
pnode = if !isnothing(parent) parent.node end
108+
pkind = if !isnothing(parent) kind(parent.node) end
109+
face = if nkind == K"Identifier"
110+
if pkind == K"::" && JuliaSyntax.is_trivia(pnode)
111+
:julia_type
112+
elseif pkind == K"curly" && kind(lnode) == K"curly" && !isnothing(parent.parent) && kind(parent.parent.node) == K"call"
113+
:julia_identifier
114+
elseif pkind == K"curly"
115+
:julia_type
116+
elseif pkind == K"braces" && lnode != pnode
117+
:julia_type
118+
elseif kind(lnode) == K"::" && JuliaSyntax.is_trivia(lnode)
119+
:julia_type
120+
elseif kind(lnode) == K":" && !JuliaSyntax.is_number(llnode) &&
121+
kind(llnode) ∉ (K"Identifier", K")", K"]", K"end", K"'")
122+
highlights[end] = (highlights[end][1], :face => :julia_symbol)
123+
:julia_symbol
124+
elseif view(content, region) in SINGLETON_IDENTIFIERS
125+
:julia_singleton_identifier
126+
elseif view(content, region) == "NaN"
127+
:julia_number
128+
else
129+
:julia_identifier
130+
end
131+
elseif nkind == K"@"; :julia_macro
132+
elseif nkind == K"MacroName"; :julia_macro
133+
elseif nkind == K"StringMacroName"; :julia_macro
134+
elseif nkind == K"CmdMacroName"; :julia_macro
135+
elseif nkind == K"::"; :julia_type
136+
elseif nkind == K"Comment"; :julia_comment
137+
elseif nkind == K"String"; :julia_string
138+
elseif JuliaSyntax.is_string_delim(node); :julia_string_delim
139+
elseif nkind == K"CmdString"; :julia_cmdstring
140+
elseif nkind == K"`" || nkind == K"```"; :julia_cmdstring
141+
elseif nkind == K"Char"
142+
kind(lnode) == K"'" && !isempty(highlights) &&
143+
(highlights[end] = (highlights[end][1], :face => :julia_char_delim))
144+
:julia_char
145+
elseif nkind == K"'" && kind(lnode) == K"Char"; :julia_char_delim
146+
elseif nkind == K"true" || nkind == K"false"; :julia_bool
147+
elseif JuliaSyntax.is_number(nkind); :julia_number
148+
elseif JuliaSyntax.is_prec_assignment(nkind) && JuliaSyntax.is_trivia(node);
149+
:julia_assignment
150+
elseif JuliaSyntax.is_word_operator(nkind) && JuliaSyntax.is_trivia(node);
151+
:julia_assignment
152+
elseif nkind == K";" && pkind == K"parameters" && pnode == lnode
153+
:julia_assignment
154+
elseif JuliaSyntax.is_prec_comparison(nkind); :julia_comparator
155+
elseif JuliaSyntax.is_operator(nkind) && !JuliaSyntax.is_prec_assignment(nkind) &&
156+
!JuliaSyntax.is_word_operator(nkind) && nkind != K"." &&
157+
(JuliaSyntax.is_trivia(node) || iszero(flags(node)));
158+
:julia_operator
159+
elseif JuliaSyntax.is_keyword(nkind) && JuliaSyntax.is_trivia(node); :julia_keyword
160+
elseif JuliaSyntax.is_error(nkind); :julia_error
161+
elseif ((depthchange, ptype) = paren_type(nkind)) |> last != :none
162+
if nkind == K"(" && !isempty(highlights) && kind(lnode) == K"Identifier" && last(last(highlights[end])) == :julia_identifier
163+
highlights[end] = (highlights[end][1], :face => :julia_funcall)
164+
end
165+
depthref = getfield(pdepths, ptype)[]
166+
pdepth = if depthchange > 0
167+
getfield(pdepths, ptype)[] += depthchange
168+
else
169+
depth0 = getfield(pdepths, ptype)[]
170+
getfield(pdepths, ptype)[] += depthchange
171+
depth0
138172
end
139-
isnothing(face) || push!(highlighted, (range, :face => face))
140-
last2k, lastk = lastk, kind
141-
last2f, lastf = lastf, face
173+
if pdepth <= 0 && UNMATCHED_DELIMITERS_ENABLED[]
174+
:julia_unpaired_parenthetical
175+
elseif !RAINBOW_DELIMITERS_ENABLED[]
176+
:julia_parenthetical
177+
else
178+
displaydepth = mod1(pdepth, MAX_PAREN_HIGHLIGHT_DEPTH)
179+
Symbol("julia_rainbow_$(ptype)_$(displaydepth)")
180+
end
181+
end
182+
!isnothing(face) &&
183+
push!(highlights, (region, :face => face))
184+
isempty(node.args) && return
185+
llnode, lnode = node, node
186+
for child in node.args
187+
cctx = HighlightContext(content, offset, lnode, llnode, pdepths)
188+
_hl_annotations!(highlights, GreenLineage(child, lineage), cctx)
189+
llnode, lnode = lnode, child
190+
offset += child.span
142191
end
143-
highlighted
144192
end
145193

146194
"""
147-
highlight(content::Union{AbstractString, IOBuffer, IOContext{IOBuffer}})
195+
highlight(content::Union{AbstractString, IO},
196+
ast::JuliaSyntax.GreenNode = <parsed content>) -> AnnotatedString{String}
148197
149198
Apply syntax highlighting to `content` using `JuliaSyntax`.
150199
151-
Returns an `AnnotatedString{String}`.
200+
By default, `JuliaSyntax.parseall` is used to generate to `ast` with the
201+
`ignore_errors` keyword argument set to `true`. Alternatively, one may provide a
202+
pre-generated `ast`.
152203
153204
# Examples
154205
@@ -166,24 +217,29 @@ julia> JuliaSyntaxHighlighting.highlight("sum(1:8)") |> Base.annotations
166217
(8:8, :face => :julia_rainbow_paren_1)
167218
```
168219
"""
220+
function highlight end
221+
169222
highlight(str::AbstractString) =
170-
AnnotatedString(str, _hl_annotations(str, tokenize(str)))
171-
172-
function highlight(buf::IOBuffer)
173-
pos = position(buf)
174-
eof(buf) && seekstart(buf)
175-
str = read(buf, String)
176-
seek(buf, pos)
177-
highlight(str)
178-
end
223+
highlight(str, parseall(GreenNode, str, ignore_errors=true))
224+
225+
highlight(io::IO) = highlight(read(io, String))
179226

180-
highlight(buf::IOContext{IOBuffer}) = highlight(buf.io)
227+
highlight(io::IO, ast::GreenNode) =
228+
highlight(read(io, String), ast)
229+
230+
highlight(str::AbstractString, ast::GreenNode) =
231+
AnnotatedString(str, _hl_annotations(str, ast))
181232

182233
"""
183-
highlight!(content::Union{AnnotatedString, SubString{AnnotatedString}})
234+
highlight!(content::Union{AnnotatedString, SubString{AnnotatedString}},
235+
ast::JuliaSyntax.GreenNode = <parsed content>)
184236
185237
Modify `content` by applying syntax highlighting using `JuliaSyntax`.
186238
239+
By default, `JuliaSyntax.parseall` is used to generate to `ast` with the
240+
`ignore_errors` keyword argument set to `true`. Alternatively, one may provide a
241+
pre-generated `ast`.
242+
187243
# Examples
188244
189245
```jldoctest
@@ -204,15 +260,15 @@ julia> Base.annotations(str)
204260
```
205261
"""
206262
function highlight!(str::AnnotatedString)
207-
for (range, annot) in _hl_annotations(str.string, tokenize(str.string))
263+
for (range, annot) in _hl_annotations(str.string, parseall(GreenNode, str.string, ignore_errors=true))
208264
annotate!(str, range, annot)
209265
end
210266
str
211267
end
212268

213269
function highlight!(str::SubString{AnnotatedString{S}}) where {S}
214270
plainstr = SubString{S}(str.string.string, str.offset, str.ncodeunits, Val(:noshift))
215-
for (range, annot) in _hl_annotations(plainstr, tokenize(plainstr))
271+
for (range, annot) in _hl_annotations(plainstr, parseall(GreenNode, plainstr, ignore_errors=true))
216272
annotate!(str, range, annot)
217273
end
218274
str

0 commit comments

Comments
 (0)