add syntax highlighting to the REPL input (#59778)

Author: KristofferC

NEWS.md

@@ -97,6 +97,7 @@ Standard library changes
 #### REPL
 
 * The Julia REPL now support bracketed paste on Windows which should significantly speed up pasting large code blocks into the REPL ([#59825])
+* The REPL now provides syntax highlighting for input as you type. See the REPL docs for more info about customization.
 * The display of `AbstractChar`s in the main REPL mode now includes LaTeX input information like what is shown in help mode ([#58181]).
 * Display of repeated frames and cycles in stack traces has been improved by bracketing them in the trace and treating them consistently ([#55841]).
 

contrib/generate_precompile.jl

@@ -41,6 +41,7 @@ precompile(Tuple{typeof(Base.Terminals.enable_bracketed_paste), Base.Terminals.T
 precompile(Tuple{typeof(Base.Terminals.width), Base.Terminals.TTYTerminal})
 precompile(Tuple{typeof(Base.Terminals.height), Base.Terminals.TTYTerminal})
 precompile(Tuple{typeof(Base.write), Base.Terminals.TTYTerminal, Array{UInt8, 1}})
+precompile(Tuple{typeof(Base.isempty), Base.AnnotatedString{String}}
 
 # loading.jl - without these each precompile worker would precompile these because they're hit before pkgimages are loaded
 precompile(Base.__require, (Module, Symbol))

stdlib/REPL/docs/src/index.md

@@ -533,6 +533,183 @@ mmap(file::AbstractString, ::Type{T}, len::Integer) where T<:BitArray in Mmap at
 mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer; grow, shared) where T<:BitArray in Mmap at Mmap/src/Mmap.jl:322
 ```
 
+## Syntax Highlighting
+
+The REPL provides syntax highlighting for input as you type.
+Syntax highlighting is enabled by default but can be disabled in your `~/.julia/config/startup.jl`:
+
+```julia
+atreplinit() do repl
+    repl.options.style_input = false
+end
+```
+
+### Customizing Syntax Highlighting Colors
+
+The default syntax highlighting theme is quite conservative but can be customized using a TOML file `faces.toml` (https://julialang.github.io/StyledStrings.jl/dev/#stdlib-styledstrings-face-toml) in `.julia/config` (or by explicitly loading the faces from a face toml file).
+
+
+<details>
+<summary>Example: Monokai color theme (click to expand)</summary>
+
+```toml
+# Monokai color theme for Julia syntax highlighting
+
+[julia_macro]
+foreground = "#A6E22E"
+
+[julia_symbol]
+foreground = "#AE81FF"
+
+[julia_singleton_identifier]
+inherit = "julia_symbol"
+
+[julia_type]
+foreground = "#66D9EF"
+
+[julia_typedec]
+foreground = "#66D9EF"
+weight = "bold"
+
+[julia_comment]
+foreground = "#75715E"
+italic = true
+
+[julia_string]
+foreground = "#E6DB74"
+
+[julia_regex]
+inherit = "julia_string"
+
+[julia_backslash_literal]
+foreground = "#FD971F"
+inherit = "julia_string"
+
+[julia_string_delim]
+foreground = "#E6DB74"
+weight = "bold"
+
+[julia_cmdstring]
+inherit = "julia_string"
+
+[julia_char]
+inherit = "julia_string"
+
+[julia_char_delim]
+inherit = "julia_string_delim"
+
+[julia_number]
+foreground = "#AE81FF"
+
+[julia_bool]
+foreground = "#AE81FF"
+weight = "bold"
+
+[julia_funcall]
+foreground = "#A6E22E"
+
+[julia_broadcast]
+foreground = "#F92672"
+weight = "bold"
+
+[julia_builtin]
+foreground = "#66D9EF"
+weight = "bold"
+
+[julia_operator]
+foreground = "#F92672"
+
+[julia_comparator]
+inherit = "julia_operator"
+
+[julia_assignment]
+foreground = "#F92672"
+weight = "bold"
+
+[julia_keyword]
+foreground = "#F92672"
+weight = "bold"
+
+[julia_parentheses]
+foreground = "#F8F8F2"
+
+[julia_unpaired_parentheses]
+background = "#F92672"
+foreground = "#F8F8F0"
+weight = "bold"
+
+[julia_error]
+background = "#F92672"
+foreground = "#F8F8F0"
+
+[julia_rainbow_paren_1]
+foreground = "#A6E22E"
+inherit = "julia_parentheses"
+
+[julia_rainbow_paren_2]
+foreground = "#66D9EF"
+inherit = "julia_parentheses"
+
+[julia_rainbow_paren_3]
+foreground = "#FD971F"
+inherit = "julia_parentheses"
+
+[julia_rainbow_paren_4]
+inherit = "julia_rainbow_paren_1"
+
+[julia_rainbow_paren_5]
+inherit = "julia_rainbow_paren_2"
+
+[julia_rainbow_paren_6]
+inherit = "julia_rainbow_paren_3"
+
+# Rainbow brackets
+[julia_rainbow_bracket_1]
+foreground = "#AE81FF"
+inherit = "julia_parentheses"
+
+[julia_rainbow_bracket_2]
+foreground = "#E6DB74"
+inherit = "julia_parentheses"
+
+[julia_rainbow_bracket_3]
+inherit = "julia_rainbow_bracket_1"
+
+[julia_rainbow_bracket_4]
+inherit = "julia_rainbow_bracket_2"
+
+[julia_rainbow_bracket_5]
+inherit = "julia_rainbow_bracket_1"
+
+[julia_rainbow_bracket_6]
+inherit = "julia_rainbow_bracket_2"
+
+# Rainbow curlies
+[julia_rainbow_curly_1]
+foreground = "#F92672"
+inherit = "julia_parentheses"
+
+[julia_rainbow_curly_2]
+foreground = "#A6E22E"
+inherit = "julia_parentheses"
+
+[julia_rainbow_curly_3]
+inherit = "julia_rainbow_curly_1"
+
+[julia_rainbow_curly_4]
+inherit = "julia_rainbow_curly_2"
+
+[julia_rainbow_curly_5]
+inherit = "julia_rainbow_curly_1"
+
+[julia_rainbow_curly_6]
+inherit = "julia_rainbow_curly_2"
+```
+
+</details>
+
+For a complete list of customizable faces, see the [JuliaSyntaxHighlighting package documentation](https://julialang.github.io/JuliaSyntaxHighlighting.jl/dev/).
+
 ## Customizing Colors
 
 The colors used by Julia and the REPL can be customized, as well. To change the

stdlib/REPL/src/LineEdit.jl

@@ -4,10 +4,13 @@ module LineEdit
 
 import ..REPL
 using ..REPL: AbstractREPL, Options
+using ..REPL.StylingPasses: StylingPass, SyntaxHighlightPass, RegionHighlightPass, EnclosingParenHighlightPass, StylingContext, apply_styling_passes, merge_annotations
 
 using ..Terminals
 import ..Terminals: raw!, width, height, clear_line, beep
 
+using StyledStrings
+
 import Base: ensureroom, show, AnyDict, position
 using Base: something
 
@@ -58,6 +61,7 @@ mutable struct Prompt <: TextInterface
     on_done::Function
     hist::HistoryProvider  # TODO?: rename this `hp` (consistency with other TextInterfaces), or is the type-assert useful for mode(s)?
     sticky::Bool
+    styling_passes::Vector{StylingPass}  # Styling passes to apply to input
 end
 
 show(io::IO, x::Prompt) = show(io, string("Prompt(\"", prompt_string(x.prompt), "\",...)"))
@@ -565,6 +569,8 @@ function maybe_show_hint(s::PromptState)
     return nothing
 end
 
+max_highlight_size::Int = 10000 # bytes
+
 function refresh_multi_line(s::PromptState; kw...)
     if s.refresh_wait !== nothing
         close(s.refresh_wait)
@@ -611,6 +617,42 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
         reader isa Base.TTY && !Base.ispty(reader)::Bool
     else false end
 
+    # Get the styling passes from the prompt
+    prompt_obj = nothing
+    if prompt isa PromptState
+        prompt_obj = prompt.p
+    elseif prompt isa PrefixSearchState || prompt isa SearchState
+        if isdefined(prompt, :parent) && prompt.parent isa Prompt
+            prompt_obj = prompt.parent
+        end
+    end
+
+    styled_buffer = AnnotatedString("")
+    if buf.size > 0 && buf.size <= max_highlight_size
+        full_input = String(buf.data[1:buf.size])
+        if !isempty(full_input)
+            passes = StylingPass[]
+            context = StylingContext(buf_pos, regstart, regstop)
+
+            # Add prompt-specific styling passes if the prompt has them and styling is enabled
+            enable_style_input = prompt_obj === nothing ? false :
+                (isdefined(prompt_obj, :repl) && prompt_obj.repl !== nothing ?
+                    prompt_obj.repl.options.style_input : false)
+
+            if enable_style_input && prompt_obj !== nothing
+                append!(passes, prompt_obj.styling_passes)
+            end
+
+            if region_active
+                push!(passes, RegionHighlightPass())
+            end
+
+            if !isempty(passes)
+                styled_buffer = apply_styling_passes(full_input, passes, context)
+            end
+        end
+    end
+
     # Now go through the buffer line by line
     seek(buf, 0)
     moreinput = true # add a blank line if there is a trailing newline on the last line
@@ -636,12 +678,26 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
         llength = textwidth(line)
         slength = sizeof(line)
         cur_row += 1
-        # lwrite: what will be written to termbuf
-        lwrite = region_active ? highlight_region(line, regstart, regstop, written, slength) :
-                                 line
+
+        # Extract the portion of styled_buffer corresponding to this line.
+        if !isempty(styled_buffer)
+            # Calculate byte positions for this line in the buffer
+            line_start_byte = written + 1
+            line_end_byte = written + slength
+
+            # Convert to valid character indices (handles UTF-8 boundaries)
+            start_idx = thisind(styled_buffer, line_start_byte)
+            end_idx = thisind(styled_buffer, line_end_byte)
+
+            lwrite = @view styled_buffer[start_idx:end_idx]
+        else
+            lwrite = line
+        end
+
         written += slength
         cmove_col(termbuf, lindent + 1)
-        write(termbuf, lwrite)
+
+        write(IOContext(termbuf, :color => hascolor(terminal)), lwrite)
         # We expect to be line after the last valid output line (due to
         # the '\n' at the end of the previous line)
         if curs_row == -1
@@ -692,18 +748,6 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
     return InputAreaState(cur_row, curs_row)
 end
 
-function highlight_region(lwrite::Union{String,SubString{String}}, regstart::Int, regstop::Int, written::Int, slength::Int)
-    if written <= regstop <= written+slength
-        i = thisind(lwrite, regstop-written)
-        lwrite = lwrite[1:i] * Base.disable_text_style[:reverse] * lwrite[nextind(lwrite, i):end]
-    end
-    if written <= regstart <= written+slength
-        i = thisind(lwrite, regstart-written)
-        lwrite = lwrite[1:i] * Base.text_colors[:reverse] * lwrite[nextind(lwrite, i):end]
-    end
-    return lwrite
-end
-
 function refresh_multi_line(terminal::UnixTerminal, args...; kwargs...)
     outbuf = IOBuffer()
     termbuf = TerminalBuffer(outbuf)
@@ -999,7 +1043,9 @@ function edit_insert(s::PromptState, c::StringLike)
         offset += position(buf) - beginofline(buf) # size of current line
         spinner = '\0'
         delayup = !eof(buf) || old_wait
-        if offset + textwidth(str) <= w && !(after == 0 && delayup)
+        # Disable fast path when syntax highlighting is enabled
+        use_fast_path = offset + textwidth(str) <= w && !(after == 0 && delayup) && !options(s).style_input
+        if use_fast_path
             # Avoid full update when appending characters to the end
             # and an update of curs_row isn't necessary (conservatively estimated)
             write(termbuf, str)
@@ -2226,6 +2272,13 @@ function replace_line(s::PrefixSearchState, l::Union{String,SubString{String}})
     nothing
 end
 
+function write_prompt(terminal, s::SearchState, color::Bool)
+    failed = s.failed ? "failed " : ""
+    promptstr = s.backward ? "($(failed)reverse-i-search)`" : "($(failed)forward-i-search)`"
+    write(terminal, promptstr)
+    return textwidth(promptstr)
+end
+
 function refresh_multi_line(termbuf::TerminalBuffer, s::SearchState)
     buf = IOBuffer()
     unsafe_write(buf, pointer(s.query_buffer.data), s.query_buffer.ptr-1)
@@ -2236,9 +2289,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, s::SearchState)
     write(buf, read(s.response_buffer, String))
     buf.ptr = offset + ptr - 1
     s.response_buffer.ptr = ptr
-    failed = s.failed ? "failed " : ""
-    ias = refresh_multi_line(termbuf, s.terminal, buf, s.ias,
-                             s.backward ? "($(failed)reverse-i-search)`" : "($(failed)forward-i-search)`")
+    ias = refresh_multi_line(termbuf, s.terminal, buf, s.ias, s)
     s.ias = ias
     return ias
 end
@@ -2823,10 +2874,11 @@ function Prompt(prompt
     on_enter = default_enter_cb,
     on_done = ()->nothing,
     hist = EmptyHistoryProvider(),
-    sticky = false)
+    sticky = false,
+    styling_passes = StylingPass[])
 
     return Prompt(prompt, prompt_prefix, prompt_suffix, output_prefix, output_prefix_prefix, output_prefix_suffix,
-                   keymap_dict, repl, complete, on_enter, on_done, hist, sticky)
+                   keymap_dict, repl, complete, on_enter, on_done, hist, sticky, styling_passes)
 end
 
 run_interface(::Prompt) = nothing

stdlib/REPL/src/REPL.jl

@@ -66,6 +66,8 @@ using Base.Terminals
 abstract type AbstractREPL end
 
 include("options.jl")
+include("StylingPasses.jl")
+using .StylingPasses
 
 include("LineEdit.jl")
 using .LineEdit
@@ -1329,7 +1331,11 @@ function setup_interface(
             (repl.envcolors ? Base.input_color : repl.input_color) : "",
         repl = repl,
         complete = replc,
-        on_enter = return_callback)
+        on_enter = return_callback,
+        styling_passes = StylingPasses.StylingPass[
+            StylingPasses.SyntaxHighlightPass(),
+            StylingPasses.EnclosingParenHighlightPass()
+        ])
 
     # Setup help mode
     help_mode = Prompt(contextual_prompt(repl, HELP_PROMPT),