Merge pull request #468 from JuliaLang/fe/backport-release-0.4

Backport release 0.4

Author: fredrikekre

.github/workflows/CI.yml

@@ -23,7 +23,9 @@ jobs:
           - '1.6'
           - '1.7'
           - '1.8'
+          - '1.9'
           - '1'
+          - 'pre'
           - 'nightly'
         os:
           - ubuntu-latest
@@ -37,6 +39,7 @@ jobs:
           # - 1.0
           # - 1.6
           # - 1
+          # - pre
           # - nightly
           # but remove some configurations from the build matrix to reduce CI time.
           # See https://github.com/marketplace/actions/setup-julia-environment
@@ -47,6 +50,7 @@ jobs:
           - {os: 'macOS-latest', version: '1.5'}
           - {os: 'macOS-latest', version: '1.7'}
           - {os: 'macOS-latest', version: '1.8'}
+          - {os: 'macOS-latest', version: '1.9'}
           # MacOS not available on x86
           - {os: 'macOS-latest', arch: 'x86'}
           - {os: 'windows-latest', version: '1.1'}
@@ -56,16 +60,18 @@ jobs:
           - {os: 'windows-latest', version: '1.5'}
           - {os: 'windows-latest', version: '1.7'}
           - {os: 'windows-latest', version: '1.8'}
+          - {os: 'windows-latest', version: '1.9'}
           - {os: 'ubuntu-latest', version: '1.1', arch: 'x86'}
           - {os: 'ubuntu-latest', version: '1.2', arch: 'x86'}
           - {os: 'ubuntu-latest', version: '1.3', arch: 'x86'}
           - {os: 'ubuntu-latest', version: '1.4', arch: 'x86'}
           - {os: 'ubuntu-latest', version: '1.5', arch: 'x86'}
           - {os: 'ubuntu-latest', version: '1.7', arch: 'x86'}
           - {os: 'ubuntu-latest', version: '1.8', arch: 'x86'}
+          - {os: 'ubuntu-latest', version: '1.9', arch: 'x86'}
     steps:
       - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
@@ -92,7 +98,7 @@ jobs:
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: 1.6
           arch: x64

Project.toml

@@ -1,7 +1,7 @@
 name = "JuliaSyntax"
 uuid = "70703baa-626e-46a2-a12c-08ffd08c73b4"
 authors = ["Claire Foster <[email protected]> and contributors"]
-version = "0.4.8"
+version = "0.4.9"
 
 [compat]
 julia = "1.0"

src/parse_stream.jl

@@ -314,7 +314,8 @@ function ParseStream(text::AbstractString, index::Integer=1; version=VERSION)
 end
 
 # IO-based cases
-function ParseStream(io::IOBuffer; version=VERSION)
+# TODO: switch ParseStream to use a Memory internally on newer versions of Julia
+VERSION < v"1.11.0-DEV.753" && function ParseStream(io::IOBuffer; version=VERSION)
     ParseStream(io.data, io, position(io)+1, version)
 end
 function ParseStream(io::Base.GenericIOBuffer; version=VERSION)
@@ -475,7 +476,7 @@ end
 end
 
 """
-    peek(stream [, n=1]; skip_newlines=false)
+    peek(stream::ParseStream [, n=1]; skip_newlines=false)
 
 Look ahead in the stream `n` tokens, returning the token kind. Comments and
 non-newline whitespace are skipped automatically. Whitespace containing a

src/parser.jl

@@ -1211,6 +1211,7 @@ function parse_unary(ps::ParseState)
                 # -2*x    ==>  (call-i -2 * x)
                 # +0xff   ==>  0xff
                 bump_glue(ps, kind(t2), EMPTY_FLAGS)
+                parse_factor_with_initial_ex(ps, mark)
             end
             return
         end

src/tokenize.jl

@@ -733,9 +733,10 @@ function lex_whitespace(l::Lexer, c)
         if c == '\n'
             k = K"NewlineWs"
         end
-        pc = peekchar(l)
+        pc, ppc = dpeekchar(l)
         # stop on non whitespace and limit to a single newline in a token
-        if !iswhitespace(pc) || (k == K"NewlineWs" && pc == '\n')
+        if !iswhitespace(pc) ||
+                (k == K"NewlineWs" && (pc == '\n' || (pc == '\r' && ppc == '\n')))
             break
         end
         c = readchar(l)
@@ -747,8 +748,8 @@ function lex_comment(l::Lexer)
     if peekchar(l) != '='
         valid = true
         while true
-            pc = peekchar(l)
-            if pc == '\n' || pc == EOF_CHAR
+            pc, ppc = dpeekchar(l)
+            if pc == '\n' || (pc == '\r' && ppc == '\n') || pc == EOF_CHAR
                 return emit(l, valid ? K"Comment" : K"ErrorInvalidUTF8")
             end
             valid &= isvalid(pc)

test/parse_packages.jl

@@ -9,10 +9,10 @@ end
 end
 
 base_path = let
-    p = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "base") 
+    p = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "base")
     if !isdir(p)
         # For julia 1.9 images.
-        p = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "src", "base") 
+        p = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "src", "base")
         if !isdir(p)
             error("source for Julia base not found")
         end
@@ -40,8 +40,21 @@ base_tests_path = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "test")
         end
 
         if endswith(f, "core.jl")
-            # Loose comparison due to `for f() = 1:3` syntax
-            return exprs_roughly_equal
+            # The test
+            # @test Union{Tuple{T}, Tuple{T,Int}} where {T} === widen_diagonal(Union{Tuple{T}, Tuple{T,Int}} where {T})
+            # depends on a JuliaSyntax bugfix and parses differently (wrong) using
+            # flisp. This was added in julia#52228 and backported in julia#52045
+            if v"1.10.0-rc1.39" <= VERSION
+                return nothing
+            else
+                # Loose comparison due to `for f() = 1:3` syntax
+                return exprs_roughly_equal
+            end
+        end
+
+        # subtype.jl also depends on the where precedence JuliaSyntax bugfix as of julia#53034
+        if endswith(f, "subtype.jl") && v"1.11.0-DEV.1382" <= VERSION
+            return nothing
         end
 
         return exprs_equal_no_linenum

test/parser.jl

@@ -212,6 +212,7 @@ tests = [
         "-0b10010" => "(call-pre - 0x12)"
         "-0o22"    => "(call-pre - 0x12)"
         "-0x12"    => "(call-pre - 0x12)"
+        "-1::T"    => "(::-i -1 T)"
         # Standalone dotted operators are parsed as (|.| op)
         ".+"   =>  "(. +)"
         ".+\n" =>  "(. +)"

test/tokenize.jl

@@ -161,6 +161,16 @@ end
     @test untokenize(tok(str), str)==">>"
 end
 
+@testset "tokenize newlines" begin
+    n = "\n"
+    rn = "\r\n"
+    nl = K"NewlineWs"
+    for i in 0:5
+        j = 5 - i
+        @test toks(n^i * rn^j) == vcat(fill(n  => nl, i), fill(rn => nl, j))
+        @test toks(rn^i * n^j) == vcat(fill(rn => nl, i), fill(n  => nl, j))
+    end
+end
 
 @testset "test added operators" begin
     @test tok("1+=2",  2).kind == K"+="
@@ -219,6 +229,8 @@ end
     @test toks("#=   #=   =#") == ["#=   #=   =#"=>K"ErrorEofMultiComment"]
     @test toks("#=#==#=#") == ["#=#==#=#"=>K"Comment"]
     @test toks("#=#==#=")  == ["#=#==#="=>K"ErrorEofMultiComment"]
+    # comment terminated by \r\n
+    @test toks("#\r\n") == ["#" => K"Comment", "\r\n" => K"NewlineWs"]
 end