Merge branch 'main' into sp/ternary-unexpected-kw

Author: pfitzseb

src/parser.jl

@@ -201,25 +201,6 @@ function bump_semicolon_trivia(ps)
     end
 end
 
-# Like @assert, but always enabled and calls internal_error()
-macro check(ex, msgs...)
-    msg = isempty(msgs) ? ex : msgs[1]
-    if isa(msg, AbstractString)
-        msg = msg
-    elseif !isempty(msgs) && (isa(msg, Expr) || isa(msg, Symbol))
-        msg = :(string($(esc(msg))))
-    else
-        msg = string(msg)
-    end
-    return :($(esc(ex)) ? nothing : internal_error($msg))
-end
-
-# Parser internal error, used as an assertion failure for cases we expect can't
-# happen.
-@noinline function internal_error(strs...)
-    error("Internal error: ", strs...)
-end
-
 #-------------------------------------------------------------------------------
 # Parsing-specific predicates on tokens/kinds
 #

src/utils.jl

@@ -1,3 +1,21 @@
+# Internal error, used as assertion failure for cases we expect can't happen.
+@noinline function internal_error(strs...)
+    error("Internal error: ", strs...)
+end
+
+# Like @assert, but always enabled and calls internal_error()
+macro check(ex, msgs...)
+    msg = isempty(msgs) ? ex : msgs[1]
+    if isa(msg, AbstractString)
+        msg = msg
+    elseif !isempty(msgs) && (isa(msg, Expr) || isa(msg, Symbol))
+        msg = :(string($(esc(msg))))
+    else
+        msg = string(msg)
+    end
+    return :($(esc(ex)) ? nothing : internal_error($msg))
+end
+
 
 """
     Like printstyled, but allows providing RGB colors for true color terminals

src/value_parsing.jl

@@ -21,12 +21,15 @@ function julia_string_to_number(str::AbstractString, kind)
         return x
     elseif kind == K"Float"
         if !startswith(str,"0x") && 'f' in str && !('p' in str)
-            # This is kind of awful. Should we have a separate Float32 literal
-            # type produced by the lexer?  The `f` suffix is nonstandard after all.
-            return Base.parse(Float32, replace(str, 'f'=>'e'))
+            # TODO: re-detecting Float32 here is kind of awful. Should have a
+            # separate Float32 literal type produced by the lexer?
+            x, code = _parse_float(Float32, str)
         else
-            return Base.parse(Float64, str)
+            x, code = _parse_float(Float64, str)
         end
+        return code === :ok           ? x :
+               code === :underflow    ? x : # < TODO: emit warning somehow?
+               #=code === :overflow=#   ErrorVal()
     elseif kind == K"HexInt"
         ndigits = length(str)-2
         return ndigits <= 2  ? Base.parse(UInt8, str)   :
@@ -66,6 +69,95 @@ function julia_string_to_number(str::AbstractString, kind)
 end
 
 
+#-------------------------------------------------------------------------------
+"""
+Like `Base.parse(Union{Float64,Float32}, str)`, but permits float underflow
+
+Parse a Float64. str[firstind:lastind] must be a valid floating point literal
+string. If the value is outside Float64 range.
+"""
+function _parse_float(::Type{T}, str::String,
+        firstind::Integer, lastind::Integer) where {T} # force specialize with where {T} 
+    strsize = lastind - firstind + 1
+    bufsz = 50
+    if strsize < bufsz
+        buf = Ref{NTuple{bufsz, UInt8}}()
+        ptr = Base.unsafe_convert(Ptr{UInt8}, pointer_from_objref(buf))
+        GC.@preserve str buf begin
+            unsafe_copyto!(ptr, pointer(str, firstind), strsize)
+            # Ensure ptr is null terminated
+            unsafe_store!(ptr, UInt8(0), strsize + 1)
+            _unsafe_parse_float(T, ptr, strsize)
+        end
+    else
+        # Slow path with allocation.
+        buf = Vector{UInt8}(str[firstind:lastind])
+        push!(buf, 0x00)
+        ptr = pointer(buf)
+        GC.@preserve buf _unsafe_parse_float(T, ptr, strsize)
+    end
+end
+
+function _parse_float(T, str::String)
+    _parse_float(T, str, firstindex(str), lastindex(str))
+end
+
+# Internals of _parse_float, split into a separate function to avoid some
+# apparent codegen issues https://github.com/JuliaLang/julia/issues/46509
+# (perhaps we don't want the `buf` in `GC.@preserve buf` to be stack allocated
+# on one branch and heap allocated in another?)
+@inline function _unsafe_parse_float(::Type{Float64}, ptr, strsize)
+    Libc.errno(0)
+    endptr = Ref{Ptr{UInt8}}(C_NULL)
+    x = @ccall jl_strtod_c(ptr::Ptr{UInt8}, endptr::Ptr{Ptr{UInt8}})::Cdouble
+    @check endptr[] == ptr + strsize
+    status = :ok
+    if Libc.errno() == Libc.ERANGE
+        # strtod man page:
+        # * If  the  correct  value  would cause overflow, plus or
+        #   minus HUGE_VAL, HUGE_VALF, or HUGE_VALL is returned and
+        #   ERANGE is stored in errno.
+        # * If the correct value would cause underflow, a value with
+        #   magnitude no larger than DBL_MIN, FLT_MIN, or LDBL_MIN is
+        #   returned and ERANGE is stored in errno.
+        status = abs(x) < 1 ? :underflow : :overflow
+    end
+    return (x, status)
+end
+
+@inline function _unsafe_parse_float(::Type{Float32}, ptr, strsize)
+    # Convert float exponent 'f' to 'e' for strtof, eg, 1.0f0 => 1.0e0
+    # Presumes we can modify the data in ptr!
+    for p in ptr+strsize-1:-1:ptr
+        if unsafe_load(p) == UInt8('f')
+            unsafe_store!(p, UInt8('e'))
+            break
+        end
+    end
+    Libc.errno(0)
+    endptr = Ref{Ptr{UInt8}}(C_NULL)
+    status = :ok
+    @static if Sys.iswindows()
+        # Call strtod here and convert to Float32 on the Julia side because
+        # strtof seems buggy on windows and doesn't set ERANGE correctly on
+        # overflow. See also
+        # https://github.com/JuliaLang/julia/issues/46544
+        x = Float32(@ccall jl_strtod_c(ptr::Ptr{UInt8}, endptr::Ptr{Ptr{UInt8}})::Cdouble)
+        if isinf(x)
+            status = :overflow
+            # Underflow not detected, but that will only be a warning elsewhere.
+        end
+    else
+        x = @ccall jl_strtof_c(ptr::Ptr{UInt8}, endptr::Ptr{Ptr{UInt8}})::Cfloat
+    end
+    @check endptr[] == ptr + strsize
+    if Libc.errno() == Libc.ERANGE
+        status = abs(x) < 1 ? :underflow : :overflow
+    end
+    return (x, status)
+end
+
+
 #-------------------------------------------------------------------------------
 is_indentation(c) = c == ' ' || c == '\t'
 

test/parser.jl

@@ -648,6 +648,7 @@ tests = [
         "```cmd```"  =>  "(macrocall :(Core.var\"@cmd\") \"cmd\")"
         # literals
         "42"   => "42"
+        "1.0e-1000"   => "0.0"
         "0x123456789abcdefp+0" => "8.19855292164869e16"
         # closing tokens
         ")"    => "(error)"

test/value_parsing.jl

@@ -1,6 +1,32 @@
 using JuliaSyntax:
     julia_string_to_number,
-    unescape_julia_string
+    unescape_julia_string,
+    _parse_float
+
+@testset "Float parsing" begin
+    # Float64
+    @test _parse_float(Float64, "123", 1, 3)   === (123.0, :ok)
+    @test _parse_float(Float64, "123", 2, 3)   === (23.0,  :ok)
+    @test _parse_float(Float64, "123", 2, 2)   === (2.0,   :ok)
+    @test _parse_float(Float64, "1.3", 1, 3)   === (1.3,   :ok)
+    @test _parse_float(Float64, "1.3e2", 1, 5) === (1.3e2, :ok)
+    @test _parse_float(Float64, "1.0e-1000", 1, 9) === (0.0, :underflow)
+    @test _parse_float(Float64, "1.0e+1000", 1, 9) === (Inf, :overflow)
+    # Slow path (exceeds static buffer size)
+    @test _parse_float(Float64, "0.000000000000000000000000000000000000000000000000000000000001") === (1e-60, :ok)
+
+    # Float32
+    @test _parse_float(Float32, "123", 1, 3) === (123.0f0, :ok)
+    @test _parse_float(Float32, "1.3f2", 1, 5) === (1.3f2, :ok)
+    if !Sys.iswindows()
+        @test _parse_float(Float32, "1.0f-50", 1, 7) === (0.0f0, :underflow)
+    end
+    @test _parse_float(Float32, "1.0f+50", 1, 7) === (Inf32, :overflow)
+
+    # Assertions
+    @test_throws ErrorException _parse_float(Float64, "x", 1, 1)
+    @test_throws ErrorException _parse_float(Float64, "1x", 1, 2)
+end
 
 hexint(s) = julia_string_to_number(s, K"HexInt")
 binint(s) = julia_string_to_number(s, K"BinInt")