Jd/dev (#26)

* bump version

* truncate Numeric types to 20 characters in a smarter way

* fix typo

Author: joshday

Project.toml

@@ -4,6 +4,7 @@ version = "1.2.2"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 WeakRefStrings = "ea10d353-3f73-51f8-a26c-33c1cb351aa5"
 

src/DBFTables.jl

@@ -2,6 +2,7 @@ module DBFTables
 
 import Tables, WeakRefStrings
 using Dates
+using Printf: @sprintf
 
 "Field/column descriptor, part of the Header"
 struct FieldDescriptor
@@ -96,12 +97,25 @@ dbf_value(::Val{'C'}, len::UInt8, ::Missing) = ' ' ^ len
 dbf_value(::Val{'L'}, ::UInt8, x::Bool) = x ? 'T' : 'F'
 dbf_value(::Val{'L'}, ::UInt8, ::Missing) = '?'
 
-# Integer & AbstractFloat
-function dbf_value(::Val{'N'}, ::UInt8, x::Union{AbstractFloat, Integer})
-    minval = -9999999999999999999
-    maxval = 99999999999999999999
-    minval ≤ x ≤ maxval && @warn "Due to DBF limitations, a float will be clamped to fit in 20 characters."
-    rpad(clamp(x, minval, maxval), 20)
+# Integer
+function dbf_value(::Val{'N'}, ::UInt8, x::Integer)
+    s = rpad(x, 20)
+    length(s) > 20 ? error("The DBF format cannot save integers >20 characters.") : s
+end
+
+# AbstractFloat
+function dbf_value(::Val{'N'}, ::UInt8, x::AbstractFloat)
+    s = rpad(x, 20)
+    length(s) == 20 && return s
+    # Force into scientific notation with 20 decimal places
+    s2 = @sprintf "%.20e" x
+    i = findfirst('e', s2)
+    s_end = replace(s2[i:end], '+' => "")
+    len  = length(s_end)
+    n = 20 - len
+    out = s2[1:n] * s_end
+    @warn "A DBF limitation has reduced the precision of $x by $(length(s) - 20) digits."
+    return out
 end
 dbf_value(::Val{'N'}, ::UInt8, ::Missing) = ' ' ^ 20
 
@@ -121,7 +135,7 @@ julia_type(::Val{'O'}, ndec::UInt8) = Float64
 julia_type(::Val{'I'}, ndec::UInt8) = Int32
 julia_type(::Val{'+'}, ndec::UInt8) = Int64
 julia_type(::Val{'L'}, ndec::UInt8) = Bool
-julia_type(::Val{'M'}, ndec::UInt8) = String 
+julia_type(::Val{'M'}, ndec::UInt8) = String
 function julia_type(::Val{T}, ndec::UInt8) where {T}
     @warn "Unknown DBF type code '$T'.  Data will be loaded as `String"
     String
@@ -171,7 +185,7 @@ function julia_value(::Type{Bool}, ::Val{'L'}, s::AbstractString)
         return true
     elseif char in "NnFf"
         return false
-    else 
+    else
         return missing
     end
 end
@@ -258,7 +272,7 @@ end
 
 
 # ref: https://www.clicketyclick.dk/databases/xbase/format/dbf.html
-function Base.Base.write(io::IO, h::Header)
+function Base.write(io::IO, h::Header)
     out = 0
     out += Base.write(io, h.version)  # 0
     yy = UInt8(year(h.last_update) - 1900)

test/runtests.jl

@@ -118,4 +118,16 @@ row, st = iterate(dbf)
         @test !DBFTables.isdeleted(dbf, 3)
     end
 
+    @testset "Numeric 20-character Limit Nonsense" begin
+        big = BigInt(99999_99999_99999_99999)
+        @test DBFTables.dbf_value(Val('N'), 0x01, big) == string(big)
+        @test_throws Exception DBFTables.dbf_value(Val('N'), 0x01, big + 1)
+
+        negbig = -BigInt(99999_99999_99999_9999)  # one less digit for the minus sign
+        @test DBFTables.dbf_value(Val('N'), 0x01, negbig) == string(negbig)
+        @test_throws Exception DBFTables.dbf_value(Val('N'), 0x01, negbig - 1)
+
+        @test_warn r"DBF limitation" DBFTables.dbf_value(Val('N'), 0x01, prevfloat(Inf))
+        @test_warn r"DBF limitation" DBFTables.dbf_value(Val('N'), 0x01, nextfloat(-Inf))
+    end
 end  # testset "DBFTables"