Writing (#21)

* wip write method

* working write and test

* better test

* delete test file

* simplify test

* fix reserved bytes

* bump version

* generic write working for test case

* wip

* test

* add 'O' dbf type

* better tests, add support for Dates because why not

* handle Integer and AbstractFloat separately for writing

* remove Julia 1.0 from ci

* change Julia lower bound to LTS

* add test

* cleanup

* convert unicode to spaces when writing character data to DBF

* remove comment

* removed unused ifelse, create dictcolumntable in write function

Author: joshday

.github/workflows/ci.yml

@@ -15,7 +15,6 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.0'
           - '1' # automatically expands to the latest stable 1.x release of Julia
           - 'nightly'
         os:

.gitignore

@@ -2,3 +2,5 @@
 *.jl.*.cov
 *.jl.mem
 Manifest.toml
+*.dbf
+*.DS_Store

Project.toml

@@ -1,16 +1,16 @@
 name = "DBFTables"
 uuid = "75c7ada1-017a-5fb6-b8c7-2125ff2d6c93"
-version = "1.0.0"
+version = "1.1.0"
 
 [deps]
-Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 WeakRefStrings = "ea10d353-3f73-51f8-a26c-33c1cb351aa5"
 
 [compat]
 Tables = "0.2, 1"
 WeakRefStrings = "0.6, 1"
-julia = "1.0"
+julia = "1.6"
 
 [extras]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"

src/DBFTables.jl

@@ -1,19 +1,21 @@
 module DBFTables
 
-import Printf, Tables, WeakRefStrings
+import Tables, WeakRefStrings
+using Dates
 
 "Field/column descriptor, part of the Header"
 struct FieldDescriptor
     name::Symbol
     type::Type
+    dbf_type::Char
     length::UInt8
     ndec::UInt8
 end
 
 "DBF header, which also holds all field definitions"
 struct Header
     version::UInt8
-    last_update::String
+    last_update::Date
     records::UInt32
     hsize::UInt16
     rsize::UInt16
@@ -45,7 +47,7 @@ function typemap(fld::Char, ndec::UInt8)
     if fld == 'C'
         rt = String
     elseif fld == 'D'
-        rt = String
+        rt = Date
     elseif fld == 'N'
         if ndec > 0
             rt = Float64
@@ -74,16 +76,29 @@ function read_dbf_field(io::IO)
     field_dec = read(io, UInt8)
     skip(io, 14)  # reserved
     jltype = typemap(field_type, field_dec)
-    return FieldDescriptor(field_name, jltype, field_len, field_dec)
+    return FieldDescriptor(field_name, jltype, field_type, field_len, field_dec)
+end
+
+reserved(n) = fill(0x00, n)
+
+function Base.write(io::IO, fd::FieldDescriptor)
+    out = 0
+    out += Base.write(io, replace(rpad(String(fd.name), 11), ' ' => '\0'))  # 0-10
+    out += Base.write(io, fd.dbf_type)  # 11
+    out += Base.write(io, reserved(4))  # 12-15
+    out += Base.write(io, fd.length)  # 16
+    out += Base.write(io, fd.ndec)  # 17
+    out += Base.write(io, reserved(14))  # 18-31
+    return out
 end
 
 "Read a DBF header from a stream"
 function Header(io::IO)
     ver = read(io, UInt8)
-    date1 = read(io, UInt8)
-    date2 = read(io, UInt8)
-    date3 = read(io, UInt8)
-    last_update = Printf.@sprintf("%4d%02d%02d", date1 + 1900, date2, date3)
+    yy = read(io, UInt8)
+    mm = read(io, UInt8)
+    dd = read(io, UInt8)
+    last_update = Date(yy + 1900, mm, dd)
     records = read(io, UInt32)
     hsize = read(io, UInt16)
     rsize = read(io, UInt16)
@@ -130,6 +145,34 @@ function Header(io::IO)
     )
 end
 
+
+
+# ref: https://www.clicketyclick.dk/databases/xbase/format/dbf.html
+function Base.Base.write(io::IO, h::Header)
+    out = 0
+    out += Base.write(io, h.version)  # 0
+    yy = UInt8(year(h.last_update) - 1900)
+    mm = UInt8(month(h.last_update))
+    dd = UInt8(day(h.last_update))
+    out += Base.write(io, yy, mm, dd)  # 1-3
+    out += Base.write(io, h.records)  # 4-7
+    out += Base.write(io, h.hsize)  # 8-9
+    out += Base.write(io, h.rsize)  # 10-11
+    out += Base.write(io, reserved(2))  # 12-13 reserved
+    out += Base.write(io, h.incomplete)  # 14
+    out += Base.write(io, h.encrypted)  # 15
+    out += Base.write(io, reserved(12))  # 16-19, 20-27 reserved
+    out += Base.write(io, h.mdx)  # 28
+    out += Base.write(io, h.lang_id)  # 29
+    out += Base.write(io, reserved(2))  # 30-31 reserved
+    for field in h.fields
+        out += Base.write(io, field)
+    end
+    out += Base.write(io, 0xD)
+    return out
+end
+
+
 miss(x) = ifelse(x === nothing, missing, x)
 
 "Concert a DBF entry string to a Julia value"
@@ -142,10 +185,12 @@ function dbf_value(::Type{Bool}, str::AbstractString)
     elseif char == '?'
         missing
     else
-        throw(ArgumentError("Unknown logical $char"))
+        throw(ArgumentError("Unknown logical entry: $(repr(char))"))
     end
 end
 
+dbf_value(::Type{Date}, str::AbstractString) = all(isspace, str) ? missing : Date(str, dateformat"yyyymmdd")
+
 dbf_value(T::Union{Type{Int},Type{Float64}}, str::AbstractString) = miss(tryparse(T, str))
 # String to avoid returning SubString{String}
 function dbf_value(::Type{String}, str::AbstractString)
@@ -196,7 +241,7 @@ function Table(path::AbstractString)
     end
 end
 
-"Collect all the offsets and lenghts from the header to create a StringArray"
+"Collect all the offsets and lengths from the header to create a StringArray"
 function _create_stringarray(header::Header, data::AbstractVector)
     # first make the lengths and offsets for a single record
     lengths_record = UInt32.(getfield.(header.fields, :length))
@@ -308,4 +353,112 @@ function Base.getproperty(dbf::Table, name::Symbol)
     return colarr
 end
 
+
+Base.write(io::IO, dbf::Table) = Base.write(io, getfield(dbf, :header), getfield(dbf, :data), 0x1a)
+Base.write(path::AbstractString, dbf::Table) = open(io -> Base.write(io, dbf), touch(path), "w")
+
+
+"Generic .dbf writer for the Tables.jl interface."
+write(path::AbstractString, tbl) = open(io -> write(io, tbl), touch(path), "w")
+
+function write(io::IO, tbl)
+    dct = Tables.dictcolumntable(tbl)
+    fields, records = get_field_descriptors(dct)
+    fieldcolumns = Dict{Symbol,Int}(f.name => i for (i,f) in enumerate(fields))
+    hsize = UInt16(length(fields) * 32 + 32)
+    rsize = UInt16(sum(x -> x.length, fields)) + 1
+
+    version = 0x03
+    last_update = today()
+    incomplete = false
+    encrypted = false
+    mdx = false
+    lang_id = 0x00
+
+    h = Header(version, last_update, records, hsize, rsize, incomplete, encrypted, mdx, lang_id, fields, fieldcolumns)
+    out = Base.write(io, h)
+
+    for row in Tables.rows(dct)
+        out += write_record(io, fields, row)
+    end
+    out += Base.write(io, 0x1a)  # EOF marker
+    return out
+end
+
+function get_field_descriptors(dct)
+    fields = FieldDescriptor[]
+    sch = Tables.schema(dct)
+    for (name, type) in zip(sch.names, sch.types)
+        ndec = 0x0
+        len = 0x0
+        dbf_type = 'C'
+        T = Base.nonmissingtype(type)
+        if T isa Date
+            dbf_type = 'D'
+            len = 0x08
+        elseif T <: AbstractString
+            # TODO: support memos.  Currently strings > 254 bytes will error
+            len = UInt8(maximum(x -> length(string(x)), dct[name]))
+            if len > 254
+                @warn "Strings will be truncated to 254 characters."
+                len = 254
+            end
+            dbf_type = 'C'
+        elseif type === Float64
+            dbf_type = 'O'
+            len = 0x08
+            ndec = 0x01
+        elseif T <: AbstractFloat
+            dbf_type = 'F'
+            len = UInt8(20)
+            ndec = 0x01
+        elseif T <: Bool
+            dbf_type = 'L'
+            len = 0x1
+        elseif T <: Date
+            dbf_type = 'D'
+            len = 0x8
+        elseif T <: Integer
+            dbf_type = 'N'
+            len = UInt8(maximum(x -> length(string(x)), dct[name]))
+        else
+            @warn "Field $name has no known matching DBF data type for $T.  Data will be stored as the DBF character data type ('C')."
+            len = UInt8(maximum(x -> length(string(x)), dct[name]))
+        end
+        push!(fields, FieldDescriptor(name, type, dbf_type, len, ndec))
+    end
+    fields, UInt32(length(first(dct)))
+end
+
+function write_record(io::IO, fd::Vector{FieldDescriptor}, row)
+    out = 0
+    out += Base.write(io, ' ')  # deletion marker ' '=valid, '*'=deleted
+    for (field, val) in zip(fd, row)
+        out += Base.write(io, _val(field, val))
+    end
+    return out
+end
+
+function _val(field::FieldDescriptor, val)::Union{String, Float64}
+    char = field.dbf_type
+    if char == 'L'
+        ismissing(val) && return "?"
+        val ? "T" : "F"
+    elseif ismissing(val)
+        ' ' ^ field.length
+    elseif char == 'C'
+        replace(rpad(val, field.length), !isascii => ' ')
+    elseif char == 'D'
+        Dates.format(val, "yyyymmdd")
+    elseif char == 'O'
+        val  # <-- the Float64 return value
+    elseif char == 'F'
+        rpad(val, 20)[1:20]
+    elseif char == 'N'
+        rpad(val, field.length)
+    else
+        error("Unknown DBF datatype $char.")
+    end
+end
+
 end # module

test/runtests.jl

@@ -2,18 +2,46 @@ using DBFTables
 using Test
 using Tables
 using DataFrames
+using Dates
 
 test_dbf_path = joinpath(@__DIR__, "test.dbf")
 dbf = DBFTables.Table(test_dbf_path)
 df = DataFrame(dbf)
 row, st = iterate(dbf)
 
 @testset "DBFTables" begin
+    @testset "Writing" begin
+        tables_equal(tbl1, tbl2) = all(zip(Tables.columns(tbl1), Tables.columns(tbl2))) do (t1, t2)
+            all(ismissing(a) ? ismissing(b) : a == b for (a,b) in zip(t1,t2))
+        end
+        function _roundtrip(table)
+            file = joinpath(tempdir(), "test.dbf")
+            DBFTables.write(file, table)
+            table2 = DBFTables.Table(file)
+        end
+        roundtrip(table) = tables_equal(DataFrame(table), DataFrame(_roundtrip(table)))
+        @test roundtrip(df)
+        @test roundtrip(dbf)
+        @test roundtrip([(x=Float32(1), y=1), (x=Float32(2), y=2), (x=missing, y=3)])
+        @test roundtrip([(x=true, y="test"), (x=missing, y=missing)])
+        @test roundtrip([(x=today(), y=missing), (x=missing, y=today())])
+        @test roundtrip([(; x=1.0), (;x=missing)])
+        @test roundtrip([(; x=missing), (; x=missing)])
+
+        @test_warn "Data will be stored as the DBF character data type" DBFTables.write(tempname(), [(; x = rand(10))])
+
+        # Base.write for DBFTables.Table
+        file = joinpath(tempdir(), "test.dbf")
+        write(file, dbf)
+        dbf2 = DBFTables.Table(file)
+        @test tables_equal(dbf, dbf2)
+    end
+
     @testset "DataFrame indexing" begin
         @test size(df, 1) == 7 # records
         @test size(df, 2) == 6 # fields
         @test df[2, :CHAR] == "John"
-        @test df[1, :DATE] == "19900102"
+        @test df[1, :DATE] == Date("19900102", dateformat"yyyymmdd")
         @test df[3, :BOOL] == false
         @test df[1, :FLOAT] == 10.21
         @test df[2, :NUMERIC] == 12.21
@@ -30,7 +58,7 @@ row, st = iterate(dbf)
     @testset "header" begin
         h = DBFTables.Header(open(test_dbf_path))
         @test h.version == 3
-        @test h.last_update == "20140806"
+        @test h.last_update == Date("20140806", dateformat"yyyymmdd")
         @test h.records == 7
         @test length(h.fields) == 6
     end
@@ -39,18 +67,18 @@ row, st = iterate(dbf)
         @test sprint(show, row) === sprint(show, NamedTuple(row))
         # use replace to update to julia 1.4 union printing
         @test replace(sprint(show, dbf), r"\} +" => "}") ===
-              "DBFTables.Table with 7 rows and 6 columns\nTables.Schema:\n :CHAR     Union{Missing, String}\n :DATE     Union{Missing, String}\n :BOOL     Union{Missing, Bool}\n :FLOAT    Union{Missing, Float64}\n :NUMERIC  Union{Missing, Float64}\n :INTEGER  Union{Missing, $Int}\n"
+              "DBFTables.Table with 7 rows and 6 columns\nTables.Schema:\n :CHAR     Union{Missing, String}\n :DATE     Union{Missing, Date}\n :BOOL     Union{Missing, Bool}\n :FLOAT    Union{Missing, Float64}\n :NUMERIC  Union{Missing, Float64}\n :INTEGER  Union{Missing, $Int}\n"
     end
 
     @testset "iterate" begin
         @test st === 2
         @test haskey(row, :CHAR)
         @test row.CHAR === "Bob"
-        @test row[2] === "19900102"
+        @test row[2] === Date("19900102", dateformat"yyyymmdd")
         @test_throws ArgumentError row.nonexistent_field
         firstrow = (
             CHAR = "Bob",
-            DATE = "19900102",
+            DATE = Date("19900102", dateformat"yyyymmdd"),
             BOOL = false,
             FLOAT = 10.21,
             NUMERIC = 11.21,