Introduce a new DateAndTime sturct for > millisecond precision support (#178)

Fixes #165 (and #172). So the proposal here is a bit tricky, but here's
what's going on in this PR:
  * Introduce a new DateAndTime struct, which just contains separate
  `Date` and `Time` fields, where the `time` field can support up to
  Nanosecond precision
  * For _prepared_ statement execution, we actually return a DateTime
  column for > millisecond precision, and there used to be no warning.
  We now emit a warning that precision is being lost and that
  `mysql_date_and_time=true` should be passed to avoid such loss
  * For _directly executed_ queries, we throw an InexactError and now
  emit the same warning that `mysql_date_and_time=true` should be passed
  to avoid such errors. We could perhaps match the prepare behavior and
  return `DateTime`, but I wanted to avoid changing too much all at once
  * Why introduce a keyword arg instead of just making it the default?
  Well, that'd be breaking. It's also tricky because we don't
  necessarily want to force this new `DateAndTime` struct on _all_
  DATETIME/TIMESTAMP columns, especially if their precision is within
  the millisecond range. i.e. it's much more convenient for users to
  work direclty with `DateTime` objects instead of this new
  `DateAndTime` struct. So the behavior now is: emit clear warnings in
  cases where DateTime can't handle the precision, and users can pass
  `mysql_date_and_time=true` and then handle `DateAndTime` objects
  accordingly.

Author: quinnj

src/MySQL.jl

@@ -2,7 +2,7 @@ module MySQL
 
 using Dates, DBInterface, Tables, Parsers, DecFP
 
-export DBInterface
+export DBInterface, DateAndTime
 
 # For non-C-api errors that happen in MySQL.jl
 struct MySQLInterfaceError
@@ -300,11 +300,12 @@ end
 Base.close(conn::Connection) = DBInterface.close!(conn)
 Base.isopen(conn::Connection) = API.isopen(conn.mysql)
 
-function juliatype(field_type, notnullable, isunsigned, isbinary)
+function juliatype(field_type, notnullable, isunsigned, isbinary, date_and_time)
     T = API.juliatype(field_type)
     T2 = isunsigned && !(T <: AbstractFloat) ? unsigned(T) : T
     T3 = !isbinary && T2 == Vector{UInt8} ? String : T2
-    return notnullable ? T3 : Union{Missing, T3}
+    T4 = date_and_time && T3 <: DateTime ? DateAndTime : T3
+    return notnullable ? T4 : Union{Missing, T4}
 end
 
 include("execute.jl")

src/api/API.jl

@@ -2,6 +2,8 @@ module API
 
 using Dates, DecFP
 
+export DateAndTime
+
 if VERSION < v"1.3.0"
 
 # Load libmariadb from our deps.jl

src/api/apitypes.jl

@@ -116,28 +116,44 @@ const MYSQL_TIME_FORMAT = Dates.DateFormat("HH:MM:SS.s")
 const MYSQL_DATE_FORMAT = Dates.DateFormat("yyyy-mm-dd")
 const MYSQL_DATETIME_FORMAT = Dates.DateFormat("yyyy-mm-dd HH:MM:SS.s")
 
+@noinline dateandtime_warning() = @warn """a datetime value from a column has a microsecond precision > 3,
+by default, MySQL.jl attempts to return a DateTime object, which only supports millisecond precision.
+To avoid loss in precision or InexactErrors, pass `mysql_date_and_time=true` to `DBInterface.execute(stmt, sql; mysql_date_and_time=true)` or `DBInterface.prepare(stmt, sql; mysql_date_and_time=true)`.
+This will result in a column element type of `DateAndTime`, which is a simple struct of separate Date and Time parts, accessed like `dt.date` and `dt.time`.
+"""
+
 function Base.convert(::Type{DateTime}, mtime::MYSQL_TIME)
+    millis, micros = divrem(mtime.second_part, 1000)
     if mtime.year == 0 || mtime.month == 0 || mtime.day == 0
-        DateTime(1970, 1, 1,
-                 mtime.hour, mtime.minute, mtime.second)
+        dt = DateTime(1970, 1, 1,
+                 mtime.hour, mtime.minute, mtime.second, millis)
     else
-        DateTime(mtime.year, mtime.month, mtime.day,
-                 mtime.hour, mtime.minute, mtime.second)
+        dt = DateTime(mtime.year, mtime.month, mtime.day,
+                 mtime.hour, mtime.minute, mtime.second, millis)
     end
+    micros > 0 && dateandtime_warning()
+    return dt
 end
 Base.convert(::Type{Dates.Time}, mtime::MYSQL_TIME) =
-    Dates.Time(mtime.hour, mtime.minute, mtime.second)
+    Dates.Time(mtime.hour, mtime.minute, mtime.second, divrem(mtime.second_part, 1000)...)
 Base.convert(::Type{Date}, mtime::MYSQL_TIME) =
     Date(mtime.year, mtime.month, mtime.day)
+Base.convert(::Type{DateAndTime}, mtime::MYSQL_TIME) =
+    DateAndTime(Date(mtime.year, mtime.month, mtime.day),
+                Time(mtime.hour, mtime.minute, mtime.second, divrem(mtime.second_part, 1000)...))
 
 Base.convert(::Type{MYSQL_TIME}, t::Dates.Time) =
-    MYSQL_TIME(0, 0, 0, Dates.hour(t), Dates.minute(t), Dates.second(t), 0, 0, 0)
+    MYSQL_TIME(0, 0, 0, Dates.hour(t), Dates.minute(t), Dates.second(t), Dates.millisecond(t) * 1000 + Dates.microsecond(t), 0, 0)
 Base.convert(::Type{MYSQL_TIME}, dt::Date) =
     MYSQL_TIME(Dates.year(dt), Dates.month(dt), Dates.day(dt), 0, 0, 0, 0, 0, 0)
 
 Base.convert(::Type{MYSQL_TIME}, dtime::DateTime) =
     MYSQL_TIME(Dates.year(dtime), Dates.month(dtime), Dates.day(dtime),
-               Dates.hour(dtime), Dates.minute(dtime), Dates.second(dtime), 0, 0, 0)
+               Dates.hour(dtime), Dates.minute(dtime), Dates.second(dtime), Dates.millisecond(dtime) * 1000, 0, 0)
+
+Base.convert(::Type{MYSQL_TIME}, dat::DateAndTime) =
+    MYSQL_TIME(Dates.year(dat), Dates.month(dat), Dates.day(dat),
+               Dates.hour(dat), Dates.minute(dat), Dates.second(dat), Dates.millisecond(dat) * 1000 + Dates.microsecond(dat), 0, 0)
 
 # this is a helper struct, because MYSQL_BIND needs
 # to know where the bound data should live, by using this helper

src/api/consts.jl

@@ -48,6 +48,25 @@ end
 Base.show(io::IO, b::Bit) = print(io, "MySQL.API.Bit(\"$(string(b))\")")
 Base.unsigned(::Type{Bit}) = Bit
 
+struct DateAndTime <: Dates.AbstractDateTime
+    date::Date
+    time::Time
+end
+
+Dates.Date(x::DateAndTime) = x.date
+Dates.Time(x::DateAndTime) = x.time
+Dates.year(x::DateAndTime) = Dates.year(Date(x))
+Dates.month(x::DateAndTime) = Dates.month(Date(x))
+Dates.day(x::DateAndTime) = Dates.day(Date(x))
+Dates.hour(x::DateAndTime) = Dates.hour(Time(x))
+Dates.minute(x::DateAndTime) = Dates.minute(Time(x))
+Dates.second(x::DateAndTime) = Dates.second(Time(x))
+Dates.millisecond(x::DateAndTime) = Dates.millisecond(Time(x))
+Dates.microsecond(x::DateAndTime) = Dates.microsecond(Time(x))
+
+import Base.==
+==(a::DateAndTime, b::DateAndTime) = ==(a.date, b.date) && ==(a.time, b.time)
+
 mysqltype(::Type{Bit}) = MYSQL_TYPE_BIT
 mysqltype(::Union{Type{Cchar}, Type{Cuchar}}) = MYSQL_TYPE_TINY
 mysqltype(::Union{Type{Cshort}, Type{Cushort}}) = MYSQL_TYPE_SHORT
@@ -58,6 +77,7 @@ mysqltype(::Type{Dec64}) = MYSQL_TYPE_DECIMAL
 mysqltype(::Type{Cdouble}) = MYSQL_TYPE_DOUBLE
 mysqltype(::Type{Vector{UInt8}}) = MYSQL_TYPE_BLOB
 mysqltype(::Type{DateTime}) = MYSQL_TYPE_TIMESTAMP
+mysqltype(::Type{DateAndTime}) = MYSQL_TYPE_DATETIME
 mysqltype(::Type{Date}) = MYSQL_TYPE_DATE
 mysqltype(::Type{Time}) = MYSQL_TYPE_TIME
 mysqltype(::Type{Missing}) = MYSQL_TYPE_NULL

src/execute.jl

@@ -10,6 +10,7 @@ mutable struct TextCursor{buffered} <: DBInterface.Cursor
     lookup::Dict{Symbol, Int}
     current_rownumber::Int
     current_resultsetnumber::Int
+    mysql_date_and_time::Bool
 end
 
 struct TextRow{buffered} <: Tables.AbstractRow
@@ -64,13 +65,34 @@ const DATETIME_OPTIONS = Parsers.Options(dateformat=dateformat"yyyy-mm-dd HH:MM:
 
 function cast(::Type{DateTime}, ptr, len)
     buf = unsafe_wrap(Array, ptr, len)
-    x, code, pos = Parsers.typeparser(DateTime, buf, 1, len, buf[1], Int16(0), DATETIME_OPTIONS)
-    if code > 0
-        return x
+    try
+        x, code, pos = Parsers.typeparser(DateTime, buf, 1, len, buf[1], Int16(0), DATETIME_OPTIONS)
+        if code > 0
+            return x
+        end
+    catch e
+        e isa InexactError && API.dateandtime_warning()
     end
     casterror(DateTime, ptr, len)
 end
 
+const DATEANDTIME_OPTIONS = Parsers.Options(dateformat=dateformat"yyyy-mm-dd HH:MM:SS")
+
+function cast(::Type{DateAndTime}, ptr, len)
+    buf = unsafe_wrap(Array, ptr, len)
+    i = findfirst(==(UInt8('.')), buf)
+    x, code, pos = Parsers.typeparser(DateTime, buf, 1, something(i, len), buf[1], Int16(0), DATETIME_OPTIONS)
+    if code > 0
+        dt, tm = Date(x), Time(x)
+        if i !== nothing
+            y, code, pos = Parsers.typeparser(Int, buf, i + 1, len, buf[1], Int16(0), Parsers.OPTIONS)
+            tm += Dates.Microsecond(y)
+        end
+        return DateAndTime(dt, tm)
+    end
+    casterror(DateAndTime, ptr, len)
+end
+
 @noinline wrongrow(i) = throw(ArgumentError("row $i is no longer valid; mysql results are forward-only iterators where each row is only valid when iterated"))
 
 function Tables.getcolumn(r::TextRow, ::Type{T}, i::Int, nm::Symbol) where {T}
@@ -129,7 +151,7 @@ Specifying `mysql_store_result=false` will avoid buffering the full resultset to
 the query, which has memory use advantages, though ties up the database server since resultset rows must be
 fetched one at a time.
 """
-function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); mysql_store_result::Bool=true)
+function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); mysql_store_result::Bool=true, mysql_date_and_time::Bool=false)
     checkconn(conn)
     params != () && error("`DBInterface.execute(conn, sql)` does not support parameter binding; see `?DBInterface.prepare(conn, sql)`")
     clear!(conn)
@@ -154,7 +176,7 @@ function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); m
         nfields = API.numfields(result)
         fields = API.fetchfields(result, nfields)
         names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
-        types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
+        types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), mysql_date_and_time) for x in fields]
     elseif API.fieldcount(conn.mysql) == 0
         rows_affected = API.affectedrows(conn.mysql)
         names = Symbol[]
@@ -163,7 +185,7 @@ function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); m
         error("error with mysql resultset columns")
     end
     lookup = Dict(x => i for (i, x) in enumerate(names))
-    return TextCursor{buffered}(conn, sql, nfields, nrows, Core.bitcast(Int64, rows_affected), result, names, types, lookup, 0, 1)
+    return TextCursor{buffered}(conn, sql, nfields, nrows, Core.bitcast(Int64, rows_affected), result, names, types, lookup, 0, 1, mysql_date_and_time)
 end
 
 struct TextCursors{T}
@@ -186,7 +208,7 @@ function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered
             cursor.cursor.nfields = API.numfields(cursor.cursor.result)
             fields = API.fetchfields(cursor.cursor.result, cursor.cursor.nfields)
             cursor.cursor.names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
-            cursor.cursor.types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
+            cursor.cursor.types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), cursor.cursor.mysql_date_and_time) for x in fields]
         else
             return nothing
         end

src/load.jl

@@ -29,6 +29,7 @@ const SQLTYPES = Dict{Type, String}(
     Date => "DATE",
     Time => "TIME",
     DateTime => "DATETIME",
+    DateAndTime => "DATETIME(6)",
 )
 
 checkdupnames(names) = length(unique(map(x->lowercase(String(x)), names))) == length(names) || error("duplicate case-insensitive column names detected; sqlite doesn't allow duplicate column names and treats them case insensitive")

src/prepare.jl

@@ -40,7 +40,7 @@ Note that `DBInterface.close!(stmt)` should be called once statement executions
 freeing resources, it has been noted that too many unclosed statements and resultsets, used in conjunction
 with streaming queries (i.e. `mysql_store_result=false`) has led to occasional resultset corruption.
 """
-function DBInterface.prepare(conn::Connection, sql::AbstractString)
+function DBInterface.prepare(conn::Connection, sql::AbstractString; mysql_date_and_time::Bool=false)
     clear!(conn)
     stmt = API.stmtinit(conn.mysql)
     API.prepare(stmt, sql)
@@ -52,7 +52,7 @@ function DBInterface.prepare(conn::Connection, sql::AbstractString)
     if result.ptr != C_NULL
         fields = API.fetchfields(result, nfields)
         names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
-        types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
+        types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), mysql_date_and_time) for x in fields]
         valuehelpers = [API.BindHelper() for i = 1:nfields]
         values = [API.MYSQL_BIND(valuehelpers[i].length, valuehelpers[i].is_null) for i = 1:nfields]
         foreach(1:nfields) do i
@@ -147,7 +147,7 @@ Specifying `mysql_store_result=false` will avoid buffering the full resultset to
 the query, which has memory use advantages, though ties up the database server since resultset rows must be
 fetched one at a time.
 """
-function DBInterface.execute(stmt::Statement, params=(); mysql_store_result::Bool=true)
+function DBInterface.execute(stmt::Statement, params=(); mysql_store_result::Bool=true, mysql_date_and_time::Bool=false)
     checkstmt(stmt)
     paramcheck(stmt, params)
     clear!(stmt.conn)
@@ -179,7 +179,7 @@ function DBInterface.execute(stmt::Statement, params=(); mysql_store_result::Boo
         if result.ptr != C_NULL
             fields = API.fetchfields(result, nfields)
             names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
-            types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
+            types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), mysql_date_and_time) for x in fields]
             valuehelpers = [API.BindHelper() for i = 1:nfields]
             values = [API.MYSQL_BIND(valuehelpers[i].length, valuehelpers[i].is_null) for i = 1:nfields]
             foreach(1:nfields) do i
@@ -250,6 +250,7 @@ ptrhelper(helper, x::API.MYSQL_TIME) = pointer(helper.time)
 getvalue(stmt, helper, values, i, ::Type{Time}) = convert(Time, helper.time[1])
 getvalue(stmt, helper, values, i, ::Type{Date}) = convert(Date, helper.time[1])
 getvalue(stmt, helper, values, i, ::Type{DateTime}) = convert(DateTime, helper.time[1])
+getvalue(stmt, helper, values, i, ::Type{DateAndTime}) = convert(DateAndTime, helper.time[1])
 
 inithelper!(helper, x::String) = nothing
 ptrhelper(helper, x::String) = C_NULL

test/runtests.jl

@@ -242,6 +242,19 @@ res = DBInterface.execute(resstmt) |> columntable
 @test length(res) == 2
 @test res[2][1] == DateTime(1970, 1, 1, 3)
 
+# https://github.com/JuliaDatabases/MySQL.jl/issues/165
+DBInterface.execute(conn, "DROP TABLE if exists datetime6_field")
+DBInterface.execute(conn, "CREATE TABLE datetime6_field (id int(11), t DATETIME(6))")
+stmt = DBInterface.prepare(conn, "INSERT INTO datetime6_field (id, t) VALUES (?, ?);")
+DBInterface.execute(stmt, [1, DateAndTime(Date(2021, 1, 2), Time(1, 2, 3, 456, 789))])
+resstmt = DBInterface.prepare(conn, "select id, t from datetime6_field"; mysql_date_and_time=true)
+res = DBInterface.execute(resstmt) |> columntable
+@test length(res) == 2
+@test res[2][1] == DateAndTime(Date(2021, 1, 2), Time(1, 2, 3, 456, 789))
+res = DBInterface.execute(conn, "select id, t from datetime6_field"; mysql_date_and_time=true) |> columntable
+@test length(res) == 2
+@test res[2][1] == DateAndTime(Date(2021, 1, 2), Time(1, 2, 3, 456, 789))
+
 DBInterface.execute(conn, """
 CREATE PROCEDURE get_employee()
 BEGIN