Tables.jl integration (#154)

* Updates for 1.0

* Refactor Source/Sink to implement Tables.jl interface

* Updates for new Tables.jl interface

* Add Tables to REQUIRE

Author: quinnj

.travis.yml

@@ -7,7 +7,7 @@ os:
   - osx
 
 julia:
-  - 0.7
+  - 1.0
   - nightly
 
 notifications:

REQUIRE

@@ -4,4 +4,5 @@ DataFrames 0.11.0
 WeakRefStrings 0.4.0
 LegacyStrings
 Missings
-BinaryProvider
+BinaryProvider
+Tables

appveyor.yml

@@ -1,6 +1,6 @@
 environment:
   matrix:
-  - julia_version: 0.7
+  - julia_version: 1
   - julia_version: nightly
 
 platform:

docs/src/index.md

@@ -5,14 +5,8 @@
 
 ## High-level interface
 ```@docs
-SQLite.query
-SQLite.load
-```
-
-## Lower-level utilities
-```@docs
-SQLite.Source
-SQLite.Sink
+SQLite.Query
+SQLite.load!
 ```
 
 ## Types/Functions
@@ -59,11 +53,21 @@ SQLite.Sink
   Used to execute a prepared `SQLite.Stmt`. The 2nd method is a convenience method to pass in an SQL statement as a string which gets prepared and executed in one call. This method does not check for or return any results, hence it is only useful for database manipulation methods (i.e. ALTER, CREATE, UPDATE, DROP). To return results, see `SQLite.query` below.
 
 
-* `SQLite.query(db::SQLite.DB, sql::String, values=[])`
+* `SQLite.Query(db::SQLite.DB, sql::String, values=[])`
+
+  Constructs a `SQLite.Query` object by executing the SQL query `sql` against the sqlite database `db` and querying
+the columns names and types of the result set, if any.
 
-  An SQL statement `sql` is prepared, executed in the context of `db`, and results, if any, are returned. The return value is a `Data.Table` by default from the `DataStreams.jl` package. The `Data.Table` has a field `.data` which is a `Vector{NullableVector}` which holds the columns of data returned from the `sql` statement.
+Will bind `values` to any parameters in `sql`.
+`stricttypes=false` will remove strict column typing in the result set, making each column effectively `Vector{Any}`; in sqlite, individual
+column values are only loosely associated with declared column types, and instead each carry their own type information. This can lead to
+type errors when trying to query columns when a single type is expected.
+`nullable` controls whether `null` (`missing` in Julia) values are expected in a column.
 
-  The values in `values` are used in parameter binding (see `bind!` above). If your statement uses nameless parameters `values` must be a `Vector` of the values you wish to bind to your statment. If your statement uses named parameters `values` must be a Dict where the keys are of type `Symbol`. The key must match an identifier name in the statement (the name **should not** include the ':', '@' or '$' prefix).
+An `SQLite.Query` object will iterate NamedTuple rows by default, and also supports the Tables.jl interface for integrating with
+any other Tables.jl implementation. Due note however that iterating an sqlite result set is a forward-once-only operation. If you need
+to iterate over an `SQLite.Query` multiple times, but can't store the iterated NamedTuples, call `SQLite.reset!(q::SQLite.Query)` to
+re-execute the query and position the iterator back at the begining of the result set.
 
 
 * `SQLite.drop!(db::SQLite.DB,table::String;ifexists::Bool=false)`
@@ -135,15 +139,15 @@ julia> db = SQLite.DB("Chinook_Sqlite.sqlite")
 
 julia> # using SQLite's in-built syntax
 
-julia> SQLite.query(db, "SELECT FirstName, LastName FROM Employee WHERE LastName REGEXP 'e(?=a)'")
+julia> SQLite.Query(db, "SELECT FirstName, LastName FROM Employee WHERE LastName REGEXP 'e(?=a)'") |> DataFrame
 1x2 ResultSet
 | Row | "FirstName" | "LastName" |
 |-----|-------------|------------|
 | 1   | "Jane"      | "Peacock"  |
 
 julia> # explicitly calling the regexp() function
 
-julia> SQLite.query(db, "SELECT * FROM Genre WHERE regexp('e[trs]', Name)")
+julia> SQLite.Query(db, "SELECT * FROM Genre WHERE regexp('e[trs]', Name)") |> DataFrame
 6x2 ResultSet
 | Row | "GenreId" | "Name"               |
 |-----|-----------|----------------------|
@@ -156,20 +160,20 @@ julia> SQLite.query(db, "SELECT * FROM Genre WHERE regexp('e[trs]', Name)")
 
 julia> # you can even do strange things like this if you really want
 
-julia> SQLite.query(db, "SELECT * FROM Genre ORDER BY GenreId LIMIT 2")
+julia> SQLite.Query(db, "SELECT * FROM Genre ORDER BY GenreId LIMIT 2") |> DataFrame
 2x2 ResultSet
 | Row | "GenreId" | "Name" |
 |-----|-----------|--------|
 | 1   | 1         | "Rock" |
 | 2   | 2         | "Jazz" |
 
-julia> SQLite.query(db, "INSERT INTO Genre VALUES (regexp('^word', 'this is a string'), 'My Genre')")
+julia> SQLite.Query(db, "INSERT INTO Genre VALUES (regexp('^word', 'this is a string'), 'My Genre')") |> DataFrame
 1x1 ResultSet
 | Row | "Rows Affected" |
 |-----|-----------------|
 | 1   | 0               |
 
-julia> SQLite.query(db, "SELECT * FROM Genre ORDER BY GenreId LIMIT 2")
+julia> SQLite.Query(db, "SELECT * FROM Genre ORDER BY GenreId LIMIT 2") |> DataFrame
 2x2 ResultSet
 | Row | "GenreId" | "Name"     |
 |-----|-----------|------------|
@@ -180,13 +184,13 @@ julia> SQLite.query(db, "SELECT * FROM Genre ORDER BY GenreId LIMIT 2")
 Due to the heavy use of escape characters you may run into problems where julia parses out some backslashes in your query, for example `"\y"` simply becomes `"y"`. For example the following two queries are identical
 
 ```julia
-julia> SQLite.query(db, "SELECT * FROM MediaType WHERE Name REGEXP '-\d'")
+julia> SQLite.Query(db, "SELECT * FROM MediaType WHERE Name REGEXP '-\d'") |> DataFrame
 1x1 ResultSet
 | Row | "Rows Affected" |
 |-----|-----------------|
 | 1   | 0               |
 
-julia> SQLite.query(db, "SELECT * FROM MediaType WHERE Name REGEXP '-d'")
+julia> SQLite.Query(db, "SELECT * FROM MediaType WHERE Name REGEXP '-d'") |> DataFrame
 1x1 ResultSet
 | Row | "Rows Affected" |
 |-----|-----------------|
@@ -198,15 +202,15 @@ This can be avoided in two ways. You can either escape each backslash yourself o
 ```julia
 julia> # manually escaping backslashes
 
-julia> SQLite.query(db, "SELECT * FROM MediaType WHERE Name REGEXP '-\\d'")
+julia> SQLite.Query(db, "SELECT * FROM MediaType WHERE Name REGEXP '-\\d'") |> DataFrame
 1x2 ResultSet
 | Row | "MediaTypeId" | "Name"                        |
 |-----|---------------|-------------------------------|
 | 1   | 3             | "Protected MPEG-4 video file" |
 
 julia> # using sr"..."
 
-julia> SQLite.query(db, sr"SELECT * FROM MediaType WHERE Name REGEXP '-\d'")
+julia> SQLite.Query(db, sr"SELECT * FROM MediaType WHERE Name REGEXP '-\d'") |> DataFrame
 1x2 ResultSet
 | Row | "MediaTypeId" | "Name"                        |
 |-----|---------------|-------------------------------|

src/SQLite.jl

@@ -202,6 +202,47 @@ end
  #int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
  #int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
 
+function juliatype(handle, col)
+    t = SQLite.sqlite3_column_decltype(handle, col)
+    if t != C_NULL
+        T = juliatype(unsafe_string(t))
+        T !== Any && return T
+    end
+    x = SQLite.sqlite3_column_type(handle, col)
+    if x == SQLite.SQLITE_BLOB
+        val = SQLite.sqlitevalue(Any, handle, col)
+        return typeof(val)
+    else
+        return juliatype(x)
+    end
+end
+
+juliatype(x::Integer) = x == SQLITE_INTEGER ? Int : x == SQLITE_FLOAT ? Float64 : x == SQLITE_TEXT ? String : Any
+juliatype(x::String) = x == "INTEGER" ? Int : x in ("NUMERIC","REAL") ? Float64 : x == "TEXT" ? String : Any
+
+sqlitevalue(::Type{T}, handle, col) where {T <: Union{Base.BitSigned, Base.BitUnsigned}} = convert(T, sqlite3_column_int64(handle, col))
+const FLOAT_TYPES = Union{Float16, Float32, Float64} # exclude BigFloat
+sqlitevalue(::Type{T}, handle, col) where {T <: FLOAT_TYPES} = convert(T, sqlite3_column_double(handle, col))
+#TODO: test returning a WeakRefString instead of calling `unsafe_string`
+sqlitevalue(::Type{T}, handle, col) where {T <: AbstractString} = convert(T, unsafe_string(sqlite3_column_text(handle, col)))
+function sqlitevalue(::Type{T}, handle, col) where {T}
+    blob = convert(Ptr{UInt8}, sqlite3_column_blob(handle, col))
+    b = sqlite3_column_bytes(handle, col)
+    buf = zeros(UInt8, b) # global const?
+    unsafe_copyto!(pointer(buf), blob, b)
+    r = sqldeserialize(buf)::T
+    return r
+end
+
+sqlitetype(::Type{T}) where {T<:Integer} = "INT NOT NULL"
+sqlitetype(::Type{T}) where {T<:Union{Missing, Integer}} = "INT"
+sqlitetype(::Type{T}) where {T<:AbstractFloat} = "REAL NOT NULL"
+sqlitetype(::Type{T}) where {T<:Union{Missing, AbstractFloat}} = "REAL"
+sqlitetype(::Type{T}) where {T<:AbstractString} = "TEXT NOT NULL"
+sqlitetype(::Type{T}) where {T<:Union{Missing, AbstractString}} = "TEXT"
+sqlitetype(::Type{Missing}) = "NULL"
+sqlitetype(x) = "BLOB"
+
 """
 `SQLite.execute!(stmt::SQLite.Stmt)` => `Cvoid`
 
@@ -221,6 +262,7 @@ function execute!(stmt::Stmt)
     end
     return r
 end
+
 function execute!(db::DB, sql::AbstractString)
     stmt = Stmt(db, sql)
     return execute!(stmt)
@@ -238,7 +280,6 @@ function esc_id end
 esc_id(x::AbstractString) = "\"" * replace(x, "\""=>"\"\"") * "\""
 esc_id(X::AbstractVector{S}) where {S <: AbstractString} = join(map(esc_id, X), ',')
 
-
 # Transaction-based commands
 """
 `SQLite.transaction(db, mode="DEFERRED")`
@@ -264,7 +305,8 @@ function transaction(db, mode="DEFERRED")
         execute!(db, "SAVEPOINT $(mode);")
     end
 end
-function transaction(f::Function, db)
+
+@inline function transaction(f::Function, db)
     # generate a random name for the savepoint
     name = string("SQLITE", Random.randstring(10))
     execute!(db, "PRAGMA synchronous = OFF;")
@@ -388,5 +430,27 @@ end
 
 include("Source.jl")
 include("Sink.jl")
+include("tables.jl")
+
+"""
+`SQLite.tables(db, sink=DataFrame)`
+
+returns a list of tables in `db`
+"""
+tables(db::DB, sink=DataFrame) = Query(db, "SELECT name FROM sqlite_master WHERE type='table';") |> sink
+
+"""
+`SQLite.indices(db, sink=DataFrame)`
+
+returns a list of indices in `db`
+"""
+indices(db::DB, sink=DataFrame) = Query(db, "SELECT name FROM sqlite_master WHERE type='index';") |> sink
+
+"""
+`SQLite.columns(db, table, sink=DataFrame)`
+
+returns a list of columns in `table`
+"""
+columns(db::DB,table::AbstractString, sink=DataFrame) = Query(db, "PRAGMA table_info($(esc_id(table)))") |> sink
 
 end # module

src/Sink.jl

@@ -1,12 +1,3 @@
-sqlitetype(::Type{T}) where {T<:Integer} = "INT NOT NULL"
-sqlitetype(::Type{T}) where {T<:Union{Missing, Integer}} = "INT"
-sqlitetype(::Type{T}) where {T<:AbstractFloat} = "REAL NOT NULL"
-sqlitetype(::Type{T}) where {T<:Union{Missing, AbstractFloat}} = "REAL"
-sqlitetype(::Type{T}) where {T<:AbstractString} = "TEXT NOT NULL"
-sqlitetype(::Type{T}) where {T<:Union{Missing, AbstractString}} = "TEXT"
-sqlitetype(::Type{Missing}) = "NULL"
-sqlitetype(x) = "BLOB"
-
 function createtable!(db::DB, name::AbstractString, schema::Data.Schema; temp::Bool=false, ifnotexists::Bool=true)
     rows, cols = size(schema)
     temp = temp ? "TEMP" : ""
@@ -25,6 +16,7 @@ can optionally provide an existing SQLite table name or new name that a created
 `ifnotexists=false` will throw an error if `name` already exists in `db`
 """
 function Sink(db::DB, name::AbstractString, schema::Data.Schema=Data.Schema(); temp::Bool=false, ifnotexists::Bool=true, append::Bool=false)
+    Base.depwarn("`SQLite.Sink(db, name)` is deprecated in favor of calling `SQLite.load!(table, db, name)` where `table` can be any Tables.jl interface implementation", nothing)
     cols = size(Data.schema(SQLite.query(db, "pragma table_info($name)")), 1)
     if cols == 0
         createtable!(db, name, schema)
@@ -86,13 +78,22 @@ Load a Data.Source `source` into an SQLite table that will be named `tablename`
 `ifnotexists=false` will throw an error if `tablename` already exists in `db`
 """
 function load(db::SQLite.DB, name, ::Type{T}, args...; append::Bool=false, transforms::Dict=Dict{Int,Function}(), kwargs...) where {T}
+    Base.depwarn("`SQLite.load(db, name, args...)` is deprecated in favor of calling `SQLite.load!(table, db, name)` where `table` can be any Tables.jl interface implementation", nothing)
     sink = Data.stream!(T(args...), SQLite.Sink, db, name; append=append, transforms=transforms, kwargs...)
     return Data.close!(sink)
 end
 function load(db::SQLite.DB, name, source::T; append::Bool=false, transforms::Dict=Dict{Int,Function}(), kwargs...) where {T}
+    Base.depwarn("`SQLite.load(db, name, args...)` is deprecated in favor of calling `SQLite.load!(table, db, name)` where `table` can be any Tables.jl interface implementation", nothing)
     sink = Data.stream!(source, SQLite.Sink, db, name; append=append, transforms=transforms, kwargs...)
     return Data.close!(sink)
 end
 
-load(sink::Sink, ::Type{T}, args...; append::Bool=false, transforms::Dict=Dict{Int,Function}()) where {T} = (sink = Data.stream!(T(args...), sink; append=append, transforms=transforms); return Data.close!(sink))
-load(sink::Sink, source; append::Bool=false, transforms::Dict=Dict{Int,Function}()) = (sink = Data.stream!(source, sink; append=append, transforms=transforms); return Data.close!(sink))
+function load(sink::Sink, ::Type{T}, args...; append::Bool=false, transforms::Dict=Dict{Int,Function}()) where {T}
+    Base.depwarn("`SQLite.load(sink::SQLite.Sink, args...)` is deprecated in favor of calling `SQLite.load!(table, db, name)` where `table` can be any Tables.jl interface implementation", nothing)
+    (sink = Data.stream!(T(args...), sink; append=append, transforms=transforms); return Data.close!(sink))
+end
+
+function load(sink::Sink, source; append::Bool=false, transforms::Dict=Dict{Int,Function}())
+    Base.depwarn("`SQLite.load(sink::SQLite.Sink, args...)` is deprecated in favor of calling `SQLite.load!(table, db, name)` where `table` can be any Tables.jl interface implementation", nothing)
+    (sink = Data.stream!(source, sink; append=append, transforms=transforms); return Data.close!(sink))
+end