Updates to support DataFrames as default return datastructure

Author: quinnj

src/SQLite.jl

@@ -1,9 +1,20 @@
-using DataStreams
+VERSION >= v"0.4.0-dev+6521" && __precompile__(true)
 module SQLite
 
-using Compat, NullableArrays, DataStreams, CSV
-const PointerString = Data.PointerString
-const NULLSTRING = Data.NULLSTRING
+using CSV, DataStreams, DataFrames, NullableArrays, WeakRefStrings
+
+export Data
+
+if !isdefined(Core, :String)
+    typealias String UTF8String
+    unsafe_string(ptr, len) = bytestring(ptr, len)
+end
+
+if Base.VERSION < v"0.5.0-dev+4631"
+    unsafe_wrap{A<:Array}(::Type{A}, ptr, len) = pointer_to_array(ptr, len)
+    unsafe_string(ptr, len) = utf8(ptr, len)
+    unsafe_wrap(::Type{String}, ptr, len) = unsafe_string(ptr, len)
+end
 
 type SQLiteException <: Exception
     msg::AbstractString
@@ -21,16 +32,16 @@ show(io::IO,::NullType) = print(io,"#NULL")
 # Normal constructor from filename
 sqliteopen(file,handle) = sqlite3_open(file,handle)
 sqliteopen(file::UTF16String,handle) = sqlite3_open16(file,handle)
-sqliteerror() = throw(SQLiteException(Compat.bytestring(sqlite3_errmsg())))
-sqliteerror(db) = throw(SQLiteException(Compat.bytestring(sqlite3_errmsg(db.handle))))
+sqliteerror() = throw(SQLiteException(unsafe_string(sqlite3_errmsg())))
+sqliteerror(db) = throw(SQLiteException(unsafe_string(sqlite3_errmsg(db.handle))))
 
 "represents an SQLite database, either backed by an on-disk file or in-memory"
 type DB
-    file::Compat.UTF8String
+    file::String
     handle::Ptr{Void}
     changes::Int
 
-    function DB(f::Compat.UTF8String)
+    function DB(f::AbstractString)
         handle = Ref{Ptr{Void}}()
         f = isempty(f) ? f : expanduser(f)
         if @OK sqliteopen(f,handle)
@@ -44,10 +55,8 @@ type DB
         end
     end
 end
-"`SQLite.DB(file::AbstractString)` opens or creates an SQLite database with `file`"
-DB(f::AbstractString) = DB(Compat.UTF8String(f))
 "`SQLite.DB()` creates an in-memory SQLite database"
-DB() = DB(Compat.UTF8String(":memory:"))
+DB() = DB(":memory:")
 
 function _close(db::DB)
     sqlite3_close_v2(db.handle)
@@ -81,15 +90,7 @@ end
 
 sqliteprepare(db,sql,stmt,null) = @CHECK db sqlite3_prepare_v2(db.handle,sql,stmt,null)
 
-# TO DEPRECATE
-type SQLiteDB{T<:AbstractString}
-   file::T
-   handle::Ptr{Void}
-   changes::Int
-end
-SQLiteDB(file,handle) = SQLiteDB(file,handle,0)
 include("UDF.jl")
-include("old_ui.jl")
 export @sr_str, @register, register
 
 "bind a row (`values`) to nameless parameters by index"
@@ -105,11 +106,11 @@ function bind!{V}(stmt::Stmt, values::Dict{Symbol, V})
     nparams = sqlite3_bind_parameter_count(stmt.handle)
     @assert nparams == length(values) "you must provide values for all placeholders"
     for i in 1:nparams
-        name = Compat.bytestring(sqlite3_bind_parameter_name(stmt.handle, i))
+        name = unsafe_string(sqlite3_bind_parameter_name(stmt.handle, i))
         @assert !isempty(name) "nameless parameters should be passed as a Vector"
         # name is returned with the ':', '@' or '$' at the start
         name = name[2:end]
-        bind!(stmt, i, values[@compat(Symbol)(name)])
+        bind!(stmt, i, values[Symbol(name)])
     end
 end
 # Binding parameters to SQL statements
@@ -126,12 +127,12 @@ bind!(stmt::Stmt,i::Int,val::Int32)          = (sqlite3_bind_int(stmt.handle,i,v
 bind!(stmt::Stmt,i::Int,val::Int64)          = (sqlite3_bind_int64(stmt.handle,i,val); return nothing)
 bind!(stmt::Stmt,i::Int,val::NullType)       = (sqlite3_bind_null(stmt.handle,i); return nothing)
 bind!(stmt::Stmt,i::Int,val::AbstractString) = (sqlite3_bind_text(stmt.handle,i,val); return nothing)
-bind!(stmt::Stmt,i::Int,val::PointerString{UInt8})   = (sqlite3_bind_text(stmt.handle,i,val.ptr,val.len); return nothing)
-bind!(stmt::Stmt,i::Int,val::PointerString{UInt16})  = (sqlite3_bind_text16(stmt.handle,i,val.ptr,val.len*2); return nothing)
+bind!(stmt::Stmt,i::Int,val::WeakRefString{UInt8})   = (sqlite3_bind_text(stmt.handle,i,val.ptr,val.len); return nothing)
+bind!(stmt::Stmt,i::Int,val::WeakRefString{UInt16})  = (sqlite3_bind_text16(stmt.handle,i,val.ptr,val.len*2); return nothing)
 bind!(stmt::Stmt,i::Int,val::UTF16String)    = (sqlite3_bind_text16(stmt.handle,i,val); return nothing)
-function bind!(stmt::Stmt,i::Int,val::PointerString{UInt32})
+function bind!(stmt::Stmt,i::Int,val::WeakRefString{UInt32})
     A = UTF32String(pointer_to_array(val.ptr, val.len+1, false))
-    return bind!(stmt, i, convert(Compat.UTF8String,A))
+    return bind!(stmt, i, convert(String,A))
 end
 # We may want to track the new ByteVec type proposed at https://github.com/JuliaLang/julia/pull/8964
 # as the "official" bytes type instead of Vector{UInt8}
@@ -310,7 +311,7 @@ end
 type Sink <: Data.Sink # <: IO
     schema::Data.Schema
     db::DB
-    tablename::Compat.UTF8String
+    tablename::String
     stmt::Stmt
 end
 

src/Sink.jl

@@ -11,7 +11,7 @@ can optionally provide an existing SQLite table name or new name that a created
 `temp=true` will create a temporary SQLite table that will be destroyed automatically when the database is closed
 `ifnotexists=false` will throw an error if `tablename` already exists in `db`
 """
-function Sink(schema::Data.Schema,db::DB,tablename::AbstractString="julia_"*randstring();temp::Bool=false,ifnotexists::Bool=true)
+function Sink(db::DB, schema::Data.Schema, tablename::AbstractString="julia_"*randstring();temp::Bool=false,ifnotexists::Bool=true)
     rows, cols = size(schema)
     temp = temp ? "TEMP" : ""
     ifnotexists = ifnotexists ? "IF NOT EXISTS" : ""
@@ -27,12 +27,12 @@ function Sink(source::SQLite.Source, tablename::AbstractString="julia_"*randstri
     return Sink(source.schema, source.db, tablename; temp=temp, ifnotexists=ifnotexists)
 end
 "constructs a new SQLite.Sink from the given `Data.Source`; uses `source` schema to create the SQLite table"
-function Sink(source::Data.Source, db::DB, tablename::AbstractString="julia_"*randstring();temp::Bool=false,ifnotexists::Bool=true)
-    return Sink(source.schema, db, tablename; temp=temp, ifnotexists=ifnotexists)
+function Sink(db::DB, source, tablename::AbstractString="julia_"*randstring();temp::Bool=false,ifnotexists::Bool=true)
+    return Sink(db, Data.schema(source), tablename; temp=temp, ifnotexists=ifnotexists)
 end
 
 # create a new SQLite table
-# Data.Table
+# DataFrame
 function getbind!{T}(dt::NullableVector{T},row,col,stmt)
     @inbounds isnull = dt.isnull[row]
     if isnull
@@ -43,16 +43,21 @@ function getbind!{T}(dt::NullableVector{T},row,col,stmt)
     end
     return
 end
+function getbind!{T}(dt::Vector{T}, row, col, stmt)
+    @inbounds val = dt[row]::T
+    SQLite.bind!(stmt, col, val)
+    return
+end
 "stream the data in `dt` into the SQLite table represented by `sink`"
-function Data.stream!(dt::Data.Table,sink::SQLite.Sink)
+function Data.stream!(dt::DataFrame, sink::SQLite.Sink)
     rows, cols = size(dt)
     types = Data.types(dt)
     handle = sink.stmt.handle
     transaction(sink.db) do
         if rows*cols != 0
             for row = 1:rows
                 for col = 1:cols
-                    @inbounds SQLite.getbind!(Data.unsafe_column(dt,col,types[col]),row,col,sink.stmt)
+                    @inbounds SQLite.getbind!(dt.columns[col],row,col,sink.stmt)
                 end
                 SQLite.sqlite3_step(handle)
                 SQLite.sqlite3_reset(handle)

src/Source.jl

@@ -19,10 +19,10 @@ function Source(db::DB, sql::AbstractString, values=[]; rows::Int=0, stricttypes
     bind!(stmt, values)
     status = SQLite.execute!(stmt)
     cols = SQLite.sqlite3_column_count(stmt.handle)
-    header = Array(Compat.UTF8String,cols)
+    header = Array(String,cols)
     types = Array(DataType,cols)
     for i = 1:cols
-        header[i] = Compat.bytestring(SQLite.sqlite3_column_name(stmt.handle,i))
+        header[i] = unsafe_string(SQLite.sqlite3_column_name(stmt.handle,i))
         # do better column type inference; query what the column was created for?
         types[i] = stricttypes ? SQLite.juliatype(stmt.handle,i) : Any
     end
@@ -41,23 +41,23 @@ function juliatype(handle,col)
         return juliatype(x)
     end
 end
-juliatype(x) = x == SQLITE_INTEGER ? Int : x == SQLITE_FLOAT ? Float64 : x == SQLITE_TEXT ? Compat.UTF8String : Any
+juliatype(x) = x == SQLITE_INTEGER ? Int : x == SQLITE_FLOAT ? Float64 : x == SQLITE_TEXT ? String : Any
 
 sqlitevalue{T<:Union{Signed,Unsigned}}(::Type{T},handle,col) = convert(T, sqlite3_column_int64(handle,col))
 const FLOAT_TYPES = Union{Float16,Float32,Float64} # exclude BigFloat
 sqlitevalue{T<:FLOAT_TYPES}(::Type{T},handle,col) = convert(T, sqlite3_column_double(handle,col))
-#TODO: test returning a PointerString instead of calling `bytestring`
-sqlitevalue{T<:AbstractString}(::Type{T},handle,col) = convert(T,Compat.bytestring(sqlite3_column_text(handle,col)))
+#TODO: test returning a WeakRefString instead of calling `bytestring`
+sqlitevalue{T<:AbstractString}(::Type{T},handle,col) = convert(T,unsafe_string(sqlite3_column_text(handle,col)))
 function sqlitevalue{T}(::Type{T},handle,col)
-    blob = convert(Ptr{UInt8},SQLite.sqlite3_column_blob(handle,col))
-    b = SQLite.sqlite3_column_bytes(handle,col)
+    blob = convert(Ptr{UInt8},sqlite3_column_blob(handle,col))
+    b = sqlite3_column_bytes(handle,col)
     buf = zeros(UInt8,b) # global const?
     unsafe_copy!(pointer(buf), blob, b)
-    r = SQLite.sqldeserialize(buf)::T
+    r = sqldeserialize(buf)::T
     return r
 end
 
-# `T` might be Int, Float64, String, PointerString, any Julia type, Any, NullType
+# `T` might be Int, Float64, String, WeakRefString, any Julia type, Any, NullType
 # `t` (the actual type of the value we're returning), might be SQLITE_INTEGER, SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB, SQLITE_NULL
 "`SQLite.getfield` returns the next `Nullable{T}` value from the `SQLite.Source`"
 function getfield{T}(source::SQLite.Source, ::Type{T}, row, col)
@@ -73,40 +73,39 @@ function getfield{T}(source::SQLite.Source, ::Type{T}, row, col)
     return val
 end
 
-function getfield!{T}(source::SQLite.Source, dest::NullableVector{T}, ::Type{T}, row, col)
+function getfield!{T}(source::SQLite.Source, dest::NullableVector{T}, row, col)
     @inbounds dest[row] = SQLite.getfield(source, T, row, col)
     return
 end
-function pushfield!{T}(source::SQLite.Source, dest::NullableVector{T}, ::Type{T}, row, col)
+function pushfield!{T}(source::SQLite.Source, dest::NullableVector{T}, row, col)
     push!(dest, SQLite.getfield(source, T, row, col))
     return
 end
-"streams data from the SQLite.Source to a Data.Table"
-function Data.stream!(source::SQLite.Source,sink::Data.Table)
+"streams data from the SQLite.Source to a DataFrame"
+function Data.stream!(source::SQLite.Source,sink::DataFrame)
     rows, cols = size(source)
     types = Data.types(source)
     if rows == 0
         row = 0
         while !Data.isdone(source)
             for col = 1:cols
                 @inbounds T = types[col]
-                SQLite.pushfield!(source, Data.unsafe_column(sink,col,T), T, row, col)
+                SQLite.pushfield!(source, sink.columns[col], row, col)
             end
             row += 1
         end
         source.schema.rows = row
     else
         for row = 1:rows, col = 1:cols
             @inbounds T = types[col]
-            SQLite.getfield!(source, Data.unsafe_column(sink,col,T), T, row, col)
+            SQLite.getfield!(source, sink.columns[col], row, col)
         end
     end
-    sink.schema = source.schema
     return sink
 end
-"creates a new Data.Table according to `source` schema and streams `Source` data into it"
-function Data.stream!(source::SQLite.Source,::Type{Data.Table})
-    sink = Data.Table(source.schema)
+"creates a new DataFrame according to `source` schema and streams `Source` data into it"
+function Data.stream!(source::SQLite.Source,::Type{DataFrame})
+    sink = DataFrame(source.schema)
     return Data.stream!(source,sink)
 end
 "streams data from an SQLite.Source to a CSV.Sink file; `header=false` will not write the column names to the file"
@@ -127,10 +126,10 @@ function Data.stream!(source::SQLite.Source,sink::CSV.Sink;header::Bool=true)
     close(sink)
     return sink
 end
-"convenience method for executing an SQL statement and streaming the results back in a Data.Table"
+"convenience method for executing an SQL statement and streaming the results back in a DataFrame"
 function query(db::DB,sql::AbstractString, values=[];rows::Int=0,stricttypes::Bool=true)
     so = Source(db,sql,values;rows=rows,stricttypes=stricttypes)
-    return Data.stream!(so,Data.Table)
+    return Data.stream!(so,DataFrame)
 end
 
 "returns a list of tables in `db`"

src/UDF.jl

@@ -12,7 +12,7 @@ function sqlvalue(values, i)
         return sqlite3_value_double(temp_val_ptr)
     elseif valuetype == SQLITE_TEXT
         # TODO: have a way to return UTF16
-        return Compat.bytestring(sqlite3_value_text(temp_val_ptr))
+        return unsafe_string(sqlite3_value_text(temp_val_ptr))
     elseif valuetype == SQLITE_BLOB
         nbytes = sqlite3_value_bytes(temp_val_ptr)
         blob = sqlite3_value_blob(temp_val_ptr)
@@ -32,7 +32,7 @@ sqlreturn(context, val::UTF16String)    = sqlite3_result_text16(context, val)
 sqlreturn(context, val::AbstractString) = sqlite3_result_text(context, val)
 sqlreturn(context, val::Vector{UInt8})  = sqlite3_result_blob(context, val)
 
-sqlreturn(context, val::Bool) = sqlreturn(context, @compat Int(val))
+sqlreturn(context, val::Bool) = sqlreturn(context, Int(val))
 sqlreturn(context, val) = sqlreturn(context, sqlserialize(val))
 
 # Internal method for generating an SQLite scalar function from
@@ -196,7 +196,7 @@ function register(db, func::Function; nargs::Int=-1, name::AbstractString=string
     nargs < -1 && (nargs = -1)
     @assert sizeof(name) <= 255 "size of function name must be <= 255"
 
-    f = eval(scalarfunc(func,@compat(Symbol)(name)))
+    f = eval(scalarfunc(func,Symbol(name)))
 
     cfunc = cfunction(f, Void, (Ptr{Void}, Cint, Ptr{Ptr{Void}}))
     # TODO: allow the other encodings