Skip to content

Commit 51c70ca

Browse files
quinnjquinnj
committed
Update tests to run a full datastreams integration test between SQLite.Source, SQLite.Sink, and DataFrame
1 parent ba8ee89 commit 51c70ca

File tree

4 files changed

+66
-137
lines changed

4 files changed

+66
-137
lines changed

‎src/SQLite.jl

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,11 @@
1-
VERSION >= v"0.4.0-dev+6521" && __precompile__(true)
1+
__precompile__(true)
22
module SQLite
33

44
using DataStreams, DataFrames, WeakRefStrings, LegacyStrings
55
import LegacyStrings: UTF16String
66

77
export Data, DataFrame
88

9-
if !isdefined(Core, :String)
10-
typealias String UTF8String
11-
end
12-
13-
if Base.VERSION < v"0.5.0-dev+4631"
14-
unsafe_string = bytestring
15-
unsafe_wrap{A<:Array}(::Type{A}, ptr, len) = pointer_to_array(ptr, len)
16-
unsafe_wrap(::Type{String}, ptr, len) = unsafe_string(ptr, len)
17-
end
18-
199
type SQLiteException <: Exception
2010
msg::AbstractString
2111
end

‎src/Source.jl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ sqlitevalue{T<:Union{Signed,Unsigned}}(::Type{T}, handle, col) = convert(T, sqli
5050
const FLOAT_TYPES = Union{Float16, Float32, Float64} # exclude BigFloat
5151
sqlitevalue{T<:FLOAT_TYPES}(::Type{T}, handle, col) = convert(T, sqlite3_column_double(handle, col))
5252
#TODO: test returning a WeakRefString instead of calling `bytestring`
53-
sqlitevalue{T<:AbstractString}(::Type{T}, handle, col) = convert(T,unsafe_string(sqlite3_column_text(handle, col)))
53+
sqlitevalue{T<:AbstractString}(::Type{T}, handle, col) = convert(T, unsafe_string(sqlite3_column_text(handle, col)))
5454
function sqlitevalue{T}(::Type{T}, handle, col)
5555
blob = convert(Ptr{UInt8}, sqlite3_column_blob(handle, col))
5656
b = sqlite3_column_bytes(handle, col)

‎test/datastreams.jl

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
2+
# DataFrames
3+
FILE = joinpath(DSTESTDIR, "randoms_small.csv")
4+
DF = readtable(FILE)
5+
DF[:hiredate] = NullableArray(Date[isnull(x) ? Date() : Date(get(x)) for x in DF[:hiredate]], [isnull(x) for x in DF[:hiredate]])
6+
DF[:lastclockin] = NullableArray(DateTime[isnull(x) ? DateTime() : DateTime(get(x)) for x in DF[:lastclockin]], [isnull(x) for x in DF[:lastclockin]])
7+
DF2 = deepcopy(DF)
8+
dfsource = Tester("DataFrame", x->x, false, DataFrame, (:DF,), scalartransforms, vectortransforms, x->x, x->nothing)
9+
dfsink = Tester("DataFrame", x->x, false, DataFrame, (:DF2,), scalartransforms, vectortransforms, x->x, x->nothing)
10+
function DataFrames.DataFrame(sym::Symbol; append::Bool=false)
11+
return @eval $sym
12+
end
13+
function DataFrames.DataFrame(sch::Data.Schema, ::Type{Data.Field}, append::Bool, ref::Vector{UInt8}, sym::Symbol)
14+
return DataFrame(DataFrame(sym), sch, Data.Field, append, ref)
15+
end
16+
function DataFrame(sink, sch::Data.Schema, ::Type{Data.Field}, append::Bool, ref::Vector{UInt8})
17+
rows, cols = size(sch)
18+
newsize = max(0, rows) + (append ? size(sink, 1) : 0)
19+
# need to make sure we don't break a NullableVector{WeakRefString{UInt8}} when appending
20+
if append
21+
for (i, T) in enumerate(Data.types(sch))
22+
if T <: Nullable{WeakRefString{UInt8}}
23+
sink.columns[i] = NullableArray(String[string(get(x, "")) for x in sink.columns[i]])
24+
sch.types[i] = Nullable{String}
25+
end
26+
end
27+
else
28+
for (i, T) in enumerate(Data.types(sch))
29+
if T != eltype(sink.columns[i])
30+
sink.columns[i] = NullableArray(eltype(T), newsize)
31+
end
32+
end
33+
end
34+
newsize != size(sink, 1) && foreach(x->resize!(x, newsize), sink.columns)
35+
if !append
36+
for (i, T) in enumerate(eltypes(sink))
37+
if T <: Nullable{WeakRefString{UInt8}}
38+
sink.columns[i] = NullableArray{WeakRefString{UInt8}, 1}(Array{WeakRefString{UInt8}}(newsize), fill(true, newsize), isempty(ref) ? UInt8[] : ref)
39+
end
40+
end
41+
end
42+
sch.rows = newsize
43+
return sink
44+
end
45+
46+
# SQLite
47+
dbfile = joinpath(DSTESTDIR, "randoms.sqlite")
48+
dbfile2 = joinpath(DSTESTDIR, "randoms2.sqlite")
49+
cp(dbfile, dbfile2; remove_destination=true)
50+
db = SQLite.DB(dbfile)
51+
db2 = SQLite.DB(dbfile2)
52+
SQLite.createtable!(db2, "randoms2_small", Data.schema(SQLite.Source(db, "select * from randoms_small")))
53+
sqlitesource = Tester("SQLite.Source", SQLite.query, true, SQLite.Source, (db, "select * from randoms_small"), scalartransforms, vectortransforms, x->x, ()->nothing)
54+
sqlitesink = Tester("SQLite.Sink", SQLite.load, true, SQLite.Sink, (db2, "randoms2_small"), scalartransforms, vectortransforms, x->SQLite.query(db2, "select * from randoms2_small"), (x,y)->nothing)
55+
56+
DataStreamsIntegrationTests.teststream([dfsource, sqlitesource], [dfsink, sqlitesink]; rows=99)

‎test/runtests.jl

Lines changed: 8 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,8 @@
11
using SQLite
2-
using Base.Test, CSV, DataStreams, DataFrames, NullableArrays
2+
using Base.Test, DataStreams, DataFrames, NullableArrays, WeakRefStrings
33

44
import Base: +, ==
55

6-
if !isdefined(Core, :String)
7-
typealias String UTF8String
8-
end
9-
106
a = SQLite.DB()
117

128
temp = tempname()
@@ -19,107 +15,9 @@ dbfile2 = joinpath(dirname(@__FILE__),"test.sqlite")
1915
cp(dbfile, dbfile2; remove_destination=true)
2016
db = SQLite.DB(dbfile2)
2117

22-
# DataStreams interface tests
23-
source_table = "employee"
24-
sink_table = "employee2"
25-
sink_file = "employee2.csv"
26-
selall(x) = "select * from " * x
27-
28-
# SQLite.Source ==> DataFrame
29-
ds = SQLite.query(db, selall(source_table))
30-
@test size(ds) == (8,15)
31-
# SQLite.Source ==> SQLite.Sink
32-
SQLite.query(db, selall(source_table), SQLite.Sink, db, sink_table)
33-
ds2 = SQLite.query(db, selall(sink_table))
34-
@test size(ds2) == (8,15)
35-
@test Data.types(ds) == Data.types(ds2) && Data.header(ds) == Data.header(ds2)
36-
# SQLite.Source ==> CSV.Sink
37-
SQLite.query(db, selall(source_table), CSV.Sink, sink_file)
38-
ds4 = CSV.read(sink_file; dateformat="yyyy-mm-dd HH:MM:SS")
39-
@test size(ds4) == (8,15)
40-
@test Data.header(ds) == Data.header(ds4)
41-
42-
SQLite.query(db, selall(source_table), SQLite.Sink, db, sink_table; append=true)
43-
ds5 = SQLite.query(db, selall(sink_table))
44-
@test size(ds5) == (16,15)
45-
@test Data.types(ds) == Data.types(ds2) && Data.header(ds) == Data.header(ds2)
46-
47-
# constructed SQLite.Sink
48-
sink = SQLite.Sink(db, sink_table, Data.schema(ds5))
49-
SQLite.query(db, selall(source_table), sink)
50-
ds2 = SQLite.query(db, selall(sink_table))
51-
@test size(ds2) == (8,15)
52-
@test Data.types(ds) == Data.types(ds2) && Data.header(ds) == Data.header(ds2)
53-
SQLite.query(db, selall(source_table), sink; append=true)
54-
ds3 = SQLite.query(db, selall(sink_table))
55-
@test size(ds3) == (16,15)
56-
57-
# constructed SQLite.Source
58-
source = SQLite.Source(db, selall(source_table))
59-
ds = SQLite.query(source)
60-
@test size(ds) == (8,15)
61-
source = SQLite.Source(db, selall(source_table))
62-
SQLite.query(source, SQLite.Sink, db, sink_table)
63-
ds2 = SQLite.query(db, selall(sink_table))
64-
@test size(ds2) == (8,15)
65-
source = SQLite.Source(db, selall(source_table))
66-
SQLite.query(source, SQLite.Sink, db, sink_table; append=true)
67-
ds3 = SQLite.query(db, selall(sink_table))
68-
@test size(ds3) == (16,15)
69-
70-
sink = SQLite.Sink(db, sink_table, Data.schema(ds))
71-
source = SQLite.Source(db, selall(source_table))
72-
SQLite.query(source, sink)
73-
ds = SQLite.query(db, selall(sink_table))
74-
@test size(ds) == (8,15)
75-
source = SQLite.Source(db, selall(source_table))
76-
SQLite.query(source, sink; append=true)
77-
ds2 = SQLite.query(db, selall(sink_table))
78-
@test size(ds2) == (16,15)
79-
80-
# SQLite.load
81-
si = SQLite.load(db, sink_table, SQLite.Source, db, selall(source_table))
82-
ds = SQLite.query(db, selall(sink_table))
83-
@test size(ds) == (8,15)
84-
si = SQLite.load(db, sink_table, SQLite.Source, db, selall(source_table); append=true)
85-
ds2 = SQLite.query(db, selall(sink_table))
86-
@test size(ds2) == (16,15)
87-
88-
source = SQLite.Source(db, selall(source_table))
89-
si = SQLite.load(db, sink_table, source)
90-
ds = SQLite.query(db, selall(sink_table))
91-
@test size(ds) == (8,15)
92-
source = SQLite.Source(db, selall(source_table))
93-
si = SQLite.load(db, sink_table, source; append=true)
94-
ds2 = SQLite.query(db, selall(sink_table))
95-
@test size(ds2) == (16,15)
96-
97-
sink = SQLite.Sink(db, sink_table, Data.schema(ds))
98-
si = SQLite.load(sink, SQLite.Source, db, selall(source_table))
99-
ds = SQLite.query(db, selall(sink_table))
100-
@test size(ds) == (8,15)
101-
sink = SQLite.Sink(db, sink_table, Data.schema(ds); append=true)
102-
si = SQLite.load(sink, SQLite.Source, db, selall(source_table); append=true)
103-
ds2 = SQLite.query(db, selall(sink_table))
104-
@test size(ds2) == (16,15)
105-
106-
source = SQLite.Source(db, selall(source_table))
107-
sink = SQLite.Sink(db, sink_table, Data.schema(ds))
108-
si = SQLite.load(sink, source)
109-
ds = SQLite.query(db, selall(sink_table))
110-
@test size(ds) == (8,15)
111-
source = SQLite.Source(db, selall(source_table))
112-
sink = SQLite.Sink(db, sink_table, Data.schema(ds); append=true)
113-
si = SQLite.load(sink, source; append=true)
114-
ds2 = SQLite.query(db, selall(sink_table))
115-
@test size(ds2) == (16,15)
116-
SQLite.drop!(db, sink_table)
117-
SQLite.drop!(db, "sqlite_stat1")
118-
rm(sink_file)
119-
12018
# regular SQLite tests
12119
so = SQLite.Source(db,"SELECT name FROM sqlite_master WHERE type='table';")
122-
ds = Data.stream!(so,DataFrame)
20+
ds = Data.stream!(so, DataFrame)
12321
@test length(ds.columns) == 1
12422
@test Data.header(ds)[1] == "name"
12523
@test size(ds) == (11,1)
@@ -356,29 +254,8 @@ SQLite.drop!(db2, "tab2", ifexists=true)
356254
SQLite.drop!(db, "sqlite_stat1")
357255
@test size(SQLite.tables(db)) == (11,1)
358256

359-
source = SQLite.Source(db, "select * from album")
360-
temp = tempname()
361-
sink = CSV.Sink(temp)
362-
Data.stream!(source,sink)
363-
Data.close!(sink)
364-
dt = Data.stream!(CSV.Source(sink), DataFrame)
365-
@test get(dt[1,1]) == 1
366-
@test get(dt[1,2]) == "For Those About To Rock We Salute You"
367-
@test get(dt[1,3]) == 1
368-
369257
db = nothing; gc(); gc();
370258

371-
db = SQLite.DB()
372-
source = CSV.Source(temp)
373-
sink = SQLite.Sink(db, "temp", Data.schema(source))
374-
Data.stream!(source, sink)
375-
Data.close!(sink)
376-
source2 = SQLite.Source(sink)
377-
dt = Data.stream!(source2, DataFrame)
378-
@test get(dt[1,1]) == 1
379-
@test string(get(dt[1,2])) == "For Those About To Rock We Salute You"
380-
@test get(dt[1,3]) == 1
381-
382259
sink = SQLite.Sink(db, "temp2", Data.schema(dt, Data.Field))
383260
Data.stream!(dt, sink)
384261
Data.close!(sink)
@@ -430,3 +307,9 @@ stmt = SQLite.Stmt(db, "INSERT INTO tbl (a) VALUES (@a);")
430307
SQLite.bind!(stmt, "@a", 1)
431308

432309
rm(dbfile2)
310+
311+
installed = Pkg.installed()
312+
haskey(installed, "DataStreamsIntegrationTests") || Pkg.clone("https://github.com/JuliaData/DataStreamsIntegrationTests")
313+
using DataStreamsIntegrationTests
314+
315+
include("datastreams.jl")

0 commit comments

Comments
 (0)