Skip to content

Commit ca9ebd1

Browse files
quinnjquinnj
committed
Reworked query and createtable functions for 2x speedup and cleaner code. Also reworked API to prepare for readdlmsql function
1 parent f832592 commit ca9ebd1

File tree

4 files changed

+127
-133
lines changed

4 files changed

+127
-133
lines changed

‎lib/sqlite3 renamed to ‎lib/sqlite3.so

File renamed without changes.

‎src/Sqlite.jl

Lines changed: 123 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ module Sqlite
22

33
using DataFrames
44

5-
export sqlitedb #, sqlite3_column_int64, sqlite3_column_text, sqlite3_column_double, FUNCS, SQL2Julia, sqlite3_column_name, sqlite3_column_type, sqlite3_prepare_v2, sqlite3_step, sqlite3_column_count, SQLITE_NULL, sqlite3_column_int, SQLITE_ROW, SQLITE_DONE, sqlite3_reset, sqlite3_finalize
5+
export sqlitedb
66

7-
include("sqlite_consts.jl")
8-
include("sqlite_api.jl")
7+
include("Sqlite_consts.jl")
8+
include("Sqlite_api.jl")
99

1010
type SqliteDB
1111
file::String
@@ -28,168 +28,166 @@ function show(io::IO,db::SqliteDB)
2828
end
2929
end
3030

31+
typealias TableInput Union(DataFrame,String)
32+
3133
const null_resultset = DataFrame(0)
3234
const null_SqliteDB = SqliteDB("",C_NULL,null_resultset)
3335
sqlitedb = null_SqliteDB #Create default connection = null
34-
ret = "" #For returning readable error codes
3536

3637
#Core Functions
3738
function connect(file::String)
3839
global sqlitedb
3940
handle = Array(Ptr{Void},1)
4041
if @FAILED sqlite3_open(file,handle)
41-
error("[sqlite]: $ret; Error opening $file")
42+
error("[sqlite]: Error opening $file; $(bytestring(sqlite3_errmsg(conn.handle)))")
4243
else
4344
return (sqlitedb = SqliteDB(file,handle[1],null_resultset))
4445
end
4546
end
46-
function query(conn::SqliteDB=sqlitedb,q::String)
47-
if conn == null_SqliteDB
48-
error("[sqlite]: A valid SqliteDB was not specified (and no valid default SqliteDB exists)")
49-
end
47+
function internal_query(conn::SqliteDB,q::String,finalize::Bool=true,stepped::Bool=true)
5048
stmt = Array(Ptr{Void},1)
51-
# unused = Array(Ptr{Void},1)
5249
if @FAILED sqlite3_prepare_v2(conn.handle,utf8(q),stmt,[C_NULL])
53-
println("$ret: $(bytestring(sqlite3_errmsg(conn.handle)))")
54-
error("[sqlite]: Error preparing query")
50+
ret = bytestring(sqlite3_errmsg(conn.handle))
51+
internal_query(conn,"COMMIT")
52+
internal_query(conn,"PRAGMA synchronous = ON")
53+
error("[sqlite]: $ret")
5554
end
5655
stmt = stmt[1]
57-
sqlite3_step(stmt) == SQLITE_DONE && return
58-
cols = sqlite3_column_count(stmt)
59-
funcs = Array(Function,cols)
60-
colnames = Array(ASCIIString,cols)
61-
resultset = Array(Any,cols)
62-
check = zeros(Int,cols)
63-
while sum(check) <= cols
64-
for i = 1:cols
65-
if check[i] == 0
66-
t = sqlite3_column_type(stmt,i-1)
67-
if t != SQLITE_NULL
68-
coltype = get(SQL2Julia,t,String) # Retrieves the Julia mapped type
69-
resultset[i] = DataArray(coltype,0)
70-
func = get(FUNCS,t,sqlite3_column_text) # Retrieves the type-correct retrieval function
71-
func == sqlite3_column_int && WORD_SIZE == 64 && (func = sqlite3_column_int64)
72-
funcs[i] = func
73-
colnames[i] = bytestring(sqlite3_column_name(stmt,i-1))
74-
check[i] = 1
75-
end
76-
end
56+
r = 0
57+
if stepped
58+
r = sqlite3_step(stmt)
59+
end
60+
if finalize
61+
sqlite3_finalize(stmt)
62+
return C_NULL, 0
63+
else
64+
return stmt, r
65+
end
66+
end
67+
function query(q::String,conn::SqliteDB=sqlitedb)
68+
conn == null_SqliteDB && error("[sqlite]: A valid SqliteDB was not specified (and no valid default SqliteDB exists)")
69+
stmt, r = Sqlite.internal_query(conn,q,false)
70+
r == SQLITE_DONE && return DataFrame("No Rows Returned")
71+
#get resultset metadata: column count, column types, and column names
72+
ncols = Sqlite.sqlite3_column_count(stmt)
73+
colnames = Array(ASCIIString,ncols)
74+
resultset = Array(Any,ncols)
75+
check = 0
76+
for i = 1:ncols
77+
colnames[i] = bytestring(Sqlite.sqlite3_column_name(stmt,i-1))
78+
t = Sqlite.sqlite3_column_type(stmt,i-1)
79+
if t == Sqlite.SQLITE3_TEXT
80+
resultset[i] = DataArray(String,0)
81+
check += 1
82+
elseif t == Sqlite.SQLITE_FLOAT
83+
resultset[i] = DataArray(Float64,0)
84+
check += 1
85+
elseif t == Sqlite.SQLITE_INTEGER
86+
resultset[i] = DataArray(WORD_SIZE == 64 ? Int64 : Int32,0)
87+
check += 1
88+
else
89+
resultset[i] = DataArray(Any,0)
7790
end
78-
sqlite3_step(stmt) != SQLITE_ROW && break
7991
end
80-
sqlite3_reset(stmt)
81-
while sqlite3_step(stmt) != SQLITE_DONE
82-
for i = 1:cols
83-
if sqlite3_column_type(stmt,i-1) == SQLITE_NULL
84-
r = NA
92+
#retrieve resultset
93+
while true
94+
for i = 1:ncols
95+
t = sqlite3_column_type(stmt,i-1)
96+
if t == SQLITE3_TEXT
97+
r = bytestring( sqlite3_column_text(stmt,i-1) )
98+
elseif t == SQLITE_FLOAT
99+
r = sqlite3_column_double(stmt,i-1)
100+
elseif t == SQLITE_INTEGER
101+
r = WORD_SIZE == 64 ? sqlite3_column_int64(stmt,i-1) : sqlite3_column_int(stmt,i-1)
85102
else
86-
r = invoke(funcs[i],(Ptr{Void},Int),stmt,i-1)
87-
if typeof(r) == Ptr{Uint8}
88-
r = bytestring(r)
89-
end
103+
r = NA
90104
end
91105
push!(resultset[i],r)
92106
end
107+
sqlite3_step(stmt) == SQLITE_DONE && break
108+
end
109+
#this is for columns we couldn't get the type for earlier (NULL in row 1); should be the exception
110+
if check != ncols
111+
nrows = length(resultset[1])
112+
for i = 1:ncols
113+
if isna(resultset[i][1])
114+
d = resultset[i]
115+
for j = 2:nrows
116+
if !isna(d[j])
117+
t = typeof(d[j])
118+
da = DataArray(t,nrows)
119+
for k = 1:nrows
120+
da[k] = d[k]
121+
end
122+
resultset[i] = da
123+
break
124+
end
125+
end
126+
end
127+
end
93128
end
94129
sqlite3_finalize(stmt)
95130
return (conn.resultset = DataFrame(resultset,Index(colnames)))
96131
end
97-
function createtable(conn::SqliteDB=sqlitedb,df::DataFrame;name::String="")
98-
if conn == null_SqliteDB
99-
error("[sqlite]: A valid SqliteDB was not specified (and no valid default SqliteDB exists)")
100-
end
101-
sqlite3_prepare_v2(conn.handle,utf8("PRAGMA synchronous = OFF"),Array(Ptr{Void},1),[C_NULL])
102-
stmt = Array(Ptr{Void},1)
103-
sqlite3_prepare_v2(conn.handle,utf8("BEGIN TRANSACTION"),stmt,[C_NULL])
104-
sqlite3_step(stmt[1])
105-
sqlite3_finalize(stmt[1])
132+
function createtable(input::TableInput,conn::SqliteDB=sqlitedb;name::String="")
133+
conn == null_SqliteDB && error("[sqlite]: A valid SqliteDB was not specified (and no valid default SqliteDB exists)")
134+
#these 2 calls are for performance
135+
internal_query(conn,"PRAGMA synchronous = OFF")
136+
internal_query(conn,"BEGIN TRANSACTION")
137+
if typeof(input) == DataFrame
138+
r = df2table(input,conn,name)
139+
else
140+
r = 0 # dlm2table(input,conn,name)
141+
end
142+
internal_query(conn,"COMMIT")
143+
internal_query(conn,"PRAGMA synchronous = ON")
144+
return r
145+
end
146+
function df2table(df::DataFrame,conn::SqliteDB,name::String)
106147
#get column names and types
107148
ncols = length(df)
108-
columns = df.colindex.names
109-
bindfuncs = ref(Function)
110-
bindtype = ref(DataType)
111-
for i = 1:ncols
112-
jultype = eltype(df[i])
113-
if jultype <: FloatingPoint
114-
sqlitetype = "REAL"
115-
push!(bindfuncs,sqlite3_bind_double)
116-
push!(bindtype,Float64)
117-
elseif jultype <: Integer
118-
sqlitetype = "INTEGER"
119-
if WORD_SIZE == 64
120-
push!(bindfuncs,sqlite3_bind_int64)
121-
push!(bindtype,Int64)
122-
else
123-
push!(bindfuncs,sqlite3_bind_int)
124-
push!(bindtype,Int32)
125-
end
126-
else
127-
sqlitetype = "TEXT"
128-
push!(bindfuncs,sqlite3_bind_text)
129-
push!(bindtype,String)
130-
end
131-
end
132-
columns = join(columns,',')
133-
#get df name for table name if not provided
134-
dfname = name
135-
if dfname == ""
136-
for sym in names(Main)
137-
if is(eval(sym),df)
138-
dfname = string(sym)
139-
end
140-
end
141-
end
142-
q = "create table $dfname ($columns)"
143-
stmt = Array(Ptr{Void},1)
144-
# unused = Array(Ptr{Void},1)
145-
if @FAILED sqlite3_prepare_v2(conn.handle,utf8(q),stmt,[C_NULL])
146-
println("$ret: $(bytestring(sqlite3_errmsg(conn.handle)))")
147-
error("[sqlite]: Error preparing 'create table' statement")
148-
end
149-
stmt = stmt[1]
150-
sqlite3_step(stmt)
149+
colnames = join(df.colindex.names,',')
150+
#get df name for table name if not provided
151+
dfname = name
152+
if dfname == ""
153+
for sym in names(Main)
154+
if is(eval(sym),df)
155+
dfname = string(sym)
156+
end
157+
end
158+
end
159+
#should we loop through column types to specify in create table statement?
160+
internal_query(conn,"create table $dfname ($colnames)")
151161
#prepare insert table with parameters for column values
152162
params = chop(repeat("?,",ncols))
153-
q = "insert into $dfname values ($params)"
154-
stmt = Array(Ptr{Void},1)
155-
# unused = Array(Ptr{Void},1)
156-
if @FAILED sqlite3_prepare_v2(conn.handle,utf8(q),stmt,[C_NULL])
157-
println("$ret: $(bytestring(sqlite3_errmsg(conn.handle)))")
158-
error("[sqlite]: Error preparing 'create table' statement")
159-
end
160-
stmt = stmt[1]
163+
stmt, r = internal_query(conn,"insert into $dfname values ($params)",false,false)
164+
sqlite3_reset(stmt)
161165
#bind, step, reset loop for inserting values
162166
for row = 1:nrow(df)
163167
for col = 1:ncols
164-
if isna(df[row,col])
165-
sqlite3_bind_null(stmt,col-1)
166-
elseif bindfuncs[col] == sqlite3_bind_text
167-
value = df[row,col]
168-
invoke(bindfuncs[col],(Ptr{Void},Int,String,Int,Ptr{Void}),stmt,col,value,length(value),C_NULL)
169-
else
170-
invoke(bindfuncs[col],(Ptr{Void},Int,bindtype[col]),stmt,col,df[row,col])
171-
end
168+
d = df[row,col]
169+
t = typeof(d)
170+
if t <: FloatingPoint
171+
Sqlite.sqlite3_bind_double(stmt,col,d)
172+
elseif t <: Integer
173+
WORD_SIZE == 64 ? sqlite3_bind_int64(stmt,col,d) : sqlite3_bind_int(stmt,col,d)
174+
elseif <: NAtype
175+
sqlite3_bind_null(stmt,col)
176+
else
177+
sqlite3_bind_text(stmt,col,string(d),length(string(d)),C_NULL)
178+
end
172179
end
173180
sqlite3_step(stmt)
174181
sqlite3_reset(stmt)
175182
end
176183
sqlite3_finalize(stmt)
177-
stmt = Array(Ptr{Void},1)
178-
sqlite3_prepare_v2(conn.handle,utf8("COMMIT"),stmt,[C_NULL])
179-
sqlite3_step(stmt[1])
180-
sqlite3_finalize(stmt[1])
181-
sqlite3_prepare_v2(conn.handle,utf8("PRAGMA synchronous = ON"),Array(Ptr{Void},1),[C_NULL])
182-
return println("Table '$dfname' created.")
184+
return
183185
end
184-
function droptable(conn::SqliteDB=sqlitedb,table::String)
185-
if conn == null_SqliteDB
186-
error("[sqlite]: A valid SqliteDB was not specified (and no valid default SqliteDB exists)")
187-
end
188-
stmt = Array(Ptr{Void},1)
189-
sqlite3_prepare_v2(conn.handle,utf8("DROP TABLE $table"),stmt,[C_NULL])
190-
sqlite3_step(stmt[1])
191-
sqlite3_finalize(stmt[1])
192-
return 0
186+
function droptable(table::String,conn::SqliteDB=sqlitedb)
187+
conn == null_SqliteDB && error("[sqlite]: A valid SqliteDB was not specified (and no valid default SqliteDB exists)")
188+
internal_query(conn,"DROP TABLE $table")
189+
internal_query(conn,"VACUUM")
190+
return
193191
end
194192
#read raw file direct to sqlite table
195193
# function csv2table()

‎src/Sqlite_consts.jl

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,11 @@ end
2323

2424
#Macros
2525
macro SUCCEEDED(func)
26-
global ret
27-
:(ret = bytestring(sqlite3_errstr(func)))
28-
:( return_code = $func; (return_code == SQLITE_OK) ? true : false )
26+
:($func == SQLITE_OK ? true : false )
2927
end
3028

3129
macro FAILED(func)
32-
global ret
33-
:(ret = bytestring(sqlite3_errstr(func)))
34-
:( return_code = $func; (return_code != SQLITE_OK) ? true : false )
30+
:($func != SQLITE_OK ? true : false )
3531
end
3632

3733
#Return codes

‎test/test.jl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ Sqlite.query("SELECT *
1616
using DataFrames
1717
df2 = DataFrame(ones(1000000,5))
1818
@time Sqlite.createtable(df2;name="test2")
19-
Sqlite.query("drop table test2")
19+
@time Sqlite.query("SELECT * FROM test2;")
20+
Sqlite.droptable("test2")
2021

2122
using DataFrames
22-
using Sqlite
2323
df3 = DataFrame(ones(1000,5))
2424
sqldf("select * from df3")

0 commit comments

Comments
 (0)