Merge pull request #96 from amellnik/esc_str

Add a new method for removeduplicates!

Author: quinnj

src/SQLite.jl

@@ -273,15 +273,30 @@ function createindex!{S<:AbstractString}(db::DB,table::AbstractString,index::Abs
     return
 end
 
-"removes duplicate rows from `table` based on the values in `cols` which may be a single column or comma-delimited list of columns"
+"removes duplicate rows from `table` based on the values in `cols` which is an array of column names"
+function removeduplicates!{T <: AbstractString}(db,table::AbstractString,cols::AbstractArray{T})
+    colsstr = ""
+    for c in cols
+       colsstr = colsstr*esc_id(c)*","
+    end
+    colsstr = chop(colsstr)
+    transaction(db) do
+        execute!(db,"DELETE FROM $(esc_id(table)) WHERE _ROWID_ NOT IN (SELECT max(_ROWID_) from $(esc_id(table)) GROUP BY $(colsstr));")
+    end   
+    execute!(db,"ANALYZE $table")
+    return
+ end
+
 function removeduplicates!(db,table::AbstractString,cols::AbstractString)
+    warn("This method is deprecated, please provide the column names as an array of column names rather than a single string.")
     transaction(db) do
         execute!(db,"DELETE FROM $(esc_id(table)) WHERE _ROWID_ NOT IN (SELECT max(_ROWID_) from $(esc_id(table)) GROUP BY $(esc_id(cols)));")
     end
     execute!(db,"ANALYZE $table")
     return
 end
 
+
 """
 `SQLite.Source` implementes the `DataStreams` framework for interacting with SQLite databases
 """

test/runtests.jl

@@ -290,3 +290,24 @@ dt2 = SQLite.query(db, "Select * from temp")
 #There might be a better way to check this
 @test dt.data[1][1].value==dt2.data[1][1].value
 @test dt.data[1][2].isnull==dt2.data[1][2].isnull
+
+#Test removeduplicates!
+db = SQLite.DB() #In case the order of tests is changed
+ints = Int64[1,1,2,2,3]
+strs = UTF8String["A", "A", "B", "C", "C"]
+nvInts = NullableArrays.NullableArray(ints)
+nvStrs = NullableArrays.NullableArray(strs)
+schema = DataStreams.Data.Schema(["ints", "strs"], [Int64, UTF8String],5)
+d = NullableArrays.NullableVector[nvInts, nvStrs]
+dt = DataStreams.Data.Table(schema, d,0)
+SQLite.drop!(db, "temp", ifexists=true)
+sink = SQLite.Sink(dt, db, "temp")
+Data.stream!(dt, sink)
+SQLite.removeduplicates!(db, "temp", ["ints","strs"]) #New format
+dt3 = SQLite.query(db, "Select * from temp")
+@test get(dt3[1,1]) == 1
+@test get(dt3[1,2]) == "A"
+@test get(dt3[2,1]) == 2
+@test get(dt3[2,2]) == "B"
+@test get(dt3[3,1]) == 2
+@test get(dt3[3,2]) == "C"