Updates for 1.0 (#153)

* Updates for 1.0

* Fix REQUIRE'

* Switch CI to 0.7 until upstream deps are fixed

* Switch to use BinaryProvider

* Fix library reference name

Author: quinnj

.gitignore

@@ -2,6 +2,7 @@
 *.jl.*.cov
 *.jl.mem
 /deps/builds
+/deps/build.log
 /deps/usr
 CMakeLists.txt.user
 /deps/deps.jl

.travis.yml

@@ -7,13 +7,13 @@ os:
   - osx
 
 julia:
-  - 0.6
+  - 0.7
   - nightly
 
 notifications:
   email: false
 
 after_success:
-  - julia -e 'cd(Pkg.dir("SQLite")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())'
-  - julia -e 'Pkg.add("Documenter")'
-  - julia -e 'cd(Pkg.dir("SQLite")); include(joinpath("docs", "make.jl"))'
+  - julia -e 'using Pkg; cd(Pkg.dir("SQLite")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())'
+  - julia -e 'using Pkg; Pkg.add("Documenter")'
+  - julia -e 'using Pkg; cd(Pkg.dir("SQLite")); include(joinpath("docs", "make.jl"))'

REQUIRE

@@ -1,8 +1,7 @@
-julia 0.6
-BinDeps
-@osx Homebrew
-@windows WinRPM
+julia 0.7
 DataStreams 0.3.0
 DataFrames 0.11.0
 WeakRefStrings 0.4.0
 LegacyStrings
+Missings
+BinaryProvider

appveyor.yml

@@ -1,9 +1,17 @@
 environment:
   matrix:
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
-  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
-  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
+  - julia_version: 0.7
+  - julia_version: nightly
+
+platform:
+  - x86 # 32-bit
+  - x64 # 64-bit
+
+# # Uncomment the following lines to allow failures on nightly julia
+# # (tests will run but not make your overall status red)
+# matrix:
+#   allow_failures:
+#   - julia_version: nightly
 
 branches:
   only:
@@ -17,25 +25,18 @@ notifications:
     on_build_status_changed: false
 
 install:
-  - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
-# if there's a newer build queued for the same PR, cancel this one
-  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
-        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
-        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
-        throw "There are newer queued builds for this pull request, failing early." }
-# Download most recent Julia Windows binary
-  - ps: (new-object net.webclient).DownloadFile(
-        $env:JULIA_URL,
-        "C:\projects\julia-binary.exe")
-# Run installer silently, output to C:\projects\julia
-  - C:\projects\julia-binary.exe /S /D=C:\projects\julia
+  - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1"))
 
 build_script:
-# Need to convert from shallow to complete for Pkg.clone to work
-  - IF EXIST .git\shallow (git fetch --unshallow)
-  - C:\projects\julia\bin\julia -e "versioninfo();
-      Pkg.clone(pwd(), \"SQLite\"); Pkg.build(\"SQLite\")"
-  - where *sqlite3*
+  - echo "%JL_BUILD_SCRIPT%"
+  - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%"
 
 test_script:
-  - C:\projects\julia\bin\julia -e "Pkg.test(\"SQLite\")"
+  - echo "%JL_TEST_SCRIPT%"
+  - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%"
+
+# # Uncomment to support code coverage upload. Should only be enabled for packages
+# # which would have coverage gaps without running on Windows
+# on_success:
+#   - echo "%JL_CODECOV_SCRIPT%"
+#   - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%"

deps/build.jl

@@ -1,19 +1,46 @@
-using BinDeps
-using Compat
+using BinaryProvider # requires BinaryProvider 0.3.0 or later
 
-@BinDeps.setup
+# Parse some basic command-line arguments
+const verbose = "--verbose" in ARGS
+const prefix = Prefix(get([a for a in ARGS if a != "--verbose"], 1, joinpath(@__DIR__, "usr")))
+products = [
+    LibraryProduct(prefix, ["libsqlite3"], :libsqlite),
+]
 
-sqlite3_lib = library_dependency("sqlite3_lib", aliases = ["sqlite3","sqlite3-64","libsqlite3","libsqlite3-0"])
+# Download binaries from hosted location
+bin_prefix = "https://github.com/JuliaDatabases/SQLiteBuilder/releases/download/v0.9.0"
 
-if is_windows()
-    using WinRPM
-    provides(WinRPM.RPM, "libsqlite3-0",sqlite3_lib, os = :Windows)
-elseif is_apple()
-    using Homebrew
-    provides(Homebrew.HB, "sqlite", sqlite3_lib, os = :Darwin)
-elseif is_linux()
-    provides(Yum, Dict("sqlite3-devel" => sqlite3_lib))
-    provides(AptGet, Dict("sqlite3" => sqlite3_lib))
+# Listing of files generated by BinaryBuilder:
+download_info = Dict(
+    Linux(:aarch64, :glibc) => ("$bin_prefix/SQLiteBuilder.v0.1.0.aarch64-linux-gnu.tar.gz", "a72759ea789697475bc268631cd45723967403dcbd630e7090310acf3fe4e213"),
+    Linux(:aarch64, :musl) => ("$bin_prefix/SQLiteBuilder.v0.1.0.aarch64-linux-musl.tar.gz", "865ad1af7907d6fad61a217cc347e88c31a1e14fde1e8b1842be30f40caadb2f"),
+    Linux(:armv7l, :glibc, :eabihf) => ("$bin_prefix/SQLiteBuilder.v0.1.0.arm-linux-gnueabihf.tar.gz", "213587798e00e3d55600bfd8e9224ec7d25051df36443007c22d2616744d82a6"),
+    Linux(:armv7l, :musl, :eabihf) => ("$bin_prefix/SQLiteBuilder.v0.1.0.arm-linux-musleabihf.tar.gz", "28aec3c8eba103d9680dcd9a869189a7a49b16fb26df0cf696025050cfa94b7c"),
+    Linux(:i686, :glibc) => ("$bin_prefix/SQLiteBuilder.v0.1.0.i686-linux-gnu.tar.gz", "7b65157e359c44b2a57f12d69badb33dfe49af91a1550a2a4ce73fa39437384c"),
+    Linux(:i686, :musl) => ("$bin_prefix/SQLiteBuilder.v0.1.0.i686-linux-musl.tar.gz", "6bd07d37a3471faae3eccbdda15331e2fef3617dc6517cdd5957ff0f232bfc66"),
+    Windows(:i686) => ("$bin_prefix/SQLiteBuilder.v0.1.0.i686-w64-mingw32.tar.gz", "152e7c7933c3b1fcda56177a13d37e130d6dc33dea8b4af18f462ef8bc415ba4"),
+    Linux(:powerpc64le, :glibc) => ("$bin_prefix/SQLiteBuilder.v0.1.0.powerpc64le-linux-gnu.tar.gz", "994abe7690163a94b58f589808380ac14e594d2a088d519ff1faf56365f20d10"),
+    MacOS(:x86_64) => ("$bin_prefix/SQLiteBuilder.v0.1.0.x86_64-apple-darwin14.tar.gz", "acc1d370715be7f4a765e517ae8b4a9fd0ee9ffb08ffa6c81118aa5dc0db3eca"),
+    Linux(:x86_64, :glibc) => ("$bin_prefix/SQLiteBuilder.v0.1.0.x86_64-linux-gnu.tar.gz", "45ba487144ff8e11271130ae9abbb3af5722739608dee6ef44be0d0c1205fa9a"),
+    Linux(:x86_64, :musl) => ("$bin_prefix/SQLiteBuilder.v0.1.0.x86_64-linux-musl.tar.gz", "998b995bc9a2d66a99de2e85a210af865e4993af51a295ce4dd88c26136a4951"),
+    FreeBSD(:x86_64) => ("$bin_prefix/SQLiteBuilder.v0.1.0.x86_64-unknown-freebsd11.1.tar.gz", "060d7a0512d16eed4b97787eaa70cb596695c300a3859a57b72b37c4e2be54e0"),
+    Windows(:x86_64) => ("$bin_prefix/SQLiteBuilder.v0.1.0.x86_64-w64-mingw32.tar.gz", "dd323668391ede718e39994819cd475568aadc12563968dfd14900e338b78841"),
+)
+
+# Install unsatisfied or updated dependencies:
+unsatisfied = any(!satisfied(p; verbose=verbose) for p in products)
+if haskey(download_info, platform_key())
+    url, tarball_hash = download_info[platform_key()]
+    if unsatisfied || !isinstalled(url, tarball_hash; prefix=prefix)
+        # Download and install binaries
+        install(url, tarball_hash; prefix=prefix, force=true, verbose=verbose)
+    end
+elseif unsatisfied
+    # If we don't have a BinaryProvider-compatible .tar.gz to download, complain.
+    # Alternatively, you could attempt to install from a separate provider,
+    # build from source or something even more ambitious here.
+    error("Your platform $(triplet(platform_key())) is not supported by this package!")
 end
 
-@BinDeps.install Dict(:sqlite3_lib => :sqlite3_lib)
+# Write out a deps.jl file that will contain mappings for our products
+write_deps_file(joinpath(@__DIR__, "deps.jl"), products)

src/SQLite.jl

@@ -1,13 +1,9 @@
-__precompile__(true)
 module SQLite
 
-using Missings, DataStreams, WeakRefStrings, LegacyStrings, DataFrames
+using Random, Missings, DataStreams, WeakRefStrings, LegacyStrings, DataFrames
 import LegacyStrings: UTF16String
 
-if VERSION < v"0.7.0-DEV.2562"
-    import Base: finalizer
-    finalizer(f::Function, o) = finalizer(o, f)
-end
+import Serialization
 
 export Data, DataFrame
 
@@ -35,11 +31,11 @@ Constructors:
 """
 mutable struct DB
     file::String
-    handle::Ptr{Void}
+    handle::Ptr{Cvoid}
     changes::Int
 
     function DB(f::AbstractString)
-        handle = Ref{Ptr{Void}}()
+        handle = Ref{Ptr{Cvoid}}()
         f = isempty(f) ? f : expanduser(f)
         if @OK sqliteopen(f, handle)
             db = new(f, handle[], 0)
@@ -66,11 +62,11 @@ Base.show(io::IO, db::SQLite.DB) = print(io, string("SQLite.DB(", db.file == ":m
 """
 mutable struct Stmt
     db::DB
-    handle::Ptr{Void}
+    handle::Ptr{Cvoid}
 
     function Stmt(db::DB,sql::AbstractString)
-        handle = Ref{Ptr{Void}}()
-        sqliteprepare(db, sql, handle, Ref{Ptr{Void}}())
+        handle = Ref{Ptr{Cvoid}}()
+        sqliteprepare(db, sql, handle, Ref{Ptr{Cvoid}}())
         stmt = new(db, handle[])
         finalizer(_close, stmt)
         return stmt
@@ -93,7 +89,7 @@ export @sr_str, @register, register
 
 clears any bound values to a prepared SQL statement.
 """
-function Base.clear!(stmt::Stmt)
+function clear!(stmt::Stmt)
     sqlite3_clear_bindings(stmt.handle)
     return
 end
@@ -164,7 +160,7 @@ bind!(stmt::Stmt, i::Int, val::Vector{UInt8})  = (sqlite3_bind_blob(stmt.handle,
 # Fallback is BLOB and defaults to serializing the julia value
 
 # internal wrapper mutable struct to, in-effect, mark something which has been serialized
-struct Serialization
+struct Serialized
     object
 end
 
@@ -174,8 +170,8 @@ function sqlserialize(x)
     # deserialize will sometimes return a random object when called on an array
     # which has not been previously serialized, we can use this mutable struct to check
     # that the array has been serialized
-    s = Serialization(x)
-    serialize(GLOBAL_BUF, s)
+    s = Serialized(x)
+    Serialization.serialize(GLOBAL_BUF, s)
     return take!(GLOBAL_BUF)
 end
 # fallback method to bind arbitrary julia `val` to the parameter at index `i` (object is serialized)
@@ -186,18 +182,14 @@ struct SerializeError <: Exception
 end
 
 # magic bytes that indicate that a value is in fact a serialized julia value, instead of just a byte vector
-# const SERIALIZATION = UInt8[0x11,0x01,0x02,0x0d,0x53,0x65,0x72,0x69,0x61,0x6c,0x69,0x7a,0x61,0x74,0x69,0x6f,0x6e,0x23]
-if VERSION < v"0.7.0-DEV.1833"
-    const SERIALIZATION = UInt8[0x34,0x10,0x01,0x0d,0x53,0x65,0x72,0x69,0x61,0x6c,0x69,0x7a,0x61,0x74,0x69,0x6f,0x6e,0x1f]
-else
-    const SERIALIZATION = UInt8[0x37,0x4a,0x4c,0x07,0x04,0x00,0x00,0x00,0x34,0x10,0x01,0x0d,0x53,0x65,0x72,0x69,0x61,0x6c]
-end
+const SERIALIZATION = UInt8[0x37,0x4a,0x4c,0x07,0x04,0x00,0x00,0x00,0x34,0x10,0x01,0x0a,0x53,0x65,0x72,0x69,0x61,0x6c]
+
 function sqldeserialize(r)
     ret = ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt),
             SERIALIZATION, r, min(18, length(r)))
     if ret == 0
         try
-            v = deserialize(IOBuffer(r))
+            v = Serialization.deserialize(IOBuffer(r))
             return v.object
         catch e
             throw(SerializeError("Error deserializing non-primitive value out of database; this is probably due to using SQLite.jl with a different Julia version than was used to originally serialize the database values. The same Julia version that was used to serialize should be used to extract the database values into a different format (csv file, feather file, etc.) and then loaded back into the sqlite database with the current Julia version."))
@@ -211,9 +203,9 @@ end
  #int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
 
 """
-`SQLite.execute!(stmt::SQLite.Stmt)` => `Void`
+`SQLite.execute!(stmt::SQLite.Stmt)` => `Cvoid`
 
-`SQLite.execute!(db::DB, sql::String)` => `Void`
+`SQLite.execute!(db::DB, sql::String)` => `Cvoid`
 
 
 Execute a prepared SQLite statement, not checking for or returning any results.
@@ -243,7 +235,7 @@ A vector of identifiers will be separated by commas.
 """
 function esc_id end
 
-esc_id(x::AbstractString) = "\"" * replace(x,"\"","\"\"") * "\""
+esc_id(x::AbstractString) = "\"" * replace(x, "\""=>"\"\"") * "\""
 esc_id(X::AbstractVector{S}) where {S <: AbstractString} = join(map(esc_id, X), ',')
 
 
@@ -274,7 +266,7 @@ function transaction(db, mode="DEFERRED")
 end
 function transaction(f::Function, db)
     # generate a random name for the savepoint
-    name = string("SQLITE", randstring(10))
+    name = string("SQLITE", Random.randstring(10))
     execute!(db, "PRAGMA synchronous = OFF;")
     transaction(db, name)
     try

src/Sink.jl

@@ -44,13 +44,13 @@ Data.weakrefstrings(::Type{Sink}) = true
 function Sink(sch::Data.Schema, T, append::Bool, db::DB, name::AbstractString; reference::Vector{UInt8}=UInt8[], kwargs...)
     sink = Sink(db, name, sch; append=append, kwargs...)
     execute!(sink.db, "PRAGMA synchronous = OFF;")
-    sink.transaction = string("SQLITE",randstring(10))
+    sink.transaction = string("SQLITE", Random.randstring(10))
     transaction(sink.db, sink.transaction)
     return sink
 end
 function Sink(sink, sch::Data.Schema, T, append::Bool; reference::Vector{UInt8}=UInt8[])
     execute!(sink.db, "PRAGMA synchronous = OFF;")
-    sink.transaction = string("SQLITE", randstring(10))
+    sink.transaction = string("SQLITE", Random.randstring(10))
     transaction(sink.db, sink.transaction)
     !append && execute!(sink.db, "delete from $(esc_id(sink.tablename))")
     return sink

src/Source.jl

@@ -14,8 +14,8 @@ function Source(db::DB, sql::AbstractString, values=[]; rows::Union{Int, Missing
     bind!(stmt, values)
     status = SQLite.execute!(stmt)
     cols = SQLite.sqlite3_column_count(stmt.handle)
-    header = Vector{String}(cols)
-    types = Vector{Type}(cols)
+    header = Vector{String}(undef, cols)
+    types = Vector{Type}(undef, cols)
     for i = 1:cols
         header[i] = unsafe_string(SQLite.sqlite3_column_name(stmt.handle, i))
         if nullable
@@ -60,7 +60,7 @@ function sqlitevalue(::Type{T}, handle, col) where {T}
     blob = convert(Ptr{UInt8}, sqlite3_column_blob(handle, col))
     b = sqlite3_column_bytes(handle, col)
     buf = zeros(UInt8, b) # global const?
-    unsafe_copy!(pointer(buf), blob, b)
+    unsafe_copyto!(pointer(buf), blob, b)
     r = sqldeserialize(buf)::T
     return r
 end
@@ -85,7 +85,7 @@ function Data.streamfrom(source::SQLite.Source, ::Type{Data.Field}, ::Type{Union
         val = missing
     else
         TT = SQLite.juliatype(t) # native SQLite Int, Float, and Text types
-        val = SQLite.sqlitevalue(ifelse(TT === Any && !isbits(T), T, TT), handle, col)
+        val = SQLite.sqlitevalue(ifelse(TT === Any && !isbitstype(T), T, TT), handle, col)
     end
     col == source.schema.cols && (source.status = sqlite3_step(handle))
     return val::Union{T, Missing}
@@ -97,7 +97,7 @@ function Data.streamfrom(source::SQLite.Source, ::Type{Data.Field}, ::Type{T}, r
         throw(Data.NullException("encountered missing value in non-missing typed column: row = $row, col = $col"))
     else
         TT = SQLite.juliatype(t) # native SQLite Int, Float, and Text types
-        val::T = sqlitevalue(ifelse(TT === Any && !isbits(T), T, TT), handle, col)
+        val::T = sqlitevalue(ifelse(TT === Any && !isbitstype(T), T, TT), handle, col)
     end
     col == source.schema.cols && (source.status = sqlite3_step(handle))
     return val