From 0fde4330a59099368becec7d2fec7e5ebc36119c Mon Sep 17 00:00:00 2001 From: ExpandingMan Date: Mon, 7 Oct 2019 14:25:37 -0400 Subject: [PATCH 1/5] switch from DataStreams to Tables --- .travis.yml | 28 +++++++++++++++----- Project.toml | 3 +-- REQUIRE | 3 --- appveyor.yml | 7 +++-- src/JDBC.jl | 2 +- src/{datastreams.jl => tables.jl} | 43 +++++++++++++------------------ test/REQUIRE | 1 - test/runtests.jl | 1 - 8 files changed, 45 insertions(+), 43 deletions(-) delete mode 100644 REQUIRE rename src/{datastreams.jl => tables.jl} (71%) delete mode 100644 test/REQUIRE diff --git a/.travis.yml b/.travis.yml index 9f3d57e..af165fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,28 @@ + +sudo: required + language: julia + +services: + - docker + os: + - osx - linux + julia: - - 0.7 - - 1.0 + - 1.0 + - 1.2 + - nightly + +matrix: + allow_failures: + - julia: nightly + notifications: email: false -# uncomment the following lines to override the default test script -#script: -# - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi -# - julia --check-bounds=yes -e 'Pkg.clone(pwd()); Pkg.build("JDBC"); Pkg.test("JDBC"; coverage=true)' + +after_success: + - julia -e 'import Pkg, JDBC; cd(joinpath(dirname(pathof(JDBC)),"..")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' + - julia -e 'import Pkg; Pkg.add("Documenter")' + - julia -e 'import JDBC; cd(joinpath(dirname(pathof(JDBC)),"..")); include(joinpath("docs", "make.jl"))' diff --git a/Project.toml b/Project.toml index 7e4a001..17267ba 100644 --- a/Project.toml +++ b/Project.toml @@ -3,12 +3,11 @@ uuid = "6042db11-3c3d-5e84-8dba-9cbf74c9ba48" version = "0.4.1" [deps] -DataStreams = "9a8bc11e-79be-5b39-94d7-1ccc349a1a85" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JavaCall = "494afd89-becb-516b-aafa-70d2670c0337" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] -DataFrames = "< 0.18.0" JavaCall = "≥ 0.7.0" julia = "≥ 0.7.0" diff --git a/REQUIRE b/REQUIRE deleted file mode 100644 index 1b111f9..0000000 --- a/REQUIRE +++ /dev/null @@ -1,3 +0,0 @@ -julia 0.7 -JavaCall 0.7.0 -DataStreams diff --git a/appveyor.yml b/appveyor.yml index 17cfb56..2f68dad 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,7 @@ environment: matrix: - - julia_version: 0.7 - - julia_version: 1 + - julia_version: 1.0 + - julia_version: 1.2 - julia_version: nightly platform: @@ -12,8 +12,7 @@ platform: # # (tests will run but not make your overall status red) matrix: allow_failures: - - julia_version: nightly - - platform: x86 + - julia_version: nightly branches: only: diff --git a/src/JDBC.jl b/src/JDBC.jl index 4fe19b9..15da065 100644 --- a/src/JDBC.jl +++ b/src/JDBC.jl @@ -640,7 +640,7 @@ export getTableMetaData, JDBCRowIterator include("interface.jl") -include("datastreams.jl") +include("tables.jl") end # module diff --git a/src/datastreams.jl b/src/tables.jl similarity index 71% rename from src/datastreams.jl rename to src/tables.jl index 1a96c86..f2baa90 100644 --- a/src/datastreams.jl +++ b/src/tables.jl @@ -1,4 +1,4 @@ -using DataStreams +using Tables const column_types = Dict( JDBC_COLTYPE_ARRAY=>Array, @@ -57,41 +57,34 @@ colname(s::Source, col::Int) = getColumnName(s.md, col) ncols(s::Source) = getColumnCount(s.md) coltypes(s::Source) = Type[coltype(s, i) for i ∈ 1:ncols(s)] -colnames(s::Source) = String[colname(s, i) for i ∈ 1:ncols(s)] +colnames(s::Source) = Symbol[Symbol(colname(s, i)) for i ∈ 1:ncols(s)] -# WARNING: this does not seem to actually work -Data.reset!(s::Source) = beforeFirst!(s.rs) +Tables.istable(::Type{<:Source}) = true +Tables.rowaccess(::Type{<:Source}) = true +Tables.rows(s::Source) = s +Tables.schema(s::Source) = Tables.Schema(colnames(s), coltypes(s)) -Data.isdone(s::Source, row::Int, col::Int) = isdone(s.rs) - -Data.schema(s::Source) = Data.Schema(coltypes(s), colnames(s), missing) - -Data.accesspattern(s::Source) = Data.Sequential - -Data.streamtype(::Type{Source}, ::Type{Data.Field}) = true -Data.streamtype(::Type{Source}, ::Type{Data.Column}) = false +Base.IteratorSize(::Type{<:Source}) = Base.SizeUnknown() +Base.eltype(s::Source) = namedtupletype(Tables.schema(s)) +namedtupletype(::Tables.Schema{names, types}) where {names, types} = NamedTuple{names, types} +namedtupletype(s::Source) = namedtupletype(Tables.schema(s)) # TODO currently jdbc_get_method is very inefficient pullfield(s::Source, col::Int) = jdbc_get_method(getColumnType(s.md, col))(s.rs, col) -# does not store current row number as a persistent state -function Data.streamfrom(s::Source, ::Type{Data.Field}, ::Type{T}, row::Int, col::Int) where T - convert(T, pullfield(s, col))::T -end -function Data.streamfrom(s::Source, ::Type{Data.Field}, ::Type{Union{T, Missing}}, - row::Int, col::Int) where T - o = pullfield(s, col) - if wasNull(s.rs) - return missing - end - convert(T, o)::T +jdbcconvert(::Type{T}, s, x) where {T} = convert(T, x) +jdbcconvert(::Type{Union{T, Missing}}, s, x) where {T} = wasNull(s.rs) ? missing : convert(T, x) + +function Base.iterate(s::Source, NT::Type{NamedTuple{names, types}}=namedtupletype(s)) where {names, types} + isdone(s.rs) && return nothing + NT(jdbcconvert(fieldtype(types, i), s, pullfield(s, i)) for i ∈ 1:fieldcount(types)), NT end -load(::Type{T}, s::Source) where {T} = Data.close!(Data.stream!(s, T)) +load(::Type{T}, s::Source) where {T} = T(Tables.materializer(T)(s)) load(::Type{T}, rs::JResultSet) where {T} = load(T, Source(rs)) load(::Type{T}, stmt::JStatement, query::AbstractString) where {T} = load(T, Source(stmt, query)) load(::Type{T}, csr::Union{JDBC.Cursor,JDBCRowIterator}) where {T} = load(T, Source(csr)) -function load(::Type{T}, csr::Cursor, q::AbstractString) where T +function load(::Type{T}, csr::Cursor, q::AbstractString) where {T} execute!(csr, q) load(T, csr) end diff --git a/test/REQUIRE b/test/REQUIRE deleted file mode 100644 index dd3ea1d..0000000 --- a/test/REQUIRE +++ /dev/null @@ -1 +0,0 @@ -DataFrames diff --git a/test/runtests.jl b/test/runtests.jl index edaadaa..645d3a7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,7 +2,6 @@ using JavaCall using JDBC using DataFrames -using DataStreams using Test using Dates import Pkg From 47a2774c257610a7f45c5cc0df8284b094ee3184 Mon Sep 17 00:00:00 2001 From: ExpandingMan Date: Mon, 7 Oct 2019 14:28:00 -0400 Subject: [PATCH 2/5] test only on 1.0 and 1.3 --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index af165fa..b25c233 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ os: julia: - 1.0 - - 1.2 + - 1.3 - nightly matrix: diff --git a/appveyor.yml b/appveyor.yml index 2f68dad..ede6c03 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,7 @@ environment: matrix: - julia_version: 1.0 - - julia_version: 1.2 + - julia_version: 1.3 - julia_version: nightly platform: From f8e488491308c5c8e4b765b562e6ec28ee4a480a Mon Sep 17 00:00:00 2001 From: ExpandingMan Date: Mon, 7 Oct 2019 14:29:46 -0400 Subject: [PATCH 3/5] bumped version number in Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 17267ba..76a7ffc 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "JDBC" uuid = "6042db11-3c3d-5e84-8dba-9cbf74c9ba48" -version = "0.4.1" +version = "0.5.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From 663e912c80d532a668d7cc8db0da7b3910e8f987 Mon Sep 17 00:00:00 2001 From: ExpandingMan Date: Mon, 7 Oct 2019 14:30:23 -0400 Subject: [PATCH 4/5] fixed version number in Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 76a7ffc..6c14d3a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "JDBC" uuid = "6042db11-3c3d-5e84-8dba-9cbf74c9ba48" -version = "0.5.0" +version = "0.4.2" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From 28cb8298e60f01075eda414ad08c463ac0ba80eb Mon Sep 17 00:00:00 2001 From: ExpandingMan Date: Mon, 7 Oct 2019 14:33:06 -0400 Subject: [PATCH 5/5] updated README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 60d8818..9de18cb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This package enables the use of Java JDBC drivers to access databases from within Julia. It uses the [JavaCall.jl](https://github.com/aviks/JavaCall.jl) package to call into Java in order to use the JDBC drivers. The API provided by this package consists essentially of two components: a "direct" (i.e. minimally wrapped) interface directly to Java JDBC and a minimal -Julian interface with support for [DataStreams.jl](https://github.com/JuliaData/DataStreams.jl). +Julian interface with support for [Tables.jl](https://github.com/JuliaData/Tables.jl). This package currently supports only Julia v0.6 and later. @@ -121,13 +121,13 @@ end close(csr) # closes Connection, can be called on Connection or Cursor ``` -#### `DataStreams` Interface and Creating `DataFrame`s +#### `Tables` Interface and Creating `DataFrame`s -JDBC includes a [DataStreams](https://github.com/JuliaData/DataStreams.jl) interface. A DataStreams `Source` object can be created from a `JDBC.Cursor` or a -`JDBCRowIterator` simply by doing e.g. `JDBC.Source(csr)`. This object implements the DataStreams `Data.Source` interface. It can be useful for retrieving metadata -with `Data.schema`. +JDBC includes a [Tables](https://github.com/JuliaData/Tables.jl) interface. A Tables +`Source` object can be created from a `JDBC.Cursor` or a `JDBCRowIterator` simply by doing +e.g. `JDBC.Source(csr)`. It can be useful for retrieving metadata with `Tables.schema`. -This is also useful for loading data from a database into an object that implements the DataStreams `Data.Sink` interface such as a `DataFrame`. For this we +This is also useful for loading data from a database into another object that implements the Tables interface. For this we provide also the convenient `JDBC.load` function. For example, you can do