Support DataFrames 0.21 (#58)

Note DataFrames 0.18 do not support the new ! syntax.

Author: omus

Project.toml

@@ -13,7 +13,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [compat]
-DataFrames = "0.17, 0.18, 0.19, 0.20"
+DataFrames = "0.19, 0.20, 0.21"
 LearnBase = "0.2, 0.3"
 MLDataPattern = "0.5"
 MLLabelUtils = "0.4, 0.5"

src/MLDataUtils.jl

@@ -4,7 +4,7 @@ using StatsBase
 using LearnBase
 using MLLabelUtils
 using MLDataPattern
-using DataFrames
+using DataFrames: DataFrames, AbstractDataFrame, DataFrameRow, eachcol
 
 using LearnBase: ObsDimension
 import LearnBase: nobs, getobs, getobs!, datasubset, default_obsdim

src/datapattern.jl

@@ -1,7 +1,5 @@
 _throw_table_error() = throw(ArgumentError("Please specify the column that contains the targets explicitly, or provide a target-extraction-function as first parameter. see parameter 'f' in ?targets."))
 
-import DataFrames: DataFrames, AbstractDataFrame, SubDataFrame
-
 # required data container interface
 LearnBase.nobs(dt::AbstractDataFrame) = DataFrames.nrow(dt)
 LearnBase.getobs(dt::AbstractDataFrame, idx) = dt[idx,:]

src/feature_scaling.jl

@@ -92,9 +92,9 @@ end
 function center!(D::AbstractDataFrame)
     μ_vec = Float64[]
 
-    flt = Bool[T <: Real for T in eltypes(D)]
-    for colname in names(D)[flt]
-        μ = mean(D[colname])
+    flt = Bool[T <: Real for T in eltype.(eachcol(D))]
+    for colname in propertynames(D)[flt]
+        μ = mean(D[!, colname])
         center!(D, colname, μ)
         push!(μ_vec, μ)
     end
@@ -104,8 +104,8 @@ end
 function center!(D::AbstractDataFrame, colnames::AbstractVector{Symbol})
     μ_vec = Float64[]
     for colname in colnames
-        if eltype(D[colname]) <: Real
-            μ = mean(D[colname])
+        if eltype(D[!, colname]) <: Real
+            μ = mean(D[!, colname])
             if ismissing(μ)
                 @warn("Column \"$colname\" contains missing values, skipping rescaling of this column!")
                 continue
@@ -121,7 +121,7 @@ end
 
 function center!(D::AbstractDataFrame, colnames::AbstractVector{Symbol}, μ::AbstractVector)
     for (icol, colname) in enumerate(colnames)
-        if eltype(D[colname]) <: Real
+        if eltype(D[!, colname]) <: Real
             center!(D, colname, μ[icol])
         else
             @warn("Skipping \"$colname\", centering only valid for columns of type T <: Real.")
@@ -131,15 +131,15 @@ function center!(D::AbstractDataFrame, colnames::AbstractVector{Symbol}, μ::Abs
 end
 
 function center!(D::AbstractDataFrame, colname::Symbol, μ)
-    if any(ismissing, D[colname])
+    if any(ismissing, D[!, colname])
         @warn("Column \"$colname\" contains missing values, skipping centering on this column!")
     else
-        newcol::Vector{Float64} = convert(Vector{Float64}, D[colname])
+        newcol::Vector{Float64} = convert(Vector{Float64}, D[!, colname])
         nobs = length(newcol)
         @inbounds for i in eachindex(newcol)
             newcol[i] -= μ
         end
-        D[colname] = newcol
+        D[!, colname] = newcol
     end
     μ
 end
@@ -243,10 +243,10 @@ function rescale!(D::AbstractDataFrame)
     μ_vec = Float64[]
     σ_vec = Float64[]
 
-    flt = Bool[T <: Real for T in eltypes(D)]
-    for colname in names(D)[flt]
-        μ = mean(D[colname])
-        σ = std(D[colname])
+    flt = Bool[T <: Real for T in eltype.(eachcol(D))]
+    for colname in propertynames(D)[flt]
+        μ = mean(D[!, colname])
+        σ = std(D[!, colname])
         rescale!(D, colname, μ, σ)
         push!(μ_vec, μ)
         push!(σ_vec, σ)
@@ -258,9 +258,9 @@ function rescale!(D::AbstractDataFrame, colnames::Vector{Symbol})
     μ_vec = Float64[]
     σ_vec = Float64[]
     for colname in colnames
-        if eltype(D[colname]) <: Real
-            μ = mean(D[colname])
-            σ = std(D[colname])
+        if eltype(D[!, colname]) <: Real
+            μ = mean(D[!, colname])
+            σ = std(D[!, colname])
             if ismissing(μ)
                 @warn("Column \"$colname\" contains missing values, skipping rescaling of this column!")
                 continue
@@ -277,7 +277,7 @@ end
 
 function rescale!(D::AbstractDataFrame, colnames::Vector{Symbol}, μ::AbstractVector, σ::AbstractVector)
     for (icol, colname) in enumerate(colnames)
-        if eltype(D[colname]) <: Real
+        if eltype(D[!, colname]) <: Real
             rescale!(D, colname, μ[icol], σ[icol])
         else
             @warn("Skipping \"$colname\", rescaling only valid for columns of type T <: Real.")
@@ -287,16 +287,16 @@ function rescale!(D::AbstractDataFrame, colnames::Vector{Symbol}, μ::AbstractVe
 end
 
 function rescale!(D::AbstractDataFrame, colname::Symbol, μ, σ)
-    if any(ismissing, D[colname])
+    if any(ismissing, D[!, colname])
         @warn("Column \"$colname\" contains missing values, skipping rescaling of this column!")
     else
         σ_div = σ == 0 ? one(σ) : σ
-        newcol::Vector{Float64} = convert(Vector{Float64}, D[colname])
+        newcol::Vector{Float64} = convert(Vector{Float64}, D[!, colname])
         nobs = length(newcol)
         @inbounds for i in eachindex(newcol)
             newcol[i] = (newcol[i] - μ) / σ_div
         end
-        D[colname] = newcol
+        D[!, colname] = newcol
     end
     μ, σ
 end

test/tst_feature_scaling.jl

@@ -2,7 +2,7 @@ e_x, _ = noisy_sin(50; noise = 0.)
 e_X = expand_poly(e_x, degree = 5)
 df = DataFrame(A=rand(10), B=collect(1:10), C=[string(x) for x in 1:10])
 df_na = deepcopy(df)
-df_na[:A] = allowmissing(df_na[:A])
+df_na[!, :A] = allowmissing(df_na[!, :A])
 df_na[1, :A] = missing
 
 @testset "Test expand_poly" begin
@@ -65,34 +65,34 @@ end
 
     # Center DataFrame
     D = copy(df)
-    mu_check = [mean(D[colname]) for colname in names(D)[1:2]]
+    mu_check = [mean(D[!, colname]) for colname in names(D)[1:2]]
     mu = center!(D)
     @test length(mu) == 2
     @test abs(sum(mu .- mu_check)) <= 10e-10
 
     D = copy(df)
-    mu_check = [mean(D[colname]) for colname in names(D)[1:2]]
+    mu_check = [mean(D[!, colname]) for colname in names(D)[1:2]]
     mu = center!(D, [:A, :B])
     @test abs(sum(mu .- mu_check)) <= 10e-10
 
     D = copy(df)
-    mu_check = [mean(D[colname]) for colname in names(D)[1:2]]
+    mu_check = [mean(D[!, colname]) for colname in names(D)[1:2]]
     mu = center!(D, [:A, :B], mu_check)
-    @test abs(sum([mean(D[colname]) for colname in names(D)[1:2]])) <= 10e-10
+    @test abs(sum([mean(D[!, colname]) for colname in names(D)[1:2]])) <= 10e-10
 
     # skip columns that contain missing values
     D = copy(df_na)
     mu = center!(D, [:A, :B])
     @test ismissing(D[1, :A])
     @test all(D[2:end, :A] .== df_na[2:end, :A])
-    @test abs(mean(D[:B])) < 10e-10
+    @test abs(mean(D[!, :B])) < 10e-10
 
     D = copy(df_na)
-    mu_check = [mean(D[colname]) for colname in names(D)[1:2]]
+    mu_check = [mean(D[!, colname]) for colname in names(D)[1:2]]
     mu = center!(D, [:A, :B], mu_check)
     @test ismissing(D[1, :A])
     @test all(D[2:end, :A] .== df_na[2:end, :A])
-    @test abs(mean(D[:B])) < 10e-10
+    @test abs(mean(D[!, :B])) < 10e-10
 
     # Rescale Vector
     xa = copy(e_x)
@@ -146,33 +146,33 @@ end
 
     D = copy(df)
     mu, sigma = rescale!(D)
-    @test abs(sum([mean(D[colname]) for colname in names(D)[1:2]])) <= 10e-10
-    @test mean([std(D[colname]) for colname in names(D)[1:2]]) - 1 <= 10e-10
+    @test abs(sum([mean(D[!, colname]) for colname in names(D)[1:2]])) <= 10e-10
+    @test mean([std(D[!, colname]) for colname in names(D)[1:2]]) - 1 <= 10e-10
 
     D = copy(df)
     mu, sigma = rescale!(D, [:A, :B])
-    @test abs(sum([mean(D[colname]) for colname in names(D)[1:2]])) <= 10e-10
-    @test mean([std(D[colname]) for colname in names(D)[1:2]]) - 1 <= 10e-10
+    @test abs(sum([mean(D[!, colname]) for colname in names(D)[1:2]])) <= 10e-10
+    @test mean([std(D[!, colname]) for colname in names(D)[1:2]]) - 1 <= 10e-10
 
     D = copy(df)
-    mu_check = [mean(D[colname]) for colname in names(D)[1:2]]
-    sigma_check = [std(D[colname]) for colname in names(D)[1:2]]
+    mu_check = [mean(D[!, colname]) for colname in names(D)[1:2]]
+    sigma_check = [std(D[!, colname]) for colname in names(D)[1:2]]
     mu, sigma = rescale!(D, [:A, :B], mu_check, sigma_check)
-    @test abs(sum([mean(D[colname]) for colname in names(D)[1:2]])) <= 10e-10
-    @test mean([std(D[colname]) for colname in names(D)[1:2]]) - 1 <= 10e-10
+    @test abs(sum([mean(D[!, colname]) for colname in names(D)[1:2]])) <= 10e-10
+    @test mean([std(D[!, colname]) for colname in names(D)[1:2]]) - 1 <= 10e-10
 
     # skip columns that contain missing values
     D = copy(df_na)
     mu, sigma = rescale!(D, [:A, :B])
     @test ismissing(D[1, :A])
     @test all(D[2:end, :A] .== df_na[2:end, :A])
-    @test abs(mean(D[:B])) < 10e-10
-    @test abs(std(D[:B])) - 1 < 10e-10
+    @test abs(mean(D[!, :B])) < 10e-10
+    @test abs(std(D[!, :B])) - 1 < 10e-10
 
     D = copy(df_na)
-    mu_check = [mean(D[colname]) for colname in names(D)[1:2]]
+    mu_check = [mean(D[!, colname]) for colname in names(D)[1:2]]
     if VERSION >= v"0.7.0-DEV.2035"
-        sigma_check = [std(D[colname]) for colname in names(D)[1:2]]
+        sigma_check = [std(D[!, colname]) for colname in names(D)[1:2]]
         mu, sigma = rescale!(D, [:A, :B], mu_check, sigma_check)
     end
     #= @test ismissing(D[1, :A]) =#