Add StdFeats transform (#237)

* Add 'StdFeats' transform

* Apply suggestions

* Update docstring

Author: eliascarv

docs/src/transforms.md

@@ -44,6 +44,12 @@ Rename
 StdNames
 ```
 
+## StdFeats
+
+```@docs
+StdFeats
+```
+
 ## Sort
 
 ```@docs

src/TableTransforms.jl

@@ -56,6 +56,7 @@ export
   Except,
   Rename,
   StdNames,
+  StdFeats,
   Sort,
   Sample,
   Filter,

src/transforms.jl

@@ -264,10 +264,12 @@ end
 # IMPLEMENTATIONS
 # ----------------
 
+include("transforms/utils.jl")
 include("transforms/select.jl")
 include("transforms/satisfies.jl")
 include("transforms/rename.jl")
 include("transforms/stdnames.jl")
+include("transforms/stdfeats.jl")
 include("transforms/sort.jl")
 include("transforms/sample.jl")
 include("transforms/filter.jl")

src/transforms/stdfeats.jl

@@ -0,0 +1,58 @@
+# ------------------------------------------------------------------
+# Licensed under the MIT License. See LICENSE in the project root.
+# ------------------------------------------------------------------
+
+"""
+    StdFeats()
+
+Standardizes the columns of the table based on scientific types:
+
+* `Continuous`: `ZScore`
+* `Categorical`: `Identity`
+* `Unknown`: `Identity`
+"""
+struct StdFeats <: StatelessFeatureTransform end
+
+isrevertible(::Type{StdFeats}) = true
+
+_stdfun(x) = _stdfun(elscitype(x), x)
+_stdfun(::Type, x) = identity, identity
+function _stdfun(::Type{Continuous}, x)
+  μ = mean(x)
+  σ = std(x, mean=μ)
+  stdfun = x -> zscore(x, μ, σ)
+  revfun = y -> revzscore(y, μ, σ)
+  stdfun, revfun
+end
+
+function applyfeat(::StdFeats, feat, prep)
+  cols = Tables.columns(feat)
+  names = Tables.columnnames(cols)
+
+  tuples = map(names) do name
+    x = Tables.getcolumn(cols, name)
+    stdfun, revfun = _stdfun(x)
+    stdfun(x), revfun
+  end
+
+  columns = first.(tuples)
+  fcache = last.(tuples)
+
+  𝒯 = (; zip(names, columns)...)
+  newfeat = 𝒯 |> Tables.materializer(feat)
+
+  newfeat, fcache
+end
+
+function revertfeat(::StdFeats, newfeat, fcache)
+  cols = Tables.columns(newfeat)
+  names = Tables.columnnames(cols)
+
+  columns = map(names, fcache) do name, revfun
+    y = Tables.getcolumn(cols, name)
+    revfun(y)
+  end
+
+  𝒯 = (; zip(names, columns)...)
+  𝒯 |> Tables.materializer(newfeat)
+end

src/transforms/utils.jl

@@ -0,0 +1,7 @@
+# ------------------------------------------------------------------
+# Licensed under the MIT License. See LICENSE in the project root.
+# ------------------------------------------------------------------
+
+zscore(x, μ, σ) = @. (x - μ) / σ
+
+revzscore(y, μ, σ) = @. σ * y + μ

src/transforms/zscore.jl

@@ -46,6 +46,6 @@ function colcache(::ZScore, x)
   (μ=μ, σ=σ)
 end
 
-colapply(::ZScore, x, c) = @. (x - c.μ) / c.σ
+colapply(::ZScore, x, c) = zscore(x, c.μ, c.σ)
 
-colrevert(::ZScore, y, c) = @. c.σ * y + c.μ
+colrevert(::ZScore, y, c) = revzscore(y, c.μ, c.σ)

test/shows.jl

@@ -85,6 +85,18 @@
     └─ spec = :upperflat"""
   end
 
+  @testset "StdFeats" begin
+    T = StdFeats()
+
+    # compact mode
+    iostr = sprint(show, T)
+    @test iostr == "StdFeats()"
+
+    # full mode
+    iostr = sprint(show, MIME("text/plain"), T)
+    @test iostr == "StdFeats transform"
+  end
+
   @testset "Sort" begin
     T = Sort([:a, :c], rev=true)
 

test/transforms.jl

@@ -3,6 +3,7 @@ transformfiles = [
   "rename.jl",
   "satisfies.jl",
   "stdnames.jl",
+  "stdfeats.jl",
   "sort.jl",
   "sample.jl",
   "filter.jl",

test/transforms/stdfeats.jl

@@ -0,0 +1,26 @@
+@testset "StdFeats" begin
+  @test isrevertible(StdFeats())
+
+  a = rand(1:10, 100)
+  b = rand(Normal(7, 10), 100)
+  c = rand('a':'z', 100)
+  d = rand(Normal(15, 2), 100)
+  e = rand(["y", "n"], 100)
+  t = Table(; a, b, c, d, e)
+
+  T = StdFeats()
+  n, c = apply(T, t)
+  @test n.a == t.a
+  @test isapprox(mean(n.b), 0; atol=1e-6)
+  @test isapprox(std(n.b), 1; atol=1e-6)
+  @test n.c == t.c
+  @test isapprox(mean(n.d), 0; atol=1e-6)
+  @test isapprox(std(n.d), 1; atol=1e-6)
+  @test n.e == t.e
+  tₒ = revert(T, n, c)
+  @test tₒ.a == t.a
+  @test tₒ.b ≈ t.b
+  @test tₒ.c == t.c
+  @test tₒ.d ≈ t.d
+  @test tₒ.e == t.e
+end