Slightly nicer error messages when a variable is missing in the data table. (#235)

* Attempt to make nicer error messages when a variable has been misnamed

* attempt shorter stack trace

* cleanup comments

* split error stuff out to its own file and apply to `concrete_terms`.

* typo

* Make slightly nicer error messages when creating concrete_terms from tables.

* Update src/schema.jl

Co-authored-by: Dave Kleinschmidt <[email protected]>

* Update src/schema.jl

Co-authored-by: Dave Kleinschmidt <[email protected]>

* Update src/errormessages.jl

Co-authored-by: Dave Kleinschmidt <[email protected]>

* Update src/errormessages.jl

Co-authored-by: Dave Kleinschmidt <[email protected]>

* Update src/errormessages.jl

Co-authored-by: Dave Kleinschmidt <[email protected]>

* Update src/modelframe.jl

Co-authored-by: Dave Kleinschmidt <[email protected]>

Co-authored-by: Dave Kleinschmidt <[email protected]>

Author: grahamstark

Project.toml

@@ -7,6 +7,7 @@ DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 ShiftedArrays = "1277b4bf-5013-50f5-be3d-901d8477a67a"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"

src/StatsModels.jl

@@ -15,6 +15,8 @@ using LinearAlgebra
 
 using Tables: ColumnTable
 
+using REPL: levenshtein
+
 export
     #re-export from StatsBase:
     StatisticalModel,
@@ -67,6 +69,7 @@ export
 include("traits.jl")
 include("contrasts.jl")
 include("terms.jl")
+include( "errormessages.jl")
 include("schema.jl")
 include("temporal_terms.jl")
 include("formula.jl")

src/errormessages.jl

@@ -0,0 +1,44 @@
+
+"""
+This is borrowed from [DataFrames.jl](). 
+Return between 0 and 2 names from `colnames` closest to `name`.
+`colnames` : some iterable collection of symbols
+"""
+function fuzzymatch(colnames, name::Symbol)
+    ucname = uppercase(string(name))
+    dist = [(levenshtein(uppercase(string(x)), ucname), x) for x in colnames]
+    sort!(dist)
+    c = [count(x -> x[1] <= i, dist) for i in 0:2]
+    maxd = max(0, searchsortedlast(c, 8) - 1)
+    return [s for (d, s) in dist if d <= maxd]
+end
+
+"""
+Return a nice-ish error message if the Symbol `name` isn't a column name in `table`, otherwise a zero-length string.
+"""
+function checkcol(table, name::Symbol)
+    i = Tables.columnindex(table, name)
+    if i == 0 # if no such column
+        names = Tables.columnnames(table)
+        nearestnames = join(fuzzymatch(names, name),", " )
+        return "There isn't a variable called '$name' in your data; the nearest names appear to be: $nearestnames"
+    end
+    return ""
+end
+
+"""
+Check that each name in the given model `f` exists in the data source `t` and return a message if not. Return a zero string otherwise.
+`t` is something that implements the `Tables` interface.
+"""
+function checknamesexist(f::FormulaTerm, t)
+    if ! Tables.istable(t)
+        throw(ArgumentError( "$(typeof(t)) isn't a valid Table type" ))
+    end
+    for n in StatsModels.termvars(f)
+        msg = checkcol(t, n)
+        if msg != ""
+            return msg
+        end
+    end    
+    return ""
+end

src/modelframe.jl

@@ -68,9 +68,14 @@ end
 
 missing_omit(data::T, formula::AbstractTerm) where T<:ColumnTable =
     missing_omit(NamedTuple{tuple(termvars(formula)...)}(data))
-
 function ModelFrame(f::FormulaTerm, data::ColumnTable;
                     model::Type{M}=StatisticalModel, contrasts=Dict{Symbol,Any}()) where M
+    
+    msg = checknamesexist( f, data )
+    if msg != ""
+        throw(ArgumentError(msg))
+    end
+
     data, _ = missing_omit(data, f)
 
     sch = schema(f, data, contrasts)

src/schema.jl

@@ -166,12 +166,25 @@ julia> concrete_term(term(:a), (a = [1, 2, 3], b = [0.0, 0.5, 1.0]))
 a(continuous)
 ```
 """
-concrete_term(t::Term, d, hints::Dict{Symbol}) =
-    concrete_term(t, d, get(hints, t.sym, nothing))
-concrete_term(t::Term, dt::ColumnTable, hint) =
-    concrete_term(t, getproperty(dt, t.sym), hint)
-concrete_term(t::Term, dt::ColumnTable, hints::Dict{Symbol}) =
-    concrete_term(t, getproperty(dt, t.sym), get(hints, t.sym, nothing))
+concrete_term(t::Term, d, hints::Dict{Symbol}) = concrete_term(t, d, get(hints, t.sym, nothing))
+
+function concrete_term(t::Term, dt::ColumnTable, hint)
+    msg::String = checkcol( dt, t.sym )
+    if msg != ""
+        throw(ArgumentError(msg))
+    end
+    return concrete_term(t, getproperty(dt, t.sym), hint)
+end
+
+function concrete_term(t::Term, dt::ColumnTable, hints::Dict{Symbol})
+    msg::String = checkcol( dt, t.sym )
+    if msg != ""
+        throw(ArgumentError(msg))
+    end
+    return concrete_term(t, getproperty(dt, t.sym), get(hints, t.sym, nothing))
+end
+
+
 concrete_term(t::Term, d) = concrete_term(t, d, nothing)
 
 # if the "hint" is already an AbstractTerm, use that

test/modelframe.jl

@@ -8,6 +8,11 @@
         mf = ModelFrame(f, df)
         mm = ModelMatrix(mf)
         @test mm.assign == [1, 2, 2, 3, 4, 4]
+        
+        # `q` isn't in df  - should throw ArgumentError
+        f2 = @formula(x ~ y * q)
+        @test_throws ArgumentError ModelFrame(f2, df)
+        
     end
 
 end

test/statsmodel.jl

@@ -189,7 +189,7 @@ Base.show(io::IO, m::DummyModTwo) = println(io, m.msg)
     @test predict(m, d4) == predict(m, d)
 
     ## attempting to fit with d4 should fail since it doesn't have :y
-    @test_throws ErrorException fit(DummyMod, f, d4)
+    @test_throws ArgumentError fit(DummyMod, f, d4)
 
     ## fit with contrasts specified
     d.x2p = categorical(d.x2)

test/terms.jl

@@ -210,4 +210,10 @@ StatsModels.apply_schema(mt::MultiTerm, sch::StatsModels.Schema, Mod::Type) =
         @test "$((a, ()))" == "(a, ())"
     end
 
+    @testset "concrete_term error messages" begin
+        t = (a = [1, 2, 3], b = [0.0, 0.5, 1.0])
+        @test Tables.istable(t)
+        @test_throws ArgumentError concrete_term(term(:not_there), t )
+    end
+
 end