fix up nesting (slash) syntax and add tests for fixefs too (#257)

* fix nesting syntax in formula

* add tests and use FullRank schema to make sure promotion works

Author: kleinschmidt

src/randomeffectsterm.jl

@@ -27,8 +27,14 @@ function StatsModels.apply_schema(
     if length(t.args_parsed) ≠ 2
         throw(ArgumentError("malformed nesting term: $t (Exactly two arguments required"))
     end
-    first, second = apply_schema.(t.args_parsed, Ref(sch.schema), Mod)
-    return first + first & second
+
+    first, second = apply_schema.(t.args_parsed, Ref(sch), Mod)
+    
+    if !(typeof(first) <: CategoricalTerm)
+        throw(ArgumentError("nesting terms requires categorical grouping term, got $first.  Manually specify $first as `CategoricalTerm` in hints/contrasts"))
+    end
+
+    return first + fulldummy(first) & second
 end
 
 RandomEffectsTerm(lhs, rhs::NTuple{2,AbstractTerm}) =

test/FactorReTerm.jl

@@ -175,5 +175,20 @@ end
 
         @test_broken fit(MixedModel, @formula(Y ~ 1 + (1|H/c)), dat[:Pastes])
 
+        # in fixed effects:
+        d2 = (a = rand(20), b = repeat([:X, :Y], outer=10), c = repeat([:S,:T],outer=10))
+        f2 = apply_schema(@formula(0 ~ 1 + b/a), schema(d2), MixedModel)
+        @test modelcols(f2.rhs, d2) == [ones(20) d2.b .== :Y (d2.b .== :X).*d2.a (d2.b .== :Y).*d2.a]
+        @test coefnames(f2.rhs) == ["(Intercept)", "b: Y", "b: X & a", "b: Y & a"]
+
+        f3 = apply_schema(@formula(0 ~ 0 + b/a), schema(d2), MixedModel)
+        @test modelcols(f3.rhs, d2) == [d2.b .== :X d2.b .== :Y (d2.b .== :X).*d2.a (d2.b .== :Y).*d2.a]
+        @test coefnames(f3.rhs) == ["b: X", "b: Y", "b: X & a", "b: Y & a"]
+
+        # errors for continuous grouping
+        @test_throws ArgumentError apply_schema(@formula(0 ~ 1 + a/b), schema(d2), MixedModel)
+
+        # errors for too much nesting
+        @test_throws ArgumentError apply_schema(@formula(0 ~ 1 + b/c/a), schema(d2), MixedModel)
     end
 end