Fix continuous GLMMEncoder

Author: janmotl

category_encoders/glmm.py

@@ -277,7 +277,7 @@ def _train(self, X, y):
                         estimate = pd.Series(md.vc_mean, index=index_names)
                     else:
                         # Regression, returns (regularized) mean deviation of the observation's category from the global mean
-                        md = smf.mixedlm('target ~ 1', data, groups=data[col]).fit()
+                        md = smf.mixedlm('target ~ 1', data, groups=data['feature']).fit()
                         tmp = dict()
                         for key, value in md.random_effects.items():
                             tmp[key] = value[0]

tests/test_glmm.py

@@ -0,0 +1,21 @@
+import numpy as np
+from unittest2 import TestCase
+import category_encoders as encoders
+import tests.helpers as th
+
+# data definitions
+X = th.create_dataset(n_rows=100)
+np_y = np.random.randn(100) > 0.5
+
+class TestGLMMEncoder(TestCase):
+    def test_continuous(self):
+        cols = ['unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical', 'categorical_int']
+        enc = encoders.GLMMEncoder(cols=cols, binomial_target=False)
+        enc.fit(X, np_y)
+        th.verify_numeric(enc.transform(X))
+
+    def test_binary(self):
+        cols = ['unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical', 'categorical_int']
+        enc = encoders.GLMMEncoder(cols=cols, binomial_target=True)
+        enc.fit(X, np_y)
+        th.verify_numeric(enc.transform(X))