Merge pull request #546 from bashtage/fix-wights-absorbed

BUG: Fix weights absorbed

Author: bashtage

linearmodels/panel/model.py

@@ -1830,7 +1830,7 @@ def fit(
             if not weighted:
                 y, x, ybar = self._fast_path(low_memory=low_memory)
                 y_effects = np.array([0.0])
-                x_effects = np.zeros(x.shape[1])
+                x_effects = np.zeros(x.shape)
             else:
                 y, x, ybar, y_effects, x_effects = self._weighted_fast_path(
                     low_memory=low_memory
@@ -1883,7 +1883,7 @@ def fit(
 
                     # Adjust exog
                     self.exog = PanelData(self.exog.dataframe.iloc[:, retain])
-                    x_effects = x_effects[retain]
+                    x_effects = x_effects[:, retain]
 
         params = _lstsq(x, y, rcond=None)[0]
         nobs = self.dependent.dataframe.shape[0]

linearmodels/tests/panel/test_panel_ols.py

@@ -6,6 +6,7 @@
 import pandas as pd
 import pytest
 
+from linearmodels.datasets import wage_panel
 from linearmodels.iv.model import IV2SLS
 from linearmodels.panel.data import PanelData
 from linearmodels.panel.model import PanelOLS, PooledOLS
@@ -1343,7 +1344,7 @@ def test_singleton_removal_other_effects(data):
 def test_singleton_removal_mixed(singleton_data, other_effects):
     if other_effects == 1:
         other_effects = PanelData(singleton_data.c).dataframe.iloc[:, [0]]
-    elif other_effects == 2:
+    else:
         other_effects = singleton_data.c
     mod = PanelOLS(singleton_data.y, singleton_data.x, other_effects=other_effects)
     res_keep = mod.fit(use_lsmr=True)
@@ -1531,3 +1532,26 @@ def test_entity_into():
     ti = res.time_info
     assert ti["total"] == 4
     assert ti["min"] == 16
+
+
+@pytest.mark.parametrize("path", ["use_lsdv", "low_memory", ""])
+def test_absorbed_with_weights(path):
+    data = wage_panel.load().copy()
+    year = pd.Categorical(data.year)
+    data = data.set_index(["nr", "year"])
+    data["year"] = year
+    # and random number between 0 and 1 for weights
+    data["rand"] = np.random.rand(data.shape[0])
+    data["absorbe"] = data.groupby("nr")["union"].transform("mean")
+
+    fit_options = {}
+    if path:
+        fit_options[path] = True
+
+    with pytest.warns(AbsorbingEffectWarning, match="Variables have been"):
+        PanelOLS.from_formula(
+            "lwage ~1+absorbe + married + EntityEffects",
+            data=data,
+            weights=data["rand"],
+            drop_absorbed=True,
+        ).fit(**fit_options)

linearmodels/tests/system/test_formulas.py

@@ -50,6 +50,15 @@
     key += " : " + str(m[0].__name__)
     ids.append(key)
 
+escape_fmlas = fmlas[:2] + fmlas[3:]
+escape_params = list(product(escape_fmlas, models))
+
+escape_ids = []
+for f, m in escape_params:
+    key = "--".join([value for value in f.values()])
+    key += " : " + str(m[0].__name__)
+    escape_ids.append(key)
+
 
 def sigmoid(v):
     return np.exp(v) / (1 + np.exp(v))
@@ -62,6 +71,13 @@ def config(request):
     return fmla, model, interface
 
 
+@pytest.fixture(scope="module", params=escape_params, ids=escape_ids)
+def escape_config(request):
+    fmla, model_interace = request.param
+    model, interface = model_interace
+    return fmla, model, interface
+
+
 def test_formula(config):
     fmla, model, interface = config
     for key in fmla:
@@ -182,8 +198,8 @@ def test_parser(config):
                 assert_frame_equal(eq1[key], eq2[key])
 
 
-def test_formula_escaped(config):
-    fmla, model, interface = config
+def test_formula_escaped(escape_config):
+    fmla, model, interface = escape_config
     for key in fmla:
         if "[" in fmla[key] and model not in (IVSystemGMM, IV3SLS):
             return
@@ -205,6 +221,7 @@ def fix_formula(fmla):
     escaped_fmla = fix_formula(fmla)
     data = joined.copy()
     data.columns = cols
+
     mod = model.from_formula(escaped_fmla, data)
     pmod = pickle.loads(pickle.dumps(mod))
     mod_fmla = interface(escaped_fmla, data)

pyproject.toml

@@ -26,3 +26,13 @@ exclude = '''
 
 [tool.setuptools_scm]
 write_to = "linearmodels/_version.py"
+
+[tool.pyright]
+exclude = [
+    "**/tests/**",
+]
+
+[tool.mypy]
+exclude = [
+    "tests",
+]

requirements-dev.txt

@@ -1,6 +1,6 @@
 xarray>=0.16
 mypy>=1.3
-black[jupyter]==23.3.0
+black[jupyter]==23.7.0
 pytest>=7.3.0
 isort>=5.12
 ipython
@@ -10,6 +10,7 @@ nbconvert
 nbformat
 nbsphinx
 numpydoc
+pandas-stubs
 pytest-xdist
 pytest-cov
 seaborn

requirements-test.txt

@@ -1,9 +1,9 @@
-black[jupyter]==23.3.0
+black[jupyter]==23.7.0
 coverage
 flake8
 isort
 matplotlib
-pytest>=7.0,<7.1
+pytest>=7.3.0
 pytest-xdist
 pytest-cov
 seaborn