Handle string columns of type pandas.StringDType properly (#478)

* Treat new-style pandas string columns like objects

* Make tests work with pandas 2.3

* Formulaic can't do it yet

* Changelog

* Not yet

* Release today

* Use a non-retired OS

* Maybe this?

* Move include path to D:

* Revert "Maybe this?"

This reverts commit c149990.

---------

Co-authored-by: Jan Tilly <[email protected]>

Author: stanmart

.github/workflows/build-wheels.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, ubuntu-22.04-arm, windows-2019, macos-13, macos-14]
+        os: [ubuntu-22.04, ubuntu-22.04-arm, windows-2022, macos-13, macos-14]
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

CHANGELOG.rst

@@ -7,12 +7,13 @@
 Changelog
 =========
 
-4.1.2 - UNRELEASED
+4.1.2 - 2025-07-17
 ------------------
 
 **Bug fix:**
 
 - Fixed a bug which caused issues when constructing tabmat matrices from existing ``ModelSpec``\s when they contained categorical columns with all levels dropped.
+- We can now treat dedicated pandas string series - which are the defaults for strings since pandas 2.3 - as categoricals.
 
 
 4.1.1 - 2025-01-30

pyproject.toml

@@ -74,7 +74,7 @@ before-all = [
 ]
 
 [tool.cibuildwheel.windows.environment]
-INCLUDE="C:\\\\a\\\\tabmat\\\\tabmat\\\\envs\\\\build\\\\Library\\\\include"
+INCLUDE="D:\\\\a\\\\tabmat\\\\tabmat\\\\envs\\\\build\\\\Library\\\\include"
 
 [tool.cibuildwheel.linux]
 before-all = [

src/tabmat/constructor.py

@@ -94,7 +94,13 @@ def from_df(
     for dfcolidx, colname in enumerate(df.columns):
         coldata = df[:, dfcolidx]
         if object_as_cat:
-            if isinstance(coldata.dtype, (nw.String, nw.Object)):
+            if isinstance(coldata.dtype, (nw.String, nw.Object)) or (
+                pd is not None  # until Narwhals handles it natively
+                and isinstance(
+                    nw.to_native(coldata).dtype,
+                    pd.StringDtype,
+                )
+            ):
                 coldata = coldata.cast(nw.Categorical)
 
         # deal with Pandas sparse dtype (not supported by narwhals)

src/tabmat/formula.py

@@ -60,7 +60,7 @@ def _init(self):
     def _is_categorical(self, values):
         if isinstance(values, (pd.Series, pd.Categorical)):
             return values.dtype == object or isinstance(
-                values.dtype, pd.CategoricalDtype
+                values.dtype, (pd.CategoricalDtype, pd.StringDtype)
             )
         return super()._is_categorical(values)
 

tests/test_constructor.py

@@ -43,6 +43,7 @@ def construct_data(backend):
 
 def test_pandas_to_matrix():
     df = construct_data("pandas")
+    original_dtypes = df.dtypes.copy()
 
     mat = tm.from_df(
         df, dtype=np.float64, sparse_threshold=0.3, cat_threshold=4, object_as_cat=True
@@ -63,8 +64,8 @@ def test_pandas_to_matrix():
 
     # Prevent a regression where the column type of sparsified dense columns
     # was being changed in place.
-    assert df["cl"].dtype == object
-    assert df["ds"].dtype == np.float64
+    assert df["cl"].dtype == original_dtypes["cl"]
+    assert df["ds"].dtype == original_dtypes["ds"]
 
 
 @pytest.mark.parametrize("categorical_dtype", [pl.Categorical, pl.Enum(["a", "b"])])

tests/test_formula.py

@@ -305,7 +305,13 @@ def test_matrix_against_expectation_qcl(df, formula, expected):
             "C(cat_1, spans_intercept=False) * cat_2 * cat_3",
             id="custom_contrasts",
         ),
-        pytest.param("str_1", id="string_as_categorical"),
+        pytest.param(
+            "str_1",
+            id="string_as_categorical",
+            marks=pytest.mark.xfail(
+                reason="Formulaic does not treat new-style strings as categorical yet"
+            ),
+        ),
     ],
 )
 def test_matrix_against_pandas(df, formula, ensure_full_rank):