TST(string dtype): Resolve pytable xfails

Author: rhshadrach

pandas/io/pytables.py

@@ -3382,6 +3382,8 @@ def read(
             if (
                 using_string_dtype()
                 and isinstance(values, np.ndarray)
+                # TODO: Should is_string_array return True for an empty object ndarray?
+                and values.size != 0
                 and is_string_array(values, skipna=True)
             ):
                 df = df.astype(StringDtype(na_value=np.nan))
@@ -4148,6 +4150,9 @@ def _create_axes(
                 meta = "category"
                 metadata = np.asarray(data_converted.categories).ravel()
 
+            if isinstance(blk.dtype, StringDtype):
+                meta = str(blk.dtype)
+
             data, dtype_name = _get_data_and_dtype_name(data_converted)
 
             col = klass(
@@ -4407,7 +4412,8 @@ def read_column(
                     errors=self.errors,
                 )
                 cvs = col_values[1]
-                return Series(cvs, name=column, copy=False)
+                dtype = getattr(self.table.attrs, f"{column}_meta")
+                return Series(cvs, name=column, copy=False, dtype=dtype)
 
         raise KeyError(f"column [{column}] not found in the table")
 
@@ -4523,7 +4529,6 @@ def write_data(self, chunksize: int | None, dropna: bool = False) -> None:
                 mask = isna(a.data).all(axis=0)
                 if isinstance(mask, np.ndarray):
                     masks.append(mask.astype("u1", copy=False))
-
         # consolidate masks
         if len(masks):
             mask = masks[0]
@@ -5112,6 +5117,8 @@ def _maybe_convert_for_string_atom(
     errors,
     columns: list[str],
 ):
+    if isinstance(bvalues.dtype, StringDtype):
+        bvalues = bvalues.to_numpy()
     if bvalues.dtype != object:
         return bvalues
 

pandas/tests/io/pytables/test_append.py

@@ -5,8 +5,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas._libs.tslibs import Timestamp
 from pandas.compat import PY312
 
@@ -27,7 +25,6 @@
 
 pytestmark = [
     pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
 ]
 
 tables = pytest.importorskip("tables")
@@ -40,7 +37,7 @@ def test_append(setup_path):
         # tables.NaturalNameWarning):
         df = DataFrame(
             np.random.default_rng(2).standard_normal((20, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=20, freq="B"),
         )
         _maybe_remove(store, "df1")
@@ -203,7 +200,7 @@ def test_append_some_nans(setup_path):
         tm.assert_frame_equal(store["df3"], df3, check_index_type=True)
 
 
-def test_append_all_nans(setup_path):
+def test_append_all_nans(setup_path, using_infer_string):
     with ensure_clean_store(setup_path) as store:
         df = DataFrame(
             {
@@ -255,7 +252,13 @@ def test_append_all_nans(setup_path):
             _maybe_remove(store, "df")
             store.append("df", df[:10], dropna=True)
             store.append("df", df[10:], dropna=True)
-            tm.assert_frame_equal(store["df"], df, check_index_type=True)
+            result = store["df"]
+            expected = df
+            if using_infer_string:
+                # TODO: Test is incorrect when not using_infer_string.
+                #       Should take the last 4 rows uncondiationally.
+                expected = expected[16:]
+            tm.assert_frame_equal(result, expected, check_index_type=True)
 
             _maybe_remove(store, "df2")
             store.append("df2", df[:10], dropna=False)
@@ -294,7 +297,7 @@ def test_append_frame_column_oriented(setup_path, request):
         # column oriented
         df = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=10, freq="B"),
         )
         df.index = df.index._with_freq(None)  # freq doesn't round-trip
@@ -426,7 +429,7 @@ def check_col(key, name, size):
             {
                 "A": [0.0, 1.0, 2.0, 3.0, 4.0],
                 "B": [0.0, 1.0, 0.0, 1.0, 0.0],
-                "C": Index(["foo1", "foo2", "foo3", "foo4", "foo5"], dtype=object),
+                "C": Index(["foo1", "foo2", "foo3", "foo4", "foo5"]),
                 "D": date_range("20130101", periods=5),
             }
         ).set_index("C")
@@ -453,7 +456,7 @@ def check_col(key, name, size):
         _maybe_remove(store, "df")
         df = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=10, freq="B"),
         )
         df["string"] = "foo"
@@ -517,7 +520,7 @@ def test_append_with_data_columns(setup_path):
     with ensure_clean_store(setup_path) as store:
         df = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=10, freq="B"),
         )
         df.iloc[0, df.columns.get_loc("B")] = 1.0
@@ -693,8 +696,12 @@ def test_append_misc(setup_path):
     with ensure_clean_store(setup_path) as store:
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(
+                list("ABCD"),
+            ),
+            index=Index(
+                [f"i-{i}" for i in range(30)],
+            ),
         )
         store.append("df", df, chunksize=1)
         result = store.select("df")
@@ -710,8 +717,12 @@ def test_append_misc_chunksize(setup_path, chunksize):
     # more chunksize in append tests
     df = DataFrame(
         1.1 * np.arange(120).reshape((30, 4)),
-        columns=Index(list("ABCD"), dtype=object),
-        index=Index([f"i-{i}" for i in range(30)], dtype=object),
+        columns=Index(
+            list("ABCD"),
+        ),
+        index=Index(
+            [f"i-{i}" for i in range(30)],
+        ),
     )
     df["string"] = "foo"
     df["float322"] = 1.0
@@ -747,15 +758,19 @@ def test_append_misc_empty_frame(setup_path):
         tm.assert_frame_equal(store.select("df2"), df)
 
 
-def test_append_raise(setup_path):
+def test_append_raise(setup_path, using_infer_string):
     with ensure_clean_store(setup_path) as store:
         # test append with invalid input to get good error messages
 
         # list in column
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(
+                list("ABCD"),
+            ),
+            index=Index(
+                [f"i-{i}" for i in range(30)],
+            ),
         )
         df["invalid"] = [["a"]] * len(df)
         assert df.dtypes["invalid"] == np.object_
@@ -775,8 +790,12 @@ def test_append_raise(setup_path):
         # datetime with embedded nans as object
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(
+                list("ABCD"),
+            ),
+            index=Index(
+                [f"i-{i}" for i in range(30)],
+            ),
         )
         s = Series(datetime.datetime(2001, 1, 2), index=df.index)
         s = s.astype(object)
@@ -803,8 +822,12 @@ def test_append_raise(setup_path):
         # appending an incompatible table
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(
+                list("ABCD"),
+            ),
+            index=Index(
+                [f"i-{i}" for i in range(30)],
+            ),
         )
         store.append("df", df)
 
@@ -822,10 +845,11 @@ def test_append_raise(setup_path):
         df["foo"] = Timestamp("20130101")
         store.append("df", df)
         df["foo"] = "bar"
+        shape = "(30,)" if using_infer_string else "(1, 30)"
         msg = re.escape(
             "invalid combination of [values_axes] on appending data "
             "[name->values_block_1,cname->values_block_1,"
-            "dtype->bytes24,kind->string,shape->(1, 30)] "
+            f"dtype->bytes24,kind->string,shape->{shape}] "
             "vs current table "
             "[name->values_block_1,cname->values_block_1,"
             "dtype->datetime64[s],kind->datetime64[s],shape->None]"
@@ -884,7 +908,9 @@ def test_append_with_timedelta(setup_path):
 def test_append_to_multiple(setup_path):
     df1 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(
+            list("ABCD"),
+        ),
         index=date_range("2000-01-01", periods=10, freq="B"),
     )
     df2 = df1.copy().rename(columns="{}_2".format)
@@ -921,12 +947,16 @@ def test_append_to_multiple(setup_path):
 def test_append_to_multiple_dropna(setup_path):
     df1 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(
+            list("ABCD"),
+        ),
         index=date_range("2000-01-01", periods=10, freq="B"),
     )
     df2 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(
+            list("ABCD"),
+        ),
         index=date_range("2000-01-01", periods=10, freq="B"),
     ).rename(columns="{}_2".format)
     df1.iloc[1, df1.columns.get_indexer(["A", "B"])] = np.nan
@@ -946,7 +976,9 @@ def test_append_to_multiple_dropna(setup_path):
 def test_append_to_multiple_dropna_false(setup_path):
     df1 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(
+            list("ABCD"),
+        ),
         index=date_range("2000-01-01", periods=10, freq="B"),
     )
     df2 = df1.copy().rename(columns="{}_2".format)

pandas/tests/io/pytables/test_categorical.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     Categorical,
     DataFrame,
@@ -18,7 +16,6 @@
 
 pytestmark = [
     pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
 ]
 
 

pandas/tests/io/pytables/test_complex.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -13,10 +11,6 @@
 
 from pandas.io.pytables import read_hdf
 
-pytestmark = pytest.mark.xfail(
-    using_string_dtype(), reason="TODO(infer_string)", strict=False
-)
-
 
 def test_complex_fixed(tmp_path, setup_path):
     df = DataFrame(

pandas/tests/io/pytables/test_errors.py

@@ -5,8 +5,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     CategoricalIndex,
     DataFrame,
@@ -26,7 +24,6 @@
 
 pytestmark = [
     pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
 ]
 
 
@@ -93,9 +90,14 @@ def test_unimplemented_dtypes_table_columns(setup_path):
 
     with ensure_clean_store(setup_path) as store:
         # this fails because we have a date in the object block......
-        msg = re.escape(
-            """Cannot serialize the column [datetime1]
-because its data contents are not [string] but [date] object dtype"""
+        msg = "|".join(
+            [
+                re.escape(
+                    "Cannot serialize the column [datetime1] because its data contents "
+                    "are not [string] but [date] object dtype"
+                ),
+                re.escape("[date] is not implemented as a table column"),
+            ]
         )
         with pytest.raises(TypeError, match=msg):
             store.append("df_unimplemented", df)

pandas/tests/io/pytables/test_file_handling.py

@@ -3,8 +3,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.compat import (
     PY311,
     is_ci_environment,
@@ -329,7 +327,6 @@ def test_complibs(tmp_path, lvl, lib, request):
                 assert node.filters.complib == lib
 
 
-@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
 @pytest.mark.skipif(
     not is_platform_little_endian(), reason="reason platform is not little endian"
 )
@@ -347,7 +344,6 @@ def test_encoding(setup_path):
         tm.assert_frame_equal(result, expected)
 
 
-@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
 @pytest.mark.parametrize(
     "val",
     [
@@ -362,7 +358,7 @@ def test_encoding(setup_path):
         [b"A\xf8\xfc", np.nan, b"", b"b", b"c"],
     ],
 )
-@pytest.mark.parametrize("dtype", ["category", object])
+@pytest.mark.parametrize("dtype", ["category", None])
 def test_latin_encoding(tmp_path, setup_path, dtype, val):
     enc = "latin-1"
     nan_rep = ""

pandas/tests/io/pytables/test_keys.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     DataFrame,
     HDFStore,
@@ -17,7 +15,6 @@
 
 pytestmark = [
     pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
 ]