TST(string dtype): Resolve xfails in pytables

Author: rhshadrach

pandas/io/pytables.py

@@ -3382,6 +3382,8 @@ def read(
             if (
                 using_string_dtype()
                 and isinstance(values, np.ndarray)
+                # TODO: Should is_string_array return True for an empty object ndarray?
+                and values.size != 0
                 and is_string_array(values, skipna=True)
             ):
                 df = df.astype(StringDtype(na_value=np.nan))
@@ -5112,6 +5114,8 @@ def _maybe_convert_for_string_atom(
     errors,
     columns: list[str],
 ):
+    if isinstance(bvalues.dtype, StringDtype):
+        bvalues = bvalues.to_numpy()
     if bvalues.dtype != object:
         return bvalues
 

pandas/tests/io/pytables/test_append.py

@@ -5,8 +5,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas._libs.tslibs import Timestamp
 from pandas.compat import PY312
 
@@ -25,10 +23,7 @@
     ensure_clean_store,
 )
 
-pytestmark = [
-    pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
-]
+pytestmark = [pytest.mark.single_cpu]
 
 tables = pytest.importorskip("tables")
 
@@ -40,7 +35,7 @@ def test_append(setup_path):
         # tables.NaturalNameWarning):
         df = DataFrame(
             np.random.default_rng(2).standard_normal((20, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=20, freq="B"),
         )
         _maybe_remove(store, "df1")
@@ -203,7 +198,7 @@ def test_append_some_nans(setup_path):
         tm.assert_frame_equal(store["df3"], df3, check_index_type=True)
 
 
-def test_append_all_nans(setup_path):
+def test_append_all_nans(setup_path, using_infer_string):
     with ensure_clean_store(setup_path) as store:
         df = DataFrame(
             {
@@ -255,7 +250,13 @@ def test_append_all_nans(setup_path):
             _maybe_remove(store, "df")
             store.append("df", df[:10], dropna=True)
             store.append("df", df[10:], dropna=True)
-            tm.assert_frame_equal(store["df"], df, check_index_type=True)
+            result = store["df"]
+            expected = df
+            if using_infer_string:
+                # TODO: Test is incorrect when not using_infer_string.
+                #       Should take the last 4 rows uncondiationally.
+                expected = expected[16:]
+            tm.assert_frame_equal(result, expected, check_index_type=True)
 
             _maybe_remove(store, "df2")
             store.append("df2", df[:10], dropna=False)
@@ -294,7 +295,7 @@ def test_append_frame_column_oriented(setup_path, request):
         # column oriented
         df = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=10, freq="B"),
         )
         df.index = df.index._with_freq(None)  # freq doesn't round-trip
@@ -426,7 +427,7 @@ def check_col(key, name, size):
             {
                 "A": [0.0, 1.0, 2.0, 3.0, 4.0],
                 "B": [0.0, 1.0, 0.0, 1.0, 0.0],
-                "C": Index(["foo1", "foo2", "foo3", "foo4", "foo5"], dtype=object),
+                "C": Index(["foo1", "foo2", "foo3", "foo4", "foo5"]),
                 "D": date_range("20130101", periods=5),
             }
         ).set_index("C")
@@ -453,7 +454,7 @@ def check_col(key, name, size):
         _maybe_remove(store, "df")
         df = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=10, freq="B"),
         )
         df["string"] = "foo"
@@ -517,7 +518,7 @@ def test_append_with_data_columns(setup_path):
     with ensure_clean_store(setup_path) as store:
         df = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)),
-            columns=Index(list("ABCD"), dtype=object),
+            columns=Index(list("ABCD")),
             index=date_range("2000-01-01", periods=10, freq="B"),
         )
         df.iloc[0, df.columns.get_loc("B")] = 1.0
@@ -693,8 +694,8 @@ def test_append_misc(setup_path):
     with ensure_clean_store(setup_path) as store:
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(list("ABCD")),
+            index=Index([f"i-{i}" for i in range(30)]),
         )
         store.append("df", df, chunksize=1)
         result = store.select("df")
@@ -710,8 +711,8 @@ def test_append_misc_chunksize(setup_path, chunksize):
     # more chunksize in append tests
     df = DataFrame(
         1.1 * np.arange(120).reshape((30, 4)),
-        columns=Index(list("ABCD"), dtype=object),
-        index=Index([f"i-{i}" for i in range(30)], dtype=object),
+        columns=Index(list("ABCD")),
+        index=Index([f"i-{i}" for i in range(30)]),
     )
     df["string"] = "foo"
     df["float322"] = 1.0
@@ -747,15 +748,15 @@ def test_append_misc_empty_frame(setup_path):
         tm.assert_frame_equal(store.select("df2"), df)
 
 
-def test_append_raise(setup_path):
+def test_append_raise(setup_path, using_infer_string):
     with ensure_clean_store(setup_path) as store:
         # test append with invalid input to get good error messages
 
         # list in column
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(list("ABCD")),
+            index=Index([f"i-{i}" for i in range(30)]),
         )
         df["invalid"] = [["a"]] * len(df)
         assert df.dtypes["invalid"] == np.object_
@@ -775,8 +776,8 @@ def test_append_raise(setup_path):
         # datetime with embedded nans as object
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(list("ABCD")),
+            index=Index([f"i-{i}" for i in range(30)]),
         )
         s = Series(datetime.datetime(2001, 1, 2), index=df.index)
         s = s.astype(object)
@@ -803,8 +804,8 @@ def test_append_raise(setup_path):
         # appending an incompatible table
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
-            columns=Index(list("ABCD"), dtype=object),
-            index=Index([f"i-{i}" for i in range(30)], dtype=object),
+            columns=Index(list("ABCD")),
+            index=Index([f"i-{i}" for i in range(30)]),
         )
         store.append("df", df)
 
@@ -822,10 +823,11 @@ def test_append_raise(setup_path):
         df["foo"] = Timestamp("20130101")
         store.append("df", df)
         df["foo"] = "bar"
+        shape = "(30,)" if using_infer_string else "(1, 30)"
         msg = re.escape(
             "invalid combination of [values_axes] on appending data "
             "[name->values_block_1,cname->values_block_1,"
-            "dtype->bytes24,kind->string,shape->(1, 30)] "
+            f"dtype->bytes24,kind->string,shape->{shape}] "
             "vs current table "
             "[name->values_block_1,cname->values_block_1,"
             "dtype->datetime64[s],kind->datetime64[s],shape->None]"
@@ -884,7 +886,7 @@ def test_append_with_timedelta(setup_path):
 def test_append_to_multiple(setup_path):
     df1 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(list("ABCD")),
         index=date_range("2000-01-01", periods=10, freq="B"),
     )
     df2 = df1.copy().rename(columns="{}_2".format)
@@ -921,12 +923,12 @@ def test_append_to_multiple(setup_path):
 def test_append_to_multiple_dropna(setup_path):
     df1 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(list("ABCD")),
         index=date_range("2000-01-01", periods=10, freq="B"),
     )
     df2 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(list("ABCD")),
         index=date_range("2000-01-01", periods=10, freq="B"),
     ).rename(columns="{}_2".format)
     df1.iloc[1, df1.columns.get_indexer(["A", "B"])] = np.nan
@@ -946,7 +948,7 @@ def test_append_to_multiple_dropna(setup_path):
 def test_append_to_multiple_dropna_false(setup_path):
     df1 = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
-        columns=Index(list("ABCD"), dtype=object),
+        columns=Index(list("ABCD")),
         index=date_range("2000-01-01", periods=10, freq="B"),
     )
     df2 = df1.copy().rename(columns="{}_2".format)

pandas/tests/io/pytables/test_categorical.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     Categorical,
     DataFrame,
@@ -16,10 +14,7 @@
     ensure_clean_store,
 )
 
-pytestmark = [
-    pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
-]
+pytestmark = [pytest.mark.single_cpu]
 
 
 def test_categorical(setup_path):

pandas/tests/io/pytables/test_complex.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -13,10 +11,6 @@
 
 from pandas.io.pytables import read_hdf
 
-pytestmark = pytest.mark.xfail(
-    using_string_dtype(), reason="TODO(infer_string)", strict=False
-)
-
 
 def test_complex_fixed(tmp_path, setup_path):
     df = DataFrame(

pandas/tests/io/pytables/test_errors.py

@@ -5,8 +5,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     CategoricalIndex,
     DataFrame,
@@ -24,10 +22,7 @@
     _maybe_adjust_name,
 )
 
-pytestmark = [
-    pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
-]
+pytestmark = [pytest.mark.single_cpu]
 
 
 def test_pass_spec_to_storer(setup_path):
@@ -93,9 +88,14 @@ def test_unimplemented_dtypes_table_columns(setup_path):
 
     with ensure_clean_store(setup_path) as store:
         # this fails because we have a date in the object block......
-        msg = re.escape(
-            """Cannot serialize the column [datetime1]
-because its data contents are not [string] but [date] object dtype"""
+        msg = "|".join(
+            [
+                re.escape(
+                    "Cannot serialize the column [datetime1] because its data contents "
+                    "are not [string] but [date] object dtype"
+                ),
+                re.escape("[date] is not implemented as a table column"),
+            ]
         )
         with pytest.raises(TypeError, match=msg):
             store.append("df_unimplemented", df)

pandas/tests/io/pytables/test_file_handling.py

@@ -3,8 +3,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.compat import (
     PY311,
     is_ci_environment,
@@ -35,9 +33,7 @@
 from pandas.io import pytables
 from pandas.io.pytables import Term
 
-pytestmark = [
-    pytest.mark.single_cpu,
-]
+pytestmark = [pytest.mark.single_cpu]
 
 
 @pytest.mark.parametrize("mode", ["r", "r+", "a", "w"])
@@ -329,7 +325,6 @@ def test_complibs(tmp_path, lvl, lib, request):
                 assert node.filters.complib == lib
 
 
-@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
 @pytest.mark.skipif(
     not is_platform_little_endian(), reason="reason platform is not little endian"
 )
@@ -347,7 +342,6 @@ def test_encoding(setup_path):
         tm.assert_frame_equal(result, expected)
 
 
-@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
 @pytest.mark.parametrize(
     "val",
     [
@@ -362,7 +356,7 @@ def test_encoding(setup_path):
         [b"A\xf8\xfc", np.nan, b"", b"b", b"c"],
     ],
 )
-@pytest.mark.parametrize("dtype", ["category", object])
+@pytest.mark.parametrize("dtype", ["category", None])
 def test_latin_encoding(tmp_path, setup_path, dtype, val):
     enc = "latin-1"
     nan_rep = ""

pandas/tests/io/pytables/test_keys.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     DataFrame,
     HDFStore,
@@ -15,10 +13,7 @@
     tables,
 )
 
-pytestmark = [
-    pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
-]
+pytestmark = [pytest.mark.single_cpu]
 
 
 def test_keys(setup_path):

pandas/tests/io/pytables/test_put.py

@@ -22,9 +22,7 @@
 )
 from pandas.util import _test_decorators as td
 
-pytestmark = [
-    pytest.mark.single_cpu,
-]
+pytestmark = [pytest.mark.single_cpu]
 
 
 def test_format_type(tmp_path, setup_path):