Backport PR #60615: TST(string dtype): Resolve some HDF5 xfails (#60626)

Author: rhshadrach

pandas/io/pytables.py

@@ -5274,6 +5274,8 @@ def _dtype_to_kind(dtype_str: str) -> str:
         kind = "integer"
     elif dtype_str == "object":
         kind = "object"
+    elif dtype_str == "str":
+        kind = "str"
     else:
         raise ValueError(f"cannot interpret dtype of [{dtype_str}]")
 

pandas/tests/io/pytables/test_file_handling.py

@@ -36,12 +36,11 @@
 
 pytestmark = [
     pytest.mark.single_cpu,
-    pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False),
 ]
 
 
 @pytest.mark.parametrize("mode", ["r", "r+", "a", "w"])
-def test_mode(setup_path, tmp_path, mode):
+def test_mode(setup_path, tmp_path, mode, using_infer_string):
     df = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
         columns=Index(list("ABCD"), dtype=object),
@@ -90,10 +89,12 @@ def test_mode(setup_path, tmp_path, mode):
             read_hdf(path, "df", mode=mode)
     else:
         result = read_hdf(path, "df", mode=mode)
+        if using_infer_string:
+            df.columns = df.columns.astype("str")
         tm.assert_frame_equal(result, df)
 
 
-def test_default_mode(tmp_path, setup_path):
+def test_default_mode(tmp_path, setup_path, using_infer_string):
     # read_hdf uses default mode
     df = DataFrame(
         np.random.default_rng(2).standard_normal((10, 4)),
@@ -103,7 +104,10 @@ def test_default_mode(tmp_path, setup_path):
     path = tmp_path / setup_path
     df.to_hdf(path, key="df", mode="w")
     result = read_hdf(path, "df")
-    tm.assert_frame_equal(result, df)
+    expected = df.copy()
+    if using_infer_string:
+        expected.columns = expected.columns.astype("str")
+    tm.assert_frame_equal(result, expected)
 
 
 def test_reopen_handle(tmp_path, setup_path):
@@ -162,7 +166,7 @@ def test_reopen_handle(tmp_path, setup_path):
     assert not store.is_open
 
 
-def test_open_args(setup_path):
+def test_open_args(setup_path, using_infer_string):
     with tm.ensure_clean(setup_path) as path:
         df = DataFrame(
             1.1 * np.arange(120).reshape((30, 4)),
@@ -177,8 +181,13 @@ def test_open_args(setup_path):
         store["df"] = df
         store.append("df2", df)
 
-        tm.assert_frame_equal(store["df"], df)
-        tm.assert_frame_equal(store["df2"], df)
+        expected = df.copy()
+        if using_infer_string:
+            expected.index = expected.index.astype("str")
+            expected.columns = expected.columns.astype("str")
+
+        tm.assert_frame_equal(store["df"], expected)
+        tm.assert_frame_equal(store["df2"], expected)
 
         store.close()
 
@@ -193,7 +202,7 @@ def test_flush(setup_path):
         store.flush(fsync=True)
 
 
-def test_complibs_default_settings(tmp_path, setup_path):
+def test_complibs_default_settings(tmp_path, setup_path, using_infer_string):
     # GH15943
     df = DataFrame(
         1.1 * np.arange(120).reshape((30, 4)),
@@ -206,7 +215,11 @@ def test_complibs_default_settings(tmp_path, setup_path):
     tmpfile = tmp_path / setup_path
     df.to_hdf(tmpfile, key="df", complevel=9)
     result = read_hdf(tmpfile, "df")
-    tm.assert_frame_equal(result, df)
+    expected = df.copy()
+    if using_infer_string:
+        expected.index = expected.index.astype("str")
+        expected.columns = expected.columns.astype("str")
+    tm.assert_frame_equal(result, expected)
 
     with tables.open_file(tmpfile, mode="r") as h5file:
         for node in h5file.walk_nodes(where="/df", classname="Leaf"):
@@ -217,7 +230,11 @@ def test_complibs_default_settings(tmp_path, setup_path):
     tmpfile = tmp_path / setup_path
     df.to_hdf(tmpfile, key="df", complib="zlib")
     result = read_hdf(tmpfile, "df")
-    tm.assert_frame_equal(result, df)
+    expected = df.copy()
+    if using_infer_string:
+        expected.index = expected.index.astype("str")
+        expected.columns = expected.columns.astype("str")
+    tm.assert_frame_equal(result, expected)
 
     with tables.open_file(tmpfile, mode="r") as h5file:
         for node in h5file.walk_nodes(where="/df", classname="Leaf"):
@@ -228,7 +245,11 @@ def test_complibs_default_settings(tmp_path, setup_path):
     tmpfile = tmp_path / setup_path
     df.to_hdf(tmpfile, key="df")
     result = read_hdf(tmpfile, "df")
-    tm.assert_frame_equal(result, df)
+    expected = df.copy()
+    if using_infer_string:
+        expected.index = expected.index.astype("str")
+        expected.columns = expected.columns.astype("str")
+    tm.assert_frame_equal(result, expected)
 
     with tables.open_file(tmpfile, mode="r") as h5file:
         for node in h5file.walk_nodes(where="/df", classname="Leaf"):
@@ -302,6 +323,7 @@ def test_complibs(tmp_path, lvl, lib, request):
                 assert node.filters.complib == lib
 
 
+@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
 @pytest.mark.skipif(
     not is_platform_little_endian(), reason="reason platform is not little endian"
 )
@@ -319,6 +341,7 @@ def test_encoding(setup_path):
         tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
 @pytest.mark.parametrize(
     "val",
     [

pandas/tests/io/pytables/test_subclass.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     DataFrame,
     Series,
@@ -19,7 +17,6 @@
 
 class TestHDFStoreSubclass:
     # GH 33748
-    @pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)")
     def test_supported_for_subclass_dataframe(self, tmp_path):
         data = {"a": [1, 2], "b": [3, 4]}
         sdf = tm.SubclassedDataFrame(data, dtype=np.intp)

pandas/tests/io/test_common.py

@@ -18,8 +18,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.compat import is_platform_windows
 import pandas.util._test_decorators as td
 
@@ -379,7 +377,6 @@ def test_write_fspath_all(self, writer_name, writer_kwargs, module):
                     expected = f_path.read()
                     assert result == expected
 
-    @pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string) hdf support")
     def test_write_fspath_hdf5(self):
         # Same test as write_fspath_all, except HDF5 files aren't
         # necessarily byte-for-byte identical for a given dataframe, so we'll