fixup pyarrow datetimetz compat and update parquet tests

Author: jorisvandenbossche

pandas/core/internals/__init__.py

@@ -48,9 +48,9 @@ def __getattr__(name: str):
             stacklevel=2,
         )
         if name == "DatetimeTZBlock":
-            from pandas.core.internals.blocks import DatetimeLikeBlock
+            from pandas.core.internals.api import _DatetimeTZBlock as DatetimeTZBlock
 
-            return DatetimeLikeBlock
+            return DatetimeTZBlock
         if name == "ExtensionBlock":
             from pandas.core.internals.blocks import ExtensionBlock
 

pandas/core/internals/api.py

@@ -29,6 +29,7 @@
 )
 from pandas.core.construction import extract_array
 from pandas.core.internals.blocks import (
+    DatetimeLikeBlock,
     check_ndim,
     ensure_block_shape,
     extract_pandas_array,
@@ -74,6 +75,14 @@ def _make_block(values: ArrayLike, placement: np.ndarray) -> Block:
     return klass(values, ndim=2, placement=placement_obj)
 
 
+class _DatetimeTZBlock(DatetimeLikeBlock):
+    """implement a datetime64 block with a tz attribute"""
+
+    values: DatetimeArray
+
+    __slots__ = ()
+
+
 def make_block(
     values, placement, klass=None, ndim=None, dtype: Dtype | None = None
 ) -> Block:
@@ -114,6 +123,16 @@ def make_block(
         dtype = dtype or values.dtype
         klass = get_block_type(dtype)
 
+    elif klass is _DatetimeTZBlock and not isinstance(values.dtype, DatetimeTZDtype):
+        # pyarrow calls get here (pyarrow<15)
+        values = DatetimeArray._simple_new(
+            # error: Argument "dtype" to "_simple_new" of "DatetimeArray" has
+            # incompatible type "Union[ExtensionDtype, dtype[Any], None]";
+            # expected "Union[dtype[datetime64], DatetimeTZDtype]"
+            values,
+            dtype=dtype,  # type: ignore[arg-type]
+        )
+
     if not isinstance(placement, BlockPlacement):
         placement = BlockPlacement(placement)
 

pandas/tests/io/test_parquet.py

@@ -728,7 +728,14 @@ def test_to_bytes_without_path_or_buf_provided(self, pa, df_full):
 
         expected = df_full.copy()
         expected.loc[1, "string_with_nan"] = None
-        expected["datetime_with_nat"] = expected["datetime_with_nat"].astype("M8[ms]")
+        if pa_version_under13p0:
+            expected["datetime_with_nat"] = expected["datetime_with_nat"].astype(
+                "M8[ns]"
+            )
+        else:
+            expected["datetime_with_nat"] = expected["datetime_with_nat"].astype(
+                "M8[ms]"
+            )
         tm.assert_frame_equal(res, expected)
 
     def test_duplicate_columns(self, pa):
@@ -972,15 +979,12 @@ def test_additional_extension_types(self, pa):
 
     def test_timestamp_nanoseconds(self, pa):
         # with version 2.6, pyarrow defaults to writing the nanoseconds, so
-        # this should work without error
-        # Note in previous pyarrows(<7.0.0), only the pseudo-version 2.0 was available
+        # this should work without error, even for pyarrow < 13
         ver = "2.6"
         df = pd.DataFrame({"a": pd.date_range("2017-01-01", freq="1ns", periods=10)})
         check_round_trip(df, pa, write_kwargs={"version": ver})
 
     def test_timezone_aware_index(self, pa, timezone_aware_date_list):
-        pytest.importorskip("pyarrow", "11.0.0")
-
         idx = 5 * [timezone_aware_date_list]
         df = pd.DataFrame(index=idx, data={"index_as_col": idx})
 
@@ -995,6 +999,8 @@ def test_timezone_aware_index(self, pa, timezone_aware_date_list):
         # this use-case sets the resolution to 1 minute
 
         expected = df[:]
+        if pa_version_under13p0:
+            expected.index = expected.index.as_unit("ns")
         if timezone_aware_date_list.tzinfo != datetime.timezone.utc:
             # pyarrow returns pytz.FixedOffset while pandas constructs datetime.timezone
             # https://github.com/pandas-dev/pandas/issues/37286
@@ -1178,7 +1184,7 @@ def test_infer_string_large_string_type(self, tmp_path, pa):
 
     def test_non_nanosecond_timestamps(self, temp_file):
         # GH#49236
-        pa = pytest.importorskip("pyarrow", "11.0.0")
+        pa = pytest.importorskip("pyarrow", "13.0.0")
         pq = pytest.importorskip("pyarrow.parquet")
 
         arr = pa.array([datetime.datetime(1600, 1, 1)], type=pa.timestamp("us"))

scripts/validate_unwanted_patterns.py

@@ -53,6 +53,7 @@
     "_get_option",
     "_fill_limit_area_1d",
     "_make_block",
+    "_DatetimeTZBlock",
 }