remove usage of pa_version_under11p0

Author: jorisvandenbossche

pandas/compat/__init__.py

@@ -26,7 +26,7 @@
 from pandas.compat.numpy import is_numpy_dev
 from pandas.compat.pyarrow import (
     HAS_PYARROW,
-    pa_version_under11p0,
+    pa_version_under12p1,
     pa_version_under13p0,
     pa_version_under14p0,
     pa_version_under14p1,
@@ -159,7 +159,7 @@ def is_ci_environment() -> bool:
     "PYPY",
     "WASM",
     "is_numpy_dev",
-    "pa_version_under11p0",
+    "pa_version_under12p1",
     "pa_version_under13p0",
     "pa_version_under14p0",
     "pa_version_under14p1",

pandas/compat/pyarrow.py

@@ -8,7 +8,6 @@
     import pyarrow as pa
 
     _palv = Version(Version(pa.__version__).base_version)
-    pa_version_under11p0 = _palv < Version("11.0.0")
     pa_version_under12p1 = _palv < Version("12.0.1")
     pa_version_under13p0 = _palv < Version("13.0.0")
     pa_version_under14p0 = _palv < Version("14.0.0")
@@ -21,7 +20,6 @@
     pa_version_under20p0 = _palv < Version("20.0.0")
     HAS_PYARROW = _palv >= Version("12.0.1")
 except ImportError:
-    pa_version_under11p0 = True
     pa_version_under12p1 = True
     pa_version_under13p0 = True
     pa_version_under14p0 = True

pandas/core/arrays/_arrow_string_mixins.py

@@ -13,7 +13,6 @@
 from pandas._libs import lib
 from pandas.compat import (
     HAS_PYARROW,
-    pa_version_under11p0,
     pa_version_under13p0,
     pa_version_under17p0,
 )
@@ -132,10 +131,6 @@ def _str_get(self, i: int) -> Self:
     def _str_slice(
         self, start: int | None = None, stop: int | None = None, step: int | None = None
     ) -> Self:
-        if pa_version_under11p0:
-            # GH#59724
-            result = self._apply_elementwise(lambda val: val[start:stop:step])
-            return type(self)(pa.chunked_array(result, type=self._pa_array.type))
         if start is None:
             if step is not None and step < 0:
                 # GH#59710

pandas/core/arrays/arrow/accessors.py

@@ -11,10 +11,7 @@
     cast,
 )
 
-from pandas.compat import (
-    HAS_PYARROW,
-    pa_version_under11p0,
-)
+from pandas.compat import HAS_PYARROW
 
 from pandas.core.dtypes.common import is_list_like
 
@@ -171,11 +168,6 @@ def __getitem__(self, key: int | slice) -> Series:
                 name=self._data.name,
             )
         elif isinstance(key, slice):
-            if pa_version_under11p0:
-                raise NotImplementedError(
-                    f"List slice not supported by pyarrow {pa.__version__}."
-                )
-
             # TODO: Support negative start/stop/step, ideally this would be added
             # upstream in pyarrow.
             start, stop, step = key.start, key.stop, key.step

pandas/core/arrays/arrow/array.py

@@ -23,7 +23,6 @@
 )
 from pandas.compat import (
     HAS_PYARROW,
-    pa_version_under11p0,
     pa_version_under12p1,
     pa_version_under13p0,
 )
@@ -209,16 +208,6 @@ def floordiv_compat(
     from pandas.core.arrays.timedeltas import TimedeltaArray
 
 
-def get_unit_from_pa_dtype(pa_dtype) -> str:
-    # https://github.com/pandas-dev/pandas/pull/50998#discussion_r1100344804
-    if pa_version_under11p0:
-        unit = str(pa_dtype).split("[", 1)[-1][:-1]
-        if unit not in ["s", "ms", "us", "ns"]:
-            raise ValueError(pa_dtype)
-        return unit
-    return pa_dtype.unit
-
-
 def to_pyarrow_type(
     dtype: ArrowDtype | pa.DataType | Dtype | None,
 ) -> pa.DataType | None:
@@ -1200,10 +1189,6 @@ def factorize(
         null_encoding = "mask" if use_na_sentinel else "encode"
 
         data = self._pa_array
-        pa_type = data.type
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            # https://github.com/apache/arrow/issues/15226#issuecomment-1376578323
-            data = data.cast(pa.int64())
 
         if pa.types.is_dictionary(data.type):
             if null_encoding == "encode":
@@ -1228,8 +1213,6 @@ def factorize(
             )
             uniques = type(self)(combined.dictionary)
 
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            uniques = cast(ArrowExtensionArray, uniques.astype(self.dtype))
         return indices, uniques
 
     def reshape(self, *args, **kwargs):
@@ -1516,19 +1499,7 @@ def unique(self) -> Self:
         -------
         ArrowExtensionArray
         """
-        pa_type = self._pa_array.type
-
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            # https://github.com/apache/arrow/issues/15226#issuecomment-1376578323
-            data = self._pa_array.cast(pa.int64())
-        else:
-            data = self._pa_array
-
-        pa_result = pc.unique(data)
-
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            pa_result = pa_result.cast(pa_type)
-
+        pa_result = pc.unique(self._pa_array)
         return type(self)(pa_result)
 
     def value_counts(self, dropna: bool = True) -> Series:
@@ -1548,18 +1519,12 @@ def value_counts(self, dropna: bool = True) -> Series:
         --------
         Series.value_counts
         """
-        pa_type = self._pa_array.type
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            # https://github.com/apache/arrow/issues/15226#issuecomment-1376578323
-            data = self._pa_array.cast(pa.int64())
-        else:
-            data = self._pa_array
-
         from pandas import (
             Index,
             Series,
         )
 
+        data = self._pa_array
         vc = data.value_counts()
 
         values = vc.field(0)
@@ -1569,9 +1534,6 @@ def value_counts(self, dropna: bool = True) -> Series:
             values = values.filter(mask)
             counts = counts.filter(mask)
 
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            values = values.cast(pa_type)
-
         counts = ArrowExtensionArray(counts)
 
         index = Index(type(self)(values))
@@ -1865,8 +1827,7 @@ def pyarrow_meth(data, skip_nulls, min_count=0):  # type: ignore[misc]
             if pa.types.is_duration(pa_type):
                 result = result.cast(pa_type)
             elif pa.types.is_time(pa_type):
-                unit = get_unit_from_pa_dtype(pa_type)
-                result = result.cast(pa.duration(unit))
+                result = result.cast(pa.duration(pa_type.unit))
             elif pa.types.is_date(pa_type):
                 # go with closest available unit, i.e. "s"
                 result = result.cast(pa.duration("s"))

pandas/core/strings/accessor.py

@@ -305,8 +305,6 @@ def _wrap_result(
             if isinstance(result.dtype, ArrowDtype):
                 import pyarrow as pa
 
-                from pandas.compat import pa_version_under11p0
-
                 from pandas.core.arrays.arrow.array import ArrowExtensionArray
 
                 value_lengths = pa.compute.list_value_length(result._pa_array)
@@ -319,26 +317,14 @@ def _wrap_result(
                     )
                 if min_len < max_len:
                     # append nulls to each scalar list element up to max_len
-                    if not pa_version_under11p0:
-                        result = ArrowExtensionArray(
-                            pa.compute.list_slice(
-                                result._pa_array,
-                                start=0,
-                                stop=max_len,
-                                return_fixed_size_list=True,
-                            )
+                    result = ArrowExtensionArray(
+                        pa.compute.list_slice(
+                            result._pa_array,
+                            start=0,
+                            stop=max_len,
+                            return_fixed_size_list=True,
                         )
-                    else:
-                        all_null = np.full(max_len, fill_value=None, dtype=object)
-                        values = result.to_numpy()
-                        new_values = []
-                        for row in values:
-                            if len(row) < max_len:
-                                nulls = all_null[: max_len - len(row)]
-                                row = np.append(row, nulls)
-                            new_values.append(row)
-                        pa_type = result._pa_array.type
-                        result = ArrowExtensionArray(pa.array(new_values, type=pa_type))
+                    )
                 if name is None:
                     name = range(max_len)
                 result = (

pandas/tests/extension/test_arrow.py

@@ -39,7 +39,6 @@
     PY312,
     is_ci_environment,
     is_platform_windows,
-    pa_version_under11p0,
     pa_version_under13p0,
     pa_version_under14p0,
     pa_version_under19p0,
@@ -353,15 +352,6 @@ def test_from_sequence_of_strings_pa_array(self, data, request):
                     reason="Nanosecond time parsing not supported.",
                 )
             )
-        elif pa_version_under11p0 and (
-            pa.types.is_duration(pa_dtype) or pa.types.is_decimal(pa_dtype)
-        ):
-            request.applymarker(
-                pytest.mark.xfail(
-                    raises=pa.ArrowNotImplementedError,
-                    reason=f"pyarrow doesn't support parsing {pa_dtype}",
-                )
-            )
         elif pa.types.is_timestamp(pa_dtype) and pa_dtype.tz is not None:
             _require_timezone_database(request)
 
@@ -3288,9 +3278,6 @@ def test_pow_missing_operand():
     tm.assert_series_equal(result, expected)
 
 
-@pytest.mark.skipif(
-    pa_version_under11p0, reason="Decimal128 to string cast implemented in pyarrow 11"
-)
 def test_decimal_parse_raises():
     # GH 56984
     ser = pd.Series(["1.2345"], dtype=ArrowDtype(pa.string()))
@@ -3300,9 +3287,6 @@ def test_decimal_parse_raises():
         ser.astype(ArrowDtype(pa.decimal128(1, 0)))
 
 
-@pytest.mark.skipif(
-    pa_version_under11p0, reason="Decimal128 to string cast implemented in pyarrow 11"
-)
 def test_decimal_parse_succeeds():
     # GH 56984
     ser = pd.Series(["1.2345"], dtype=ArrowDtype(pa.string()))

pandas/tests/io/test_parquet.py

@@ -13,7 +13,6 @@
 
 from pandas.compat import is_platform_windows
 from pandas.compat.pyarrow import (
-    pa_version_under11p0,
     pa_version_under13p0,
     pa_version_under15p0,
     pa_version_under17p0,
@@ -729,14 +728,7 @@ def test_to_bytes_without_path_or_buf_provided(self, pa, df_full):
 
         expected = df_full.copy()
         expected.loc[1, "string_with_nan"] = None
-        if pa_version_under11p0:
-            expected["datetime_with_nat"] = expected["datetime_with_nat"].astype(
-                "M8[ns]"
-            )
-        else:
-            expected["datetime_with_nat"] = expected["datetime_with_nat"].astype(
-                "M8[ms]"
-            )
+        expected["datetime_with_nat"] = expected["datetime_with_nat"].astype("M8[ms]")
         tm.assert_frame_equal(res, expected)
 
     def test_duplicate_columns(self, pa):
@@ -1003,8 +995,6 @@ def test_timezone_aware_index(self, pa, timezone_aware_date_list):
         # this use-case sets the resolution to 1 minute
 
         expected = df[:]
-        if pa_version_under11p0:
-            expected.index = expected.index.as_unit("ns")
         if timezone_aware_date_list.tzinfo != datetime.timezone.utc:
             # pyarrow returns pytz.FixedOffset while pandas constructs datetime.timezone
             # https://github.com/pandas-dev/pandas/issues/37286
@@ -1140,7 +1130,6 @@ def test_string_inference(self, tmp_path, pa, using_infer_string):
         )
         tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.skipif(pa_version_under11p0, reason="not supported before 11.0")
     def test_roundtrip_decimal(self, tmp_path, pa):
         # GH#54768
         import pyarrow as pa

pandas/tests/series/accessors/test_list_accessor.py

@@ -10,8 +10,6 @@
 
 pa = pytest.importorskip("pyarrow")
 
-from pandas.compat import pa_version_under11p0
-
 
 @pytest.mark.parametrize(
     "list_dtype",
@@ -57,20 +55,14 @@ def test_list_getitem_slice():
         index=[1, 3, 7],
         name="a",
     )
-    if pa_version_under11p0:
-        with pytest.raises(
-            NotImplementedError, match="List slice not supported by pyarrow "
-        ):
-            ser.list[1:None:None]
-    else:
-        actual = ser.list[1:None:None]
-        expected = Series(
-            [[2, 3], [None, 5], None],
-            dtype=ArrowDtype(pa.list_(pa.int64())),
-            index=[1, 3, 7],
-            name="a",
-        )
-        tm.assert_series_equal(actual, expected)
+    actual = ser.list[1:None:None]
+    expected = Series(
+        [[2, 3], [None, 5], None],
+        dtype=ArrowDtype(pa.list_(pa.int64())),
+        index=[1, 3, 7],
+        name="a",
+    )
+    tm.assert_series_equal(actual, expected)
 
 
 def test_list_len():
@@ -105,14 +97,8 @@ def test_list_getitem_slice_invalid():
         [[1, 2, 3], [4, None, 5], None],
         dtype=ArrowDtype(pa.list_(pa.int64())),
     )
-    if pa_version_under11p0:
-        with pytest.raises(
-            NotImplementedError, match="List slice not supported by pyarrow "
-        ):
-            ser.list[1:None:0]
-    else:
-        with pytest.raises(pa.lib.ArrowInvalid, match=re.escape("`step` must be >= 1")):
-            ser.list[1:None:0]
+    with pytest.raises(pa.lib.ArrowInvalid, match=re.escape("`step` must be >= 1")):
+        ser.list[1:None:0]
 
 
 def test_list_accessor_non_list_dtype():

pandas/tests/series/accessors/test_struct_accessor.py

@@ -2,10 +2,7 @@
 
 import pytest
 
-from pandas.compat.pyarrow import (
-    pa_version_under11p0,
-    pa_version_under13p0,
-)
+from pandas.compat.pyarrow import pa_version_under13p0
 
 from pandas import (
     ArrowDtype,
@@ -105,7 +102,6 @@ def test_struct_accessor_field_with_invalid_name_or_index():
         ser.struct.field(1.1)
 
 
-@pytest.mark.skipif(pa_version_under11p0, reason="pyarrow>=11.0.0 required")
 def test_struct_accessor_explode():
     index = Index([-100, 42, 123])
     ser = Series(