[SPARK-54885][PYTHON] Remove unreachable code after upgrading pyarrow minimum version to 18.0.0

### What changes were proposed in this pull request?
Remove unreachable code after upgrading pyarrow minimum version to 18.0.0

### Why are the changes needed?
Code clean up

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
CI

### Was this patch authored or co-authored using generative AI tooling?
No

Closes #53663 from zhengruifeng/del_old_path.

Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>

Author: zhengruifeng

python/pyspark/sql/pandas/types.py

@@ -26,7 +26,6 @@
 from typing import Any, Callable, Iterable, List, Optional, Union, TYPE_CHECKING
 
 from pyspark.errors import PySparkTypeError, UnsupportedOperationException, PySparkValueError
-from pyspark.loose_version import LooseVersion
 from pyspark.sql.types import (
     cast,
     BooleanType,
@@ -539,9 +538,7 @@ def _check_arrow_array_timestamps_localize(
                 a.items, mt.valueType, truncate, timezone
             ),
         }
-        # SPARK-48302: PyArrow added support for mask argument to pa.MapArray.from_arrays in
-        # version 17.0.0
-        if a.null_count and LooseVersion(pa.__version__) >= LooseVersion("17.0.0"):
+        if a.null_count:
             params["mask"] = a.is_null()
 
         return pa.MapArray.from_arrays(**params)

python/pyspark/sql/tests/arrow/test_arrow.py

@@ -59,7 +59,6 @@
     pyarrow_requirement_message,
 )
 from pyspark.errors import ArithmeticException, PySparkTypeError, UnsupportedOperationException
-from pyspark.loose_version import LooseVersion
 from pyspark.util import is_remote_only
 
 if have_pandas:
@@ -1658,18 +1657,7 @@ def test_arrow_map_timestamp_nulls_round_trip(self):
         )
         df = self.spark.createDataFrame(origin)
         t = df.toArrow()
-
-        # SPARK-48302: PyArrow versions before 17.0.0 replaced nulls with empty lists when
-        # reconstructing MapArray columns to localize timestamps
-        if LooseVersion(pa.__version__) >= LooseVersion("17.0.0"):
-            expected = origin
-        else:
-            expected = pa.table(
-                [[dict(ts=datetime.datetime(2023, 1, 1, 8, 0, 0)), []]],
-                schema=origin_schema,
-            )
-
-        self.assertTrue(t.equals(expected))
+        self.assertTrue(t.equals(origin))
 
     def test_createDataFrame_udt(self):
         for arrow_enabled in [True, False]: