fix: restore cleanup of SPARK_LOCAL_IP and PYARROW_IGNORE_TIMEZONE after import

PR #2123 removed the `finally` block that cleaned up environment
variables set temporarily for importing `pyspark.pandas`. This causes
SPARK_LOCAL_IP=127.0.0.1 to persist in os.environ, breaking
non-YARN distributed Spark setups where executors need to connect
to the actual driver IP instead of localhost.

Restores the pre-v0.27.0 behavior: set env vars temporarily, import
pyspark.pandas, then clean up.

Fixes #2344

Author: mabunassar

pandera/external_config.py

@@ -6,6 +6,9 @@
 def _set_pyspark_environment_variables():
     """Sets environment variables for pyspark."""
 
+    is_spark_local_ip_dirty = False
+    is_pyarrow_ignore_timezone_dirty = False
+
     try:
         # try importing pyspark to see if it exists. This is important because the
         # pandera.typing module defines a Series type that inherits from
@@ -14,10 +17,17 @@ def _set_pyspark_environment_variables():
         # https://spark.apache.org/docs/3.2.0/api/python/user_guide/pandas_on_spark/typehints.html#type-hinting-with-names
 
         if os.getenv("SPARK_LOCAL_IP") is None:
+            is_spark_local_ip_dirty = True
             os.environ["SPARK_LOCAL_IP"] = "127.0.0.1"
         if os.getenv("PYARROW_IGNORE_TIMEZONE") is None:
+            is_pyarrow_ignore_timezone_dirty = True
             os.environ["PYARROW_IGNORE_TIMEZONE"] = "1"
 
         import pyspark.pandas
     except (ImportError, ModuleNotFoundError):
         pass
+    finally:
+        if is_spark_local_ip_dirty:
+            os.environ.pop("SPARK_LOCAL_IP")
+        if is_pyarrow_ignore_timezone_dirty:
+            os.environ.pop("PYARROW_IGNORE_TIMEZONE")