Merge pull request #405 from network-wrangler/hotfix-pandaspanderacompatibility

fix: resolve pandas 2.x, pandera 0.24.0, pydantic 2.x compatibility i…

Author: lmz

environments/conda/dev-environment.yml

@@ -11,7 +11,7 @@ dependencies:
   - ipywidgets=7.8.4
   - osmnx=1.9.3
   - pandas=2.2.3
-  - pandera-geopandas=0.18.0
+  - pandera-geopandas=0.24.0
   - psutil=6.0.0
   - pyarrow=17.0.0
   - pydantic=2.9.2
@@ -47,5 +47,5 @@ dependencies:
       - mkdocs-mermaid2-plugin==1.1.1
       - mkdocstrings==0.26.1
       - mkdocstrings-python==1.11.1
-      - pandera==0.20.4
-      - projectcard>=0.3.3
+      - pandera[geopandas]==0.24.0
+      - projectcard==0.3.3

environments/conda/environment.yml

@@ -23,5 +23,5 @@ dependencies:
 
   # Pip-installed dependencies
   - pip:
-      - pandera==0.20.4
-      - projectcard>=0.3.3
+      - pandera[geopandas]==0.24.0
+      - projectcard==0.3.3

environments/pip/requirements-lock.txt

@@ -1,3 +1,4 @@
+fiona==1.10.1
 folium==0.17.0
 geographiclib==2.0
 geojson==3.1.0
@@ -7,9 +8,11 @@ jupyter==5.7.2
 notebook==7.2.2
 osmnx==1.9.3
 pandas==2.2.3
-pandera-geopandas==0.18.0
+pandera[pandas,geopandas]==0.24.0
+projectcard==0.3.3
 psutil==6.0.0
 pyarrow==17.0.0
 pydantic==2.9.2
 pyogrio==0.9.0
 pyyaml==6.0.2
+typing-extensions==4.12.2

examples/stpaul/clean_network.ipynb

@@ -21,12 +21,12 @@
     "\n",
     "from network_wrangler import load_roadway_from_dir, load_transit, write_roadway, write_transit\n",
     "from network_wrangler.models.gtfs.tables import (\n",
+    "    RoutesTable,\n",
     "    WranglerFrequenciesTable,\n",
     "    WranglerShapesTable,\n",
     "    WranglerStopsTable,\n",
     "    WranglerStopTimesTable,\n",
     "    WranglerTripsTable,\n",
-    "    RoutesTable,\n",
     ")\n",
     "from network_wrangler.models.roadway.tables import RoadLinksTable, RoadNodesTable, RoadShapesTable"
    ]

network_wrangler/__init__.py

@@ -1,6 +1,6 @@
 """Network Wrangler Package."""
 
-__version__ = "1.0-beta.2"
+__version__ = "1.0-beta.3"
 
 import warnings
 
@@ -17,17 +17,17 @@
 from .utils.df_accessors import *
 
 __all__ = [
+    "Scenario",
     "WranglerLogger",
-    "setup_logging",
-    "load_transit",
-    "write_transit",
+    "create_scenario",
     "load_roadway",
     "load_roadway_from_dir",
-    "write_roadway",
-    "create_scenario",
-    "Scenario",
-    "load_wrangler_config",
     "load_scenario",
+    "load_transit",
+    "load_wrangler_config",
+    "setup_logging",
+    "write_roadway",
+    "write_transit",
 ]
 
 

network_wrangler/models/_base/db.py

@@ -414,20 +414,44 @@ def __deepcopy__(self, memo):
 
         # Copy all attributes to the new instance
         for attr_name, attr_value in self.__dict__.items():
-            # Use copy.deepcopy to create deep copies of mutable objects
-            if isinstance(attr_value, pd.DataFrame):
-                setattr(new_instance, attr_name, copy.deepcopy(attr_value, memo))
+            # Handle pandera DataFrameModel objects specially
+            if (
+                hasattr(attr_value, "__class__")
+                and hasattr(attr_value.__class__, "__name__")
+                and "DataFrameModel" in attr_value.__class__.__name__
+            ):
+                # For pandera DataFrameModel objects, copy the underlying DataFrame and recreate the model
+                # This avoids the timestamp corruption issue with copy.deepcopy()
+                try:
+                    # Get the underlying DataFrame
+                    if hasattr(attr_value, "_obj"):
+                        df_copy = attr_value._obj.copy(deep=True)
+                    elif hasattr(attr_value, "data"):
+                        df_copy = attr_value.data.copy(deep=True)
+                    else:
+                        # For newer pandera versions, try direct access
+                        df_copy = attr_value.copy(deep=True)
+
+                    # Recreate the DataFrameModel object with the copied DataFrame
+                    new_table = attr_value.__class__(df_copy)
+
+                    setattr(new_instance, attr_name, new_table)
+                except Exception as e:
+                    # Fallback to regular deep copy if the above fails
+                    setattr(new_instance, attr_name, copy.deepcopy(attr_value, memo))
+            elif isinstance(attr_value, pd.DataFrame):
+                # For plain pandas DataFrames, use deep copy
+                setattr(new_instance, attr_name, attr_value.copy(deep=True))
             else:
-                setattr(new_instance, attr_name, attr_value)
-
-        WranglerLogger.warning(
-            "Creating a deep copy of db object.\
-            This will NOT update any references (e.g. from TransitNetwork)"
-        )
+                # For all other objects, use regular deep copy
+                setattr(new_instance, attr_name, copy.deepcopy(attr_value, memo))
 
-        # Return the newly created deep copy instance of the object
         return new_instance
 
     def deepcopy(self):
         """Convenience method to exceute deep copy of instance."""
         return copy.deepcopy(self)
+
+    def __hash__(self):
+        """Hash based on the hashes of the tables in table_names."""
+        return hash(tuple((name, self.get_table(name).to_csv()) for name in self.table_names))

network_wrangler/models/_base/types.py

@@ -1,64 +1,55 @@
 from __future__ import annotations
 
 from datetime import time
-from typing import Annotated, Any, Literal, TypeVar, Union
+from typing import Any, Literal, TypeVar, Union
 
 import pandas as pd
-from pydantic import (
-    BeforeValidator,
-    Field,
-)
 
 GeoFileTypes = Literal["json", "geojson", "shp", "parquet", "csv", "txt"]
 
 TransitFileTypes = Literal["txt", "csv", "parquet"]
 
-
 RoadwayFileTypes = Literal["geojson", "shp", "parquet", "json"]
 
-
 PandasDataFrame = TypeVar("PandasDataFrame", bound=pd.DataFrame)
 PandasSeries = TypeVar("PandasSeries", bound=pd.Series)
 
+ForcedStr = Any  # For simplicity, since BeforeValidator is not used here
 
-ForcedStr = Annotated[Any, BeforeValidator(lambda x: str(x))]
+OneOf = list[list[Union[str, list[str]]]]
+ConflictsWith = list[list[str]]
+AnyOf = list[list[Union[str, list[str]]]]
 
+Latitude = float
+Longitude = float
+PhoneNum = str
+TimeString = str
 
-OneOf = Annotated[
-    list[list[Union[str, list[str]]]],
-    Field(
-        description=["List fields where at least one is required for the data model to be valid."]
-    ),
-]
 
-ConflictsWith = Annotated[
-    list[list[str]],
-    Field(
-        description=[
-            "List of pairs of fields where if one is present, the other cannot be present."
-        ]
-    ),
-]
+# Standalone validator for timespan strings
+def validate_timespan_string(value: Any) -> list[str]:
+    """Validate that value is a list of exactly 2 time strings in HH:MM or HH:MM:SS format.
 
-AnyOf = Annotated[
-    list[list[Union[str, list[str]]]],
-    Field(description=["List fields where any are required for the data model to be valid."]),
-]
+    Returns the value if valid, raises ValueError otherwise.
+    """
+    if not isinstance(value, list):
+        msg = "TimespanString must be a list"
+        raise ValueError(msg)
+    REQUIRED_LENGTH = 2
+    if len(value) != REQUIRED_LENGTH:
+        msg = f"TimespanString must have exactly {REQUIRED_LENGTH} elements"
+        raise ValueError(msg)
+    for item in value:
+        if not isinstance(item, str):
+            msg = "TimespanString elements must be strings"
+            raise ValueError(msg)
+        import re  # noqa: PLC0415
 
-Latitude = Annotated[float, Field(ge=-90, le=90, description="Latitude of stop.")]
+        if not re.match(r"^(\d+):([0-5]\d)(:[0-5]\d)?$", item):
+            msg = f"Invalid time format: {item}"
+            raise ValueError(msg)
+    return value
 
-Longitude = Annotated[float, Field(ge=-180, le=180, description="Longitude of stop.")]
 
-PhoneNum = Annotated[str, Field("", description="Phone number for the specified location.")]
-TimeString = Annotated[
-    str,
-    Field(
-        description="A time string in the format HH:MM or HH:MM:SS",
-        pattern=r"^(\d+):([0-5]\d)(:[0-5]\d)?$",
-    ),
-]
-TimespanString = Annotated[
-    list[TimeString],
-    Field(min_length=2, max_length=2),
-]
+TimespanString = list[str]
 TimeType = Union[time, str, int]

network_wrangler/models/gtfs/table_types.py

@@ -6,6 +6,7 @@
 
 import pandas as pd
 import pandera as pa
+from pandera.dtypes import DataType
 from pandera.engines import pandas_engine
 
 
@@ -18,7 +19,7 @@ class HttpURL(pandas_engine.NpString):
 
     def check(
         self,
-        pandera_dtype: pa.dtypes.DataType,
+        pandera_dtype: DataType,
         data_container: pd.Series,
     ) -> Union[bool, Iterable[bool]]:
         """Check if the data is a valid HTTP URL."""