ENH: Improve error handling for out-of-bounds uint64 values in _to_datetime_with_unit

Author: kosiew

pandas/core/tools/datetimes.py

@@ -75,6 +75,7 @@
 from pandas.core.construction import extract_array
 from pandas.core.indexes.base import Index
 from pandas.core.indexes.datetimes import DatetimeIndex
+
 if TYPE_CHECKING:
     from collections.abc import (
         Callable,
@@ -478,72 +479,78 @@ def _array_strptime_with_fallback(
     return Index(result, dtype=result.dtype, name=name)
 
 
-
-
-
 def _to_datetime_with_unit(arg, unit, name, utc: bool, errors: str) -> DatetimeIndex:
     """
     to_datetime specialized to the case where a 'unit' is passed.
-    Fixes a bug where scalar out-of-bounds values were not raising
-    an error consistently.
     """
-    import pdb; pdb.set_trace()
-
-    # Ensure we handle both array-likes and scalars the same way.
-    # extract_array can return a scalar if 'arg' is scalar-like;
-    # so we force everything into at least 1D shape.
     arg = extract_array(arg, extract_numpy=True)
+    # Fix GH#60677
+    # Ensure scalar and array-like both become arrays
+    #    (so both paths use the same code).
     arg = np.atleast_1d(arg)
 
     # GH#30050 pass an ndarray to tslib.array_to_datetime
     # because it expects an ndarray argument
     if isinstance(arg, IntegerArray):
-        # For IntegerArray, we can directly convert
         arr = arg.astype(f"datetime64[{unit}]")
         tz_parsed = None
-
     else:
-        # Now we have a guaranteed ndarray
         arg = np.asarray(arg)
 
         if arg.dtype.kind in "iu":
             # Note we can't do "f" here because that could induce unwanted
-            # rounding GH#14156, GH#20445
+            #  rounding GH#14156, GH#20445
+            # Fix GH#60677
+            # ------------------------------------------------
+            # A) **Check for uint64 values above int64 max**
+            #    so we don't accidentally wrap around to -1, etc.
+            # ------------------------------------------------
+            if arg.dtype.kind == "u":  # unsigned
+                above_max = arg > np.iinfo(np.int64).max
+                if above_max.any():
+                    if errors == "raise":
+                        raise OutOfBoundsDatetime(
+                            "Cannot convert uint64 values above"
+                            f"{np.iinfo(np.int64).max}"
+                            "to a 64-bit signed datetime64[ns]."
+                        )
+                    else:
+                        # For errors != "raise" (e.g. "coerce" or "ignore"),
+                        # we can replace out-of-range entries with NaN (-> NaT),
+                        # then switch to the fallback object path:
+                        arg = arg.astype(object)
+                        arg[above_max] = np.nan
+                        return _to_datetime_with_unit(arg, unit, name, utc, errors)
+
+            # ------------------------------------------------
+            # B) Proceed with normal numeric -> datetime logic
+            # ------------------------------------------------
             arr = arg.astype(f"datetime64[{unit}]", copy=False)
             try:
                 arr = astype_overflowsafe(arr, np.dtype("M8[ns]"), copy=False)
             except OutOfBoundsDatetime:
                 if errors == "raise":
                     raise
-                # errors != "raise" => coerce to object and retry
                 arg = arg.astype(object)
                 return _to_datetime_with_unit(arg, unit, name, utc, errors)
             tz_parsed = None
 
         elif arg.dtype.kind == "f":
-            # Floating dtypes
             with np.errstate(over="raise"):
                 try:
                     arr = cast_from_unit_vectorized(arg, unit=unit)
                 except OutOfBoundsDatetime as err:
                     if errors != "raise":
-                        # coerce to object and retry
                         return _to_datetime_with_unit(
-                            arg.astype(object),
-                            unit,
-                            name,
-                            utc,
-                            errors,
+                            arg.astype(object), unit, name, utc, errors
                         )
                     raise OutOfBoundsDatetime(
                         f"cannot convert input with unit '{unit}'"
                     ) from err
 
             arr = arr.view("M8[ns]")
             tz_parsed = None
-
         else:
-            # Fallback: treat as object dtype
             arg = arg.astype(object, copy=False)
             arr, tz_parsed = tslib.array_to_datetime(
                 arg,
@@ -553,22 +560,21 @@ def _to_datetime_with_unit(arg, unit, name, utc: bool, errors: str) -> DatetimeI
                 creso=NpyDatetimeUnit.NPY_FR_ns.value,
             )
 
-    # Construct a DatetimeIndex from the array
     result = DatetimeIndex(arr, name=name)
 
-    # May need to localize result to parsed tz or convert to UTC if requested
+    # GH#23758: We may still need to localize the result with tz
+    # GH#25546: Apply tz_parsed first (from arg), then tz (from caller)
+    # result will be naive but in UTC
     result = result.tz_localize("UTC").tz_convert(tz_parsed)
 
     if utc:
         if result.tz is None:
             result = result.tz_localize("utc")
         else:
             result = result.tz_convert("utc")
-
     return result
 
 
-
 def _adjust_to_origin(arg, origin, unit):
     """
     Helper function for to_datetime.

pandas/tests/tools/test_to_datetime.py

@@ -1,7 +1,6 @@
 """test to_datetime"""
 
 import calendar
-import locale
 from collections import deque
 from datetime import (
     date,
@@ -10,12 +9,12 @@
     timezone,
 )
 from decimal import Decimal
+import locale
 import zoneinfo
 
+from dateutil.parser import parse
 import numpy as np
-import pandas as pd
 import pytest
-from dateutil.parser import parse
 
 from pandas._libs import tslib
 from pandas._libs.tslibs import (
@@ -30,6 +29,8 @@
 import pandas.util._test_decorators as td
 
 from pandas.core.dtypes.common import is_datetime64_ns_dtype
+
+import pandas as pd
 from pandas import (
     DataFrame,
     DatetimeIndex,
@@ -3688,22 +3689,21 @@ def test_to_datetime_wrapped_datetime64_ps():
         ["1970-01-01 00:00:01.901901901"], dtype="datetime64[ns]", freq=None
     )
     tm.assert_index_equal(result, expected)
-    
+
 
 def test_to_datetime_scalar_out_of_bounds():
     """Ensure pd.to_datetime raises an error for out-of-bounds scalar values."""
     uint64_max = np.iinfo("uint64").max
 
     # Expect an OverflowError when passing uint64_max as a scalar
     with pytest.raises(OutOfBoundsDatetime):
-        pd.to_datetime(uint64_max, unit="ns")
+        to_datetime(uint64_max, unit="ns")
 
     # Expect the same behavior when passing it as a list
     with pytest.raises(OutOfBoundsDatetime):
-        pd.to_datetime([uint64_max], unit="ns")
+        to_datetime([uint64_max], unit="ns")
 
     # Test a valid value (should not raise an error)
     valid_timestamp = 1_700_000_000_000_000_000  # A reasonable nanosecond timestamp
-    result = pd.to_datetime(valid_timestamp, unit="ns")
-    assert isinstance(result, pd.Timestamp)
-  
+    result = to_datetime(valid_timestamp, unit="ns")
+    assert isinstance(result, Timestamp)