Merge branch 'main' into fix-dt-overflow

Author: WillAyd

doc/source/whatsnew/v2.2.0.rst

@@ -355,6 +355,7 @@ Datetimelike
 - Bug in addition or subtraction of :class:`DateOffset` objects with microsecond components to ``datetime64`` :class:`Index`, :class:`Series`, or :class:`DataFrame` columns with non-nanosecond resolution (:issue:`55595`)
 - Bug in addition or subtraction of very large :class:`Tick` objects with :class:`Timestamp` or :class:`Timedelta` objects raising ``OverflowError`` instead of ``OutOfBoundsTimedelta`` (:issue:`55503`)
 - Bug in creating a :class:`Index`, :class:`Series`, or :class:`DataFrame` with a non-nanosecond :class:`DatetimeTZDtype` and inputs that would be out of bounds with nanosecond resolution incorrectly raising ``OutOfBoundsDatetime`` (:issue:`54620`)
+- Bug in creating a :class:`Index`, :class:`Series`, or :class:`DataFrame` with a non-nanosecond ``datetime64`` (or :class:`DatetimeTZDtype`) from mixed-numeric inputs treating those as nanoseconds instead of as multiples of the dtype's unit (which would happen with non-mixed numeric inputs) (:issue:`56004`)
 - Bug in creating a :class:`Index`, :class:`Series`, or :class:`DataFrame` with a non-nanosecond ``datetime64`` dtype and inputs that would be out of bounds for a ``datetime64[ns]`` incorrectly raising ``OutOfBoundsDatetime`` (:issue:`55756`)
 -
 

pandas/_libs/lib.pyx

@@ -25,7 +25,6 @@ from cpython.object cimport (
     Py_EQ,
     PyObject,
     PyObject_RichCompareBool,
-    PyTypeObject,
 )
 from cpython.ref cimport Py_INCREF
 from cpython.sequence cimport PySequence_Check
@@ -67,10 +66,6 @@ from numpy cimport (
 
 cnp.import_array()
 
-cdef extern from "Python.h":
-    # Note: importing extern-style allows us to declare these as nogil
-    # functions, whereas `from cpython cimport` does not.
-    bint PyObject_TypeCheck(object obj, PyTypeObject* type) nogil
 
 cdef extern from "numpy/arrayobject.h":
     # cython's numpy.dtype specification is incorrect, which leads to
@@ -89,9 +84,6 @@ cdef extern from "numpy/arrayobject.h":
             object fields
             tuple names
 
-    PyTypeObject PySignedIntegerArrType_Type
-    PyTypeObject PyUnsignedIntegerArrType_Type
-
 cdef extern from "pandas/parser/pd_parser.h":
     int floatify(object, float64_t *result, int *maybe_int) except -1
     void PandasParser_IMPORT()
@@ -1437,14 +1429,12 @@ cdef class Seen:
         self.sint_ = (
             self.sint_
             or (oINT64_MIN <= val < 0)
-            # Cython equivalent of `isinstance(val, np.signedinteger)`
-            or PyObject_TypeCheck(val, &PySignedIntegerArrType_Type)
+            or isinstance(val, cnp.signedinteger)
         )
         self.uint_ = (
             self.uint_
             or (oINT64_MAX < val <= oUINT64_MAX)
-            # Cython equivalent of `isinstance(val, np.unsignedinteger)`
-            or PyObject_TypeCheck(val, &PyUnsignedIntegerArrType_Type)
+            or isinstance(val, cnp.unsignedinteger)
         )
 
     @property

pandas/_libs/src/vendored/numpy/datetime/np_datetime.c

@@ -40,31 +40,19 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
 #else
 #if defined __has_builtin
 #if __has_builtin(__builtin_add_overflow)
-#if _LP64 || __LP64__ || _ILP64 || __ILP64__
-#define checked_int64_add(a, b, res) __builtin_saddl_overflow(a, b, res)
-#define checked_int64_sub(a, b, res) __builtin_ssubl_overflow(a, b, res)
-#define checked_int64_mul(a, b, res) __builtin_smull_overflow(a, b, res)
-#else
-#define checked_int64_add(a, b, res) __builtin_saddll_overflow(a, b, res)
-#define checked_int64_sub(a, b, res) __builtin_ssubll_overflow(a, b, res)
-#define checked_int64_mul(a, b, res) __builtin_smulll_overflow(a, b, res)
-#endif
+#define checked_int64_add(a, b, res) __builtin_add_overflow(a, b, res)
+#define checked_int64_sub(a, b, res) __builtin_sub_overflow(a, b, res)
+#define checked_int64_mul(a, b, res) __builtin_mul_overflow(a, b, res)
 #else
 _Static_assert(0,
                "Overflow checking not detected; please try a newer compiler");
 #endif
 // __has_builtin was added in gcc 10, but our muslinux_1_1 build environment
 // only has gcc-9.3, so fall back to __GNUC__ macro as long as we have that
 #elif __GNUC__ > 7
-#if _LP64 || __LP64__ || _ILP64 || __ILP64__
-#define checked_int64_add(a, b, res) __builtin_saddl_overflow(a, b, res)
-#define checked_int64_sub(a, b, res) __builtin_ssubl_overflow(a, b, res)
-#define checked_int64_mul(a, b, res) __builtin_smull_overflow(a, b, res)
-#else
-#define checked_int64_add(a, b, res) __builtin_saddll_overflow(a, b, res)
-#define checked_int64_sub(a, b, res) __builtin_ssubll_overflow(a, b, res)
-#define checked_int64_mul(a, b, res) __builtin_smulll_overflow(a, b, res)
-#endif
+#define checked_int64_add(a, b, res) __builtin_add_overflow(a, b, res)
+#define checked_int64_sub(a, b, res) __builtin_sub_overflow(a, b, res)
+#define checked_int64_mul(a, b, res) __builtin_mul_overflow(a, b, res)
 #else
 _Static_assert(0, "__has_builtin not detected; please try a newer compiler");
 #endif

pandas/_libs/tslib.pyx

@@ -530,7 +530,9 @@ cpdef array_to_datetime(
                     state.update_creso(item_reso)
                     if infer_reso:
                         creso = state.creso
-                    iresult[i] = cast_from_unit(val, "ns", out_reso=creso)
+
+                    # we now need to parse this as if unit=abbrev
+                    iresult[i] = cast_from_unit(val, abbrev, out_reso=creso)
                     state.found_other = True
 
             elif isinstance(val, str):
@@ -779,6 +781,13 @@ def array_to_datetime_with_tz(
         _TSObject tsobj
         bint infer_reso = creso == NPY_DATETIMEUNIT.NPY_FR_GENERIC
         DatetimeParseState state = DatetimeParseState(creso)
+        str abbrev
+
+    if infer_reso:
+        # We treat ints/floats as nanoseconds
+        abbrev = "ns"
+    else:
+        abbrev = npy_unit_to_abbrev(creso)
 
     for i in range(n):
         # Analogous to `item = values[i]`
@@ -790,7 +799,12 @@ def array_to_datetime_with_tz(
 
         else:
             tsobj = convert_to_tsobject(
-                item, tz=tz, unit="ns", dayfirst=dayfirst, yearfirst=yearfirst, nanos=0
+                item,
+                tz=tz,
+                unit=abbrev,
+                dayfirst=dayfirst,
+                yearfirst=yearfirst,
+                nanos=0,
             )
             if tsobj.value != NPY_NAT:
                 state.update_creso(tsobj.creso)

pandas/_libs/tslibs/util.pxd

@@ -22,6 +22,7 @@ cdef extern from "Python.h":
     object PyUnicode_DecodeLocale(const char *str, const char *errors) nogil
 
 
+cimport numpy as cnp
 from numpy cimport (
     PyArray_Check,
     float64_t,
@@ -54,7 +55,7 @@ cdef inline bint is_integer_object(object obj) noexcept:
     """
     Cython equivalent of
 
-    `isinstance(val, (int, long, np.integer)) and not isinstance(val, bool)`
+    `isinstance(val, (int, np.integer)) and not isinstance(val, (bool, np.timedelta64))`
 
     Parameters
     ----------
@@ -68,13 +69,13 @@ cdef inline bint is_integer_object(object obj) noexcept:
     -----
     This counts np.timedelta64 objects as integers.
     """
-    return (not PyBool_Check(obj) and PyArray_IsIntegerScalar(obj)
+    return (not PyBool_Check(obj) and isinstance(obj, (int, cnp.integer))
             and not is_timedelta64_object(obj))
 
 
 cdef inline bint is_float_object(object obj) noexcept nogil:
     """
-    Cython equivalent of `isinstance(val, (float, np.float64))`
+    Cython equivalent of `isinstance(val, (float, np.floating))`
 
     Parameters
     ----------
@@ -90,7 +91,7 @@ cdef inline bint is_float_object(object obj) noexcept nogil:
 
 cdef inline bint is_complex_object(object obj) noexcept nogil:
     """
-    Cython equivalent of `isinstance(val, (complex, np.complex128))`
+    Cython equivalent of `isinstance(val, (complex, np.complexfloating))`
 
     Parameters
     ----------

pandas/core/arrays/datetimes.py

@@ -2220,6 +2220,7 @@ def _sequence_to_dt64(
         data = cast(np.ndarray, data)
         copy = False
         if lib.infer_dtype(data, skipna=False) == "integer":
+            # Much more performant than going through array_to_datetime
             data = data.astype(np.int64)
         elif tz is not None and ambiguous == "raise":
             obj_data = np.asarray(data, dtype=object)

pandas/core/computation/eval.py

@@ -389,6 +389,9 @@ def eval(
             # to use a non-numeric indexer
             try:
                 with warnings.catch_warnings(record=True):
+                    warnings.filterwarnings(
+                        "always", "Setting a value on a view", FutureWarning
+                    )
                     # TODO: Filter the warnings we actually care about here.
                     if inplace and isinstance(target, NDFrame):
                         target.loc[:, assigner] = ret

pandas/core/dtypes/cast.py

@@ -1133,7 +1133,7 @@ def convert_dtypes(
                 base_dtype = inferred_dtype
             if (
                 base_dtype.kind == "O"  # type: ignore[union-attr]
-                and len(input_array) > 0
+                and input_array.size > 0
                 and isna(input_array).all()
             ):
                 import pyarrow as pa

pandas/core/frame.py

@@ -909,8 +909,8 @@ def __dataframe__(
         Parameters
         ----------
         nan_as_null : bool, default False
-            Whether to tell the DataFrame to overwrite null values in the data
-            with ``NaN`` (or ``NaT``).
+            `nan_as_null` is DEPRECATED and has no effect. Please avoid using
+            it; it will be removed in a future release.
         allow_copy : bool, default True
             Whether to allow memory copying when exporting. If set to False
             it would cause non-zero-copy exports to fail.
@@ -925,9 +925,6 @@ def __dataframe__(
         Details on the interchange protocol:
         https://data-apis.org/dataframe-protocol/latest/index.html
 
-        `nan_as_null` currently has no effect; once support for nullable extension
-        dtypes is added, this value should be propagated to columns.
-
         Examples
         --------
         >>> df_not_necessarily_pandas = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
@@ -947,7 +944,7 @@ def __dataframe__(
 
         from pandas.core.interchange.dataframe import PandasDataFrameXchg
 
-        return PandasDataFrameXchg(self, nan_as_null, allow_copy)
+        return PandasDataFrameXchg(self, allow_copy=allow_copy)
 
     def __dataframe_consortium_standard__(
         self, *, api_version: str | None = None
@@ -4857,6 +4854,7 @@ def eval(self, expr: str, *, inplace: bool = False, **kwargs) -> Any | None:
 
         inplace = validate_bool_kwarg(inplace, "inplace")
         kwargs["level"] = kwargs.pop("level", 0) + 1
+        # TODO(CoW) those index/column resolvers create unnecessary refs to `self`
         index_resolvers = self._get_index_resolvers()
         column_resolvers = self._get_cleaned_column_resolvers()
         resolvers = column_resolvers, index_resolvers

pandas/core/generic.py

@@ -6940,36 +6940,16 @@ def convert_dtypes(
         dtype: string
         """
         check_dtype_backend(dtype_backend)
-        if self.ndim == 1:
-            return self._convert_dtypes(
-                infer_objects,
-                convert_string,
-                convert_integer,
-                convert_boolean,
-                convert_floating,
-                dtype_backend=dtype_backend,
-            )
-        else:
-            results = [
-                col._convert_dtypes(
-                    infer_objects,
-                    convert_string,
-                    convert_integer,
-                    convert_boolean,
-                    convert_floating,
-                    dtype_backend=dtype_backend,
-                )
-                for col_name, col in self.items()
-            ]
-            if len(results) > 0:
-                result = concat(results, axis=1, copy=False, keys=self.columns)
-                cons = cast(type["DataFrame"], self._constructor)
-                result = cons(result)
-                result = result.__finalize__(self, method="convert_dtypes")
-                # https://github.com/python/mypy/issues/8354
-                return cast(Self, result)
-            else:
-                return self.copy(deep=None)
+        new_mgr = self._mgr.convert_dtypes(  # type: ignore[union-attr]
+            infer_objects=infer_objects,
+            convert_string=convert_string,
+            convert_integer=convert_integer,
+            convert_boolean=convert_boolean,
+            convert_floating=convert_floating,
+            dtype_backend=dtype_backend,
+        )
+        res = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
+        return res.__finalize__(self, method="convert_dtypes")
 
     # ----------------------------------------------------------------------
     # Filling NA's