Merge pull request numpy#27091 from seberg/copyto-safety

API,BUG: Fix copyto (and ufunc) handling of scalar cast safety

Author: seberg

doc/release/upcoming_changes/27091.change.rst

@@ -0,0 +1,24 @@
+Cast-safety fixes in ``copyto`` and ``full``
+--------------------------------------------
+``copyto`` now uses NEP 50 correctly and applies this to its cast safety.
+Python integer to NumPy integer casts and Python float to NumPy float casts
+are now considered "safe" even if assignment may fail or precision may be lost.
+This means the following examples change slightly:
+
+* ``np.copyto(int8_arr, 1000)`` previously performed an unsafe/same-kind cast
+   of the Python integer.  It will now always raise, to achieve an unsafe cast
+   you must pass an array or NumPy scalar.
+* ``np.copyto(uint8_arr, 1000, casting="safe")`` will raise an OverflowError
+  rather than a TypeError due to same-kind casting.
+* ``np.copyto(float32_arr, 1e300, casting="safe")`` will overflow to ``inf``
+  (float32 cannot hold ``1e300``) rather raising a TypeError.
+
+Further, only the dtype is used when assigning NumPy scalars (or 0-d arrays),
+meaning that the following behaves differently:
+
+* ``np.copyto(float32_arr, np.float64(3.0), casting="safe")`` raises.
+* ``np.coptyo(int8_arr, np.int64(100), casting="safe")`` raises.
+  Previously, NumPy checked whether the 100 fits the ``int8_arr``.
+
+This aligns ``copyto``, ``full``, and ``full_like`` with the correct NumPy 2
+behavior.

numpy/_core/src/multiarray/abstractdtypes.c

@@ -378,3 +378,117 @@ NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyComplexDType = {{{
     .dt_slots = &pycomplexdtype_slots,
     .scalar_type = NULL,  /* set in initialize_and_map_pytypes_to_dtypes */
 };
+
+
+/*
+ * Additional functions to deal with Python literal int, float, complex
+ */
+/*
+ * This function takes an existing array operand and if the new descr does
+ * not match, replaces it with a new array that has the correct descriptor
+ * and holds exactly the scalar value.
+ */
+NPY_NO_EXPORT int
+npy_update_operand_for_scalar(
+    PyArrayObject **operand, PyObject *scalar, PyArray_Descr *descr,
+    NPY_CASTING casting)
+{
+    if (PyArray_EquivTypes(PyArray_DESCR(*operand), descr)) {
+        /*
+        * TODO: This is an unfortunate work-around for legacy type resolvers
+        *       (see `convert_ufunc_arguments` in `ufunc_object.c`), that
+        *       currently forces us to replace the array.
+        */
+        if (!(PyArray_FLAGS(*operand) & NPY_ARRAY_WAS_PYTHON_INT)) {
+            return 0;
+        }
+    }
+    else if (NPY_UNLIKELY(casting == NPY_EQUIV_CASTING) &&
+             descr->type_num != NPY_OBJECT) {
+        /*
+         * increadibly niche, but users could pass equiv casting and we
+         * actually need to cast.  Let object pass (technically correct) but
+         * in all other cases, we don't technically consider equivalent.
+         * NOTE(seberg): I don't think we should be beholden to this logic.
+         */
+        PyErr_Format(PyExc_TypeError,
+            "cannot cast Python %s to %S under the casting rule 'equiv'",
+            Py_TYPE(scalar)->tp_name, descr);
+        return -1;
+    }
+
+    Py_INCREF(descr);
+    PyArrayObject *new = (PyArrayObject *)PyArray_NewFromDescr(
+            &PyArray_Type, descr, 0, NULL, NULL, NULL, 0, NULL);
+    Py_SETREF(*operand, new);
+    if (*operand == NULL) {
+        return -1;
+    }
+    if (scalar == NULL) {
+        /* The ufunc.resolve_dtypes paths can go here.  Anything should go. */
+        return 0;
+    }
+    return PyArray_SETITEM(new, PyArray_BYTES(*operand), scalar);
+}
+
+
+/*
+ * When a user passed a Python literal (int, float, complex), special promotion
+ * rules mean that we don't know the exact descriptor that should be used.
+ *
+ * Typically, this just doesn't really matter.  Unfortunately, there are two
+ * exceptions:
+ * 1. The user might have passed `signature=` which may not be compatible.
+ *    In that case, we cannot really assume "safe" casting.
+ * 2. It is at least fathomable that a DType doesn't deal with this directly.
+ *    or that using the original int64/object is wrong in the type resolution.
+ *
+ * The solution is to assume that we can use the common DType of the signature
+ * and the Python scalar DType (`in_DT`) as a safe intermediate.
+ */
+NPY_NO_EXPORT PyArray_Descr *
+npy_find_descr_for_scalar(
+    PyObject *scalar, PyArray_Descr *original_descr,
+    PyArray_DTypeMeta *in_DT, PyArray_DTypeMeta *op_DT)
+{
+    PyArray_Descr *res;
+    /* There is a good chance, descriptors already match... */
+    if (NPY_DTYPE(original_descr) == op_DT) {
+        Py_INCREF(original_descr);
+        return original_descr;
+    }
+
+    PyArray_DTypeMeta *common = PyArray_CommonDType(in_DT, op_DT);
+    if (common == NULL) {
+        PyErr_Clear();
+        /* This is fine.  We simply assume the original descr is viable. */
+        Py_INCREF(original_descr);
+        return original_descr;
+    }
+    /* A very likely case is that there is nothing to do: */
+    if (NPY_DTYPE(original_descr) == common) {
+        Py_DECREF(common);
+        Py_INCREF(original_descr);
+        return original_descr;
+    }
+    if (!NPY_DT_is_parametric(common) ||
+            /* In some paths we only have a scalar type, can't discover */
+            scalar == NULL ||
+            /* If the DType doesn't know the scalar type, guess at default. */
+            !NPY_DT_CALL_is_known_scalar_type(common, Py_TYPE(scalar))) {
+        if (common->singleton != NULL) {
+            Py_INCREF(common->singleton);
+            res = common->singleton;
+            Py_INCREF(res);
+        }
+        else {
+            res = NPY_DT_CALL_default_descr(common);
+        }
+    }
+    else {
+        res = NPY_DT_CALL_discover_descr_from_pyobject(common, scalar);
+    }
+
+    Py_DECREF(common);
+    return res;
+}

numpy/_core/src/multiarray/abstractdtypes.h

@@ -1,6 +1,7 @@
 #ifndef NUMPY_CORE_SRC_MULTIARRAY_ABSTRACTDTYPES_H_
 #define NUMPY_CORE_SRC_MULTIARRAY_ABSTRACTDTYPES_H_
 
+#include "numpy/ndarraytypes.h"
 #include "arrayobject.h"
 #include "dtypemeta.h"
 
@@ -68,6 +69,19 @@ npy_mark_tmp_array_if_pyscalar(
     return 0;
 }
 
+
+NPY_NO_EXPORT int
+npy_update_operand_for_scalar(
+    PyArrayObject **operand, PyObject *scalar, PyArray_Descr *descr,
+    NPY_CASTING casting);
+
+
+NPY_NO_EXPORT PyArray_Descr *
+npy_find_descr_for_scalar(
+    PyObject *scalar, PyArray_Descr *original_descr,
+    PyArray_DTypeMeta *in_DT, PyArray_DTypeMeta *op_DT);
+
+
 #ifdef __cplusplus
 }
 #endif

numpy/_core/src/multiarray/array_assign_scalar.c

@@ -243,8 +243,7 @@ PyArray_AssignRawScalar(PyArrayObject *dst,
     }
 
     /* Check the casting rule */
-    if (!can_cast_scalar_to(src_dtype, src_data,
-                            PyArray_DESCR(dst), casting)) {
+    if (!PyArray_CanCastTypeTo(src_dtype, PyArray_DESCR(dst), casting)) {
         npy_set_invalid_cast_error(
                 src_dtype, PyArray_DESCR(dst), casting, NPY_TRUE);
         return -1;

numpy/_core/src/multiarray/multiarraymodule.c

@@ -1929,29 +1929,63 @@ array_asfortranarray(PyObject *NPY_UNUSED(ignored),
 
 
 static PyObject *
-array_copyto(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds)
+array_copyto(PyObject *NPY_UNUSED(ignored),
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
 {
-    static char *kwlist[] = {"dst", "src", "casting", "where", NULL};
-    PyObject *wheremask_in = NULL;
-    PyArrayObject *dst = NULL, *src = NULL, *wheremask = NULL;
+    PyObject *dst_obj, *src_obj, *wheremask_in = NULL;
+    PyArrayObject *src = NULL, *wheremask = NULL;
     NPY_CASTING casting = NPY_SAME_KIND_CASTING;
+    NPY_PREPARE_ARGPARSER;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O&|O&O:copyto", kwlist,
-                &PyArray_Type, &dst,
-                &PyArray_Converter, &src,
-                &PyArray_CastingConverter, &casting,
-                &wheremask_in)) {
+    if (npy_parse_arguments("copyto", args, len_args, kwnames,
+            "dst", NULL, &dst_obj,
+            "src", NULL, &src_obj,
+            "|casting", &PyArray_CastingConverter, &casting,
+            "|where", NULL, &wheremask_in,
+            NULL, NULL, NULL) < 0) {
         goto fail;
     }
 
+    if (!PyArray_Check(dst_obj)) {
+        PyErr_Format(PyExc_TypeError,
+            "copyto() argument 1 must be a numpy.ndarray, not %s",
+            Py_TYPE(dst_obj)->tp_name);
+    }
+    PyArrayObject *dst = (PyArrayObject *)dst_obj;
+
+    src = (PyArrayObject *)PyArray_FromAny(src_obj, NULL, 0, 0, 0, NULL);
+    if (src == NULL) {
+        goto fail;
+    }
+    PyArray_DTypeMeta *DType = NPY_DTYPE(PyArray_DESCR(src));
+    Py_INCREF(DType);
+    if (npy_mark_tmp_array_if_pyscalar(src_obj, src, &DType)) {
+        /* The user passed a Python scalar */
+        PyArray_Descr *descr = npy_find_descr_for_scalar(
+                src_obj, PyArray_DESCR(src), DType,
+                NPY_DTYPE(PyArray_DESCR(dst)));
+        Py_DECREF(DType);
+        if (descr == NULL) {
+            goto fail;
+        }
+        int res = npy_update_operand_for_scalar(&src, src_obj, descr, casting);
+        Py_DECREF(descr);
+        if (res < 0) {
+            goto fail;
+        }
+    }
+    else {
+        Py_DECREF(DType);
+    }
+
     if (wheremask_in != NULL) {
         /* Get the boolean where mask */
-        PyArray_Descr *dtype = PyArray_DescrFromType(NPY_BOOL);
-        if (dtype == NULL) {
+        PyArray_Descr *descr = PyArray_DescrFromType(NPY_BOOL);
+        if (descr == NULL) {
             goto fail;
         }
         wheremask = (PyArrayObject *)PyArray_FromAny(wheremask_in,
-                                        dtype, 0, 0, 0, NULL);
+                                        descr, 0, 0, 0, NULL);
         if (wheremask == NULL) {
             goto fail;
         }
@@ -4431,7 +4465,7 @@ static struct PyMethodDef array_module_methods[] = {
         METH_FASTCALL | METH_KEYWORDS, NULL},
     {"copyto",
         (PyCFunction)array_copyto,
-        METH_VARARGS|METH_KEYWORDS, NULL},
+        METH_FASTCALL | METH_KEYWORDS, NULL},
     {"nested_iters",
         (PyCFunction)NpyIter_NestedIters,
         METH_VARARGS|METH_KEYWORDS, NULL},
@@ -5129,7 +5163,7 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) {
 
     // initialize static reference to a zero-like array
     npy_static_pydata.zero_pyint_like_arr = PyArray_ZEROS(
-            0, NULL, NPY_LONG, NPY_FALSE);
+            0, NULL, NPY_DEFAULT_INT, NPY_FALSE);
     if (npy_static_pydata.zero_pyint_like_arr == NULL) {
         goto err;
     }