ENH: fix thread-unsafe C API usages (numpy#27145)

Ref numpy#26159

See also the CPython HOWTO on this topic: https://docs.python.org/3.13/howto/free-threading-extensions.html#freethreading-extensions-howto.

The remaining usages of PyDict_GetItem and PyDict_Next are all around the fields attribute of structured dtypes. I'm pretty sure that dictionary is effectively frozen after the DType is constructed, so I don't worry about those uses.

It's not straightforward to write tests for this, I'm just applying static refactorings in places where the refactoring shouldn't introduce new reference counting bugs.

* ENH: fix thread-unsafe C API usages

* ENH: use critical sections in einsum

* BUG: fix error handling in loadtxt C code

* revert einsum changes

Author: ngoldbaum

numpy/_core/src/multiarray/array_coercion.c

@@ -6,6 +6,7 @@
 #include <Python.h>
 
 #include "numpy/npy_3kcompat.h"
+#include "npy_pycompat.h"
 
 #include "lowlevel_strided_loops.h"
 #include "numpy/arrayobject.h"
@@ -224,24 +225,23 @@ npy_discover_dtype_from_pytype(PyTypeObject *pytype)
     PyObject *DType;
 
     if (pytype == &PyArray_Type) {
-        DType = Py_None;
+        DType = Py_NewRef(Py_None);
     }
     else if (pytype == &PyFloat_Type) {
-        DType = (PyObject *)&PyArray_PyFloatDType;
+        DType = Py_NewRef((PyObject *)&PyArray_PyFloatDType);
     }
     else if (pytype == &PyLong_Type) {
-        DType = (PyObject *)&PyArray_PyLongDType;
+        DType = Py_NewRef((PyObject *)&PyArray_PyLongDType);
     }
     else {
-        DType = PyDict_GetItem(_global_pytype_to_type_dict,
-                               (PyObject *)pytype);
+        int res = PyDict_GetItemRef(_global_pytype_to_type_dict,
+                                    (PyObject *)pytype, (PyObject **)&DType);
 
-        if (DType == NULL) {
-            /* the python type is not known */
+        if (res <= 0) {
+            /* the python type is not known or an error was set */
             return NULL;
         }
     }
-    Py_INCREF(DType);
     assert(DType == Py_None || PyObject_TypeCheck(DType, (PyTypeObject *)&PyArrayDTypeMeta_Type));
     return (PyArray_DTypeMeta *)DType;
 }

numpy/_core/src/multiarray/textreading/rows.c

@@ -58,13 +58,18 @@ create_conv_funcs(
 
     PyObject *key, *value;
     Py_ssize_t pos = 0;
+    int error = 0;
+#if Py_GIL_DISABLED
+    Py_BEGIN_CRITICAL_SECTION(converters);
+#endif
     while (PyDict_Next(converters, &pos, &key, &value)) {
         Py_ssize_t column = PyNumber_AsSsize_t(key, PyExc_IndexError);
         if (column == -1 && PyErr_Occurred()) {
             PyErr_Format(PyExc_TypeError,
                     "keys of the converters dictionary must be integers; "
                     "got %.100R", key);
-            goto error;
+            error = 1;
+            break;
         }
         if (usecols != NULL) {
             /*
@@ -92,7 +97,8 @@ create_conv_funcs(
                 PyErr_Format(PyExc_ValueError,
                         "converter specified for column %zd, which is invalid "
                         "for the number of fields %zd.", column, num_fields);
-                goto error;
+                error = 1;
+                break;
             }
             if (column < 0) {
                 column += num_fields;
@@ -102,11 +108,20 @@ create_conv_funcs(
             PyErr_Format(PyExc_TypeError,
                     "values of the converters dictionary must be callable, "
                     "but the value associated with key %R is not", key);
-            goto error;
+            error = 1;
+            break;
         }
         Py_INCREF(value);
         conv_funcs[column] = value;
     }
+#if Py_GIL_DISABLED
+    Py_END_CRITICAL_SECTION();
+#endif
+
+    if (error) {
+        goto error;
+    }
+    
     return conv_funcs;
 
   error: