DEP: remove allocation_tracking, deprecate PyDataMem_SetEventHook (numpy#20394)

* DEP: remove allocation_tracking

* DOC: add release note

* DEP: deprecate PyDataMem_SetEventHook

* DOC: fix name of release note

* fixes from review

* DOC: document deprecation of PyDataMem_EventHookFunc

Author: mattip

doc/release/upcoming_changes/20394.deprecation.rst

@@ -0,0 +1,6 @@
+Deprecate PyDataMem_SetEventHook
+--------------------------------
+
+The ability to track allocations is now built-in to python via ``tracemalloc``.
+The hook function ``PyDataMem_SetEventHook`` has been deprecated and the
+demonstration of its use in tool/allocation_tracking has been removed.

doc/source/reference/c-api/data_memory.rst

@@ -20,8 +20,8 @@ Historical overview
 Since version 1.7.0, NumPy has exposed a set of ``PyDataMem_*`` functions
 (:c:func:`PyDataMem_NEW`, :c:func:`PyDataMem_FREE`, :c:func:`PyDataMem_RENEW`)
 which are backed by `alloc`, `free`, `realloc` respectively. In that version
-NumPy also exposed the `PyDataMem_EventHook` function described below, which
-wrap the OS-level calls.
+NumPy also exposed the `PyDataMem_EventHook` function (now deprecated)
+described below, which wrap the OS-level calls.
 
 Since those early days, Python also improved its memory management
 capabilities, and began providing
@@ -50,10 +50,10 @@ management routines can use :c:func:`PyDataMem_SetHandler`, which uses a
 :c:type:`PyDataMem_Handler` structure to hold pointers to functions used to
 manage the data memory. The calls are still wrapped by internal routines to
 call :c:func:`PyTraceMalloc_Track`, :c:func:`PyTraceMalloc_Untrack`, and will
-use the :c:func:`PyDataMem_EventHookFunc` mechanism. Since the functions may
-change during the lifetime of the process, each ``ndarray`` carries with it the
-functions used at the time of its instantiation, and these will be used to
-reallocate or free the data memory of the instance.
+use the deprecated :c:func:`PyDataMem_EventHookFunc` mechanism. Since the
+functions may change during the lifetime of the process, each ``ndarray``
+carries with it the functions used at the time of its instantiation, and these
+will be used to reallocate or free the data memory of the instance.
 
 .. c:type:: PyDataMem_Handler
 
@@ -119,7 +119,9 @@ For an example of setting up and using the PyDataMem_Handler, see the test in
     thread.  The hook should be written to be reentrant, if it performs
     operations that might cause new allocation events (such as the
     creation/destruction numpy objects, or creating/destroying Python
-    objects which might cause a gc)
+    objects which might cause a gc).
+
+    Deprecated in v1.23
 
 What happens when deallocating if there is no policy set
 --------------------------------------------------------

numpy/core/src/multiarray/alloc.c

@@ -186,6 +186,24 @@ npy_free_cache_dim(void * p, npy_uintp sz)
                     &PyArray_free);
 }
 
+/* Similar to array_dealloc in arrayobject.c */
+static NPY_INLINE void
+WARN_NO_RETURN(PyObject* warning, const char * msg) {
+    if (PyErr_WarnEx(warning, msg, 1) < 0) {
+        PyObject * s;
+
+        s = PyUnicode_FromString("PyDataMem_UserFREE");
+        if (s) {
+            PyErr_WriteUnraisable(s);
+            Py_DECREF(s);
+        }
+        else {
+            PyErr_WriteUnraisable(Py_None);
+        }
+    }
+}
+
+
 
 /* malloc/free/realloc hook */
 NPY_NO_EXPORT PyDataMem_EventHookFunc *_PyDataMem_eventhook = NULL;
@@ -210,6 +228,8 @@ NPY_NO_EXPORT void *_PyDataMem_eventhook_user_data = NULL;
  * operations that might cause new allocation events (such as the
  * creation/destruction numpy objects, or creating/destroying Python
  * objects which might cause a gc)
+ *
+ * Deprecated in 1.23
  */
 NPY_NO_EXPORT PyDataMem_EventHookFunc *
 PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook,
@@ -218,6 +238,10 @@ PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook,
     PyDataMem_EventHookFunc *temp;
     NPY_ALLOW_C_API_DEF
     NPY_ALLOW_C_API
+    /* 2021-11-18, 1.23 */
+    WARN_NO_RETURN(PyExc_DeprecationWarning,
+                     "PyDataMem_SetEventHook is deprecated, use tracemalloc "
+                     "and the 'np.lib.tracemalloc_domain' domain");
     temp = _PyDataMem_eventhook;
     _PyDataMem_eventhook = newhook;
     if (old_data != NULL) {
@@ -435,33 +459,14 @@ PyDataMem_UserNEW_ZEROED(size_t nmemb, size_t size, PyObject *mem_handler)
     return result;
 }
 
-/* Similar to array_dealloc in arrayobject.c */
-static NPY_INLINE void
-WARN_IN_FREE(PyObject* warning, const char * msg) {
-    if (PyErr_WarnEx(warning, msg, 1) < 0) {
-        PyObject * s;
-
-        s = PyUnicode_FromString("PyDataMem_UserFREE");
-        if (s) {
-            PyErr_WriteUnraisable(s);
-            Py_DECREF(s);
-        }
-        else {
-            PyErr_WriteUnraisable(Py_None);
-        }
-    }
-}
-
-
 
 NPY_NO_EXPORT void
 PyDataMem_UserFREE(void *ptr, size_t size, PyObject *mem_handler)
 {
     PyDataMem_Handler *handler = (PyDataMem_Handler *) PyCapsule_GetPointer(mem_handler, "mem_handler");
     if (handler == NULL) {
-        WARN_IN_FREE(PyExc_RuntimeWarning,
+        WARN_NO_RETURN(PyExc_RuntimeWarning,
                      "Could not get pointer to 'mem_handler' from PyCapsule");
-        PyErr_Clear();
         return;
     }
     PyTraceMalloc_Untrack(NPY_TRACE_DOMAIN, (npy_uintp)ptr);

numpy/core/tests/test_deprecations.py

@@ -13,7 +13,8 @@
 
 import numpy as np
 from numpy.testing import (
-    assert_raises, assert_warns, assert_, assert_array_equal, SkipTest, KnownFailureException
+    assert_raises, assert_warns, assert_, assert_array_equal, SkipTest,
+    KnownFailureException, break_cycles,
     )
 
 from numpy.core._multiarray_tests import fromstring_null_term_c_api
@@ -1250,3 +1251,22 @@ def test_both_passed(self, func):
             warnings.simplefilter("always", DeprecationWarning)
             with pytest.raises(TypeError):
                 func([0., 1.], 0., interpolation="nearest", method="nearest")
+
+
+class TestMemEventHook(_DeprecationTestCase):
+    # Deprecated 2021-11-18, NumPy 1.23
+    def test_mem_seteventhook(self):
+        # The actual tests are within the C code in
+        # multiarray/_multiarray_tests.c.src
+        import numpy.core._multiarray_tests as ma_tests
+        with pytest.warns(DeprecationWarning,
+                          match='PyDataMem_SetEventHook is deprecated'):
+            ma_tests.test_pydatamem_seteventhook_start()
+        # force an allocation and free of a numpy array
+        # needs to be larger then limit of small memory cacher in ctors.c
+        a = np.zeros(1000)
+        del a
+        break_cycles()
+        with pytest.warns(DeprecationWarning,
+                          match='PyDataMem_SetEventHook is deprecated'):
+            ma_tests.test_pydatamem_seteventhook_end()

numpy/core/tests/test_multiarray.py

@@ -8187,18 +8187,6 @@ def test_scalar_element_deletion():
     assert_raises(ValueError, a[0].__delitem__, 'x')
 
 
-class TestMemEventHook:
-    def test_mem_seteventhook(self):
-        # The actual tests are within the C code in
-        # multiarray/_multiarray_tests.c.src
-        _multiarray_tests.test_pydatamem_seteventhook_start()
-        # force an allocation and free of a numpy array
-        # needs to be larger then limit of small memory cacher in ctors.c
-        a = np.zeros(1000)
-        del a
-        break_cycles()
-        _multiarray_tests.test_pydatamem_seteventhook_end()
-
 class TestMapIter:
     def test_mapiter(self):
         # The actual tests are within the C code in

tools/allocation_tracking/README.md

@@ -1,11 +1,7 @@
-Example for using the `PyDataMem_SetEventHook` to track allocations inside numpy.
-
-`alloc_hook.pyx` implements a hook in Cython that calls back into a python
-function. `track_allocations.py` uses it for a simple listing of allocations.
-It can be built with the `setup.py` file in this folder.
-
 Note that since Python 3.6 the builtin tracemalloc module can be used to
 track allocations inside numpy.
 Numpy places its CPU memory allocations into the `np.lib.tracemalloc_domain`
 domain.
 See https://docs.python.org/3/library/tracemalloc.html.
+
+The tool that used to be here has been deprecated.