Add integer values. (#23)

* Add integer value functions.

* Add proper memory tracking to tests.

* Add tests for integer values.

* Fix unit test.

* Switch to Py_ssize_t

* Fix leak in backport of PyErr_SetRaisedException

* Add back cast.

* Switch to longs.

* Bump version to 1.1.0

* Switch to Py_CLEAR

* Refactor error return condition.

* Fix variable name in refactor.

* Explicitly use c_long in unit test.

* Fix use of c_ssize_t to c_long

* Fix reference leaks in error callbacks.

* Remove unneeded cleanup test.

* Don't hold strong reference to the gen wrapper.

* Nevermind.

* Fix double reference count.

* Remove memray files.

* Update changelog with previous fix.

* Apparently that broke everything.

* Remove changelog entry.

* Revert debug change.

* Some small reference fixes.

* Use preallocated callbacks for faster awaits.

* Fix reference leak with coroutine send().

* Update security policy.

* Add test for asyncio.gather()

* Update changelog.

* Add -W error to Memray tests.

* Remove -x flag from tests.

* Remove tests for gathering.

* Fix missing reference increase in backport of PyErr_SetRaisedException

Author: ZeroIntensity

.github/workflows/memory_check.yml

@@ -15,7 +15,7 @@ env:
 
 jobs:
     run:
-        name: Valgrind on Ubuntu
+        name: Check for memory leaks and errors
         runs-on: ubuntu-latest
 
         steps:
@@ -28,7 +28,7 @@ jobs:
 
             - name: Install Pytest
               run: |
-                  pip install pytest pytest-asyncio typing_extensions
+                  pip install pytest pytest-asyncio pytest-memray typing_extensions
               shell: bash
 
             - name: Build PyAwaitable
@@ -39,6 +39,9 @@ jobs:
 
             - name: Install Valgrind
               run: sudo apt-get update && sudo apt-get -y install valgrind
+            
+            - name: Run tests with Memray tracking
+              run: pytest --enable-leak-tracking -W error
 
             - name: Run tests with Valgrind
               run: valgrind --suppressions=valgrind-python.supp --error-exitcode=1 pytest -x

.github/workflows/tests.yml

@@ -24,7 +24,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                os: [ubuntu-latest, windows-latest, macos-12]
+                os: [ubuntu-latest, windows-latest, macos-latest]
                 python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
         steps:
@@ -36,13 +36,7 @@ jobs:
                   python-version: ${{ matrix.python-version }}
 
             - name: Install Pytest
-              run: |
-                  if [ "$RUNNER_OS" == "Windows" ]; then
-                       pip install pytest pytest-asyncio typing_extensions
-                  else
-                       pip install pytest pytest-asyncio pytest-memray typing_extensions
-                  fi
-              shell: bash
+              run: pip install pytest pytest-asyncio typing_extensions
 
             - name: Build PyAwaitable
               run: pip install .
@@ -51,10 +45,4 @@ jobs:
               run: pip install setuptools wheel && pip install ./tests/extension/ --no-build-isolation
 
             - name: Run tests
-              run: |
-                  if [ "$RUNNER_OS" == "Windows" ]; then
-                       pytest -W error
-                  else
-                       python3 -m pytest -W error --memray 
-                  fi
-              shell: bash
+              run: pytest -W error

CHANGELOG.md

@@ -7,8 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
-- Fix coroutine iterator reference leak.
-- Fix early exit of `pyawaitable_unpack_arb` if a `NULL` value was saved.
+- Changed error message when attempting to await a non-awaitable object (*i.e.*, it has no `__await__`).
+- Fixed coroutine iterator reference leak.
+- Fixed reference leak in error callbacks.
+- Fixed early exit of `pyawaitable_unpack_arb` if a `NULL` value was saved.
+- Added integer value saving and unpacking (`pyawaitable_save_int` and `pyawaitable_unpack_int`).
+- Callbacks are now preallocated for better performance.
+- Fixed reference leak in the coroutine `send()` method.
 
 ## [1.0.0] - 2024-06-24
 

SECURITY.md

@@ -4,10 +4,6 @@
 
 Breaking API changes are made *only* between major versions. Deprecations may be made in between minor versions, but functions will not be removed until the next major version. 
 
-| Version | Supported          |
-| ------- | ------------------ |
-| 1.0.x   | :white_check_mark: |
-
 ## Reporting a Vulnerability
 
 Depending on the severity of the vulnerability, you can make an issue on the [issue tracker](https://github.com/ZeroIntensity/pyawaitable/issues), or send an email explaining the vulnerability to <zintensitydev@gmail.com>

include/pyawaitable/awaitableobject.h

@@ -22,7 +22,7 @@ struct _PyAwaitableObject
     PyObject_HEAD
 
     // Callbacks
-    pyawaitable_callback *aw_callbacks[CALLBACK_ARRAY_SIZE];
+    pyawaitable_callback aw_callbacks[CALLBACK_ARRAY_SIZE];
     Py_ssize_t aw_callback_index;
 
     // Stored Values
@@ -33,6 +33,10 @@ struct _PyAwaitableObject
     void *aw_arb_values[VALUE_ARRAY_SIZE];
     Py_ssize_t aw_arb_values_index;
 
+    // Integer Values
+    long aw_int_values[VALUE_ARRAY_SIZE];
+    Py_ssize_t aw_int_values_index;
+
     // Awaitable State
     Py_ssize_t aw_state;
     bool aw_done;

include/pyawaitable/values.h

@@ -1,7 +1,7 @@
 #ifndef PYAWAITABLE_VALUES_H
 #define PYAWAITABLE_VALUES_H
 
-#include <Python.h> // PyObject
+#include <Python.h> // PyObject, Py_ssize_t
 
 PyObject *pyawaitable_new_impl(void);
 
@@ -13,4 +13,8 @@ int pyawaitable_save_impl(PyObject *awaitable, Py_ssize_t nargs, ...);
 
 int pyawaitable_unpack_impl(PyObject *awaitable, ...);
 
+int pyawaitable_save_int_impl(PyObject *awaitable, Py_ssize_t nargs, ...);
+
+int pyawaitable_unpack_int_impl(PyObject *awaitable, ...);
+
 #endif

setup.py

@@ -5,7 +5,7 @@
     setup(
         name="pyawaitable",
         license="MIT",
-        version = "1.0.1",
+        version = "1.1.0",
         ext_modules=[
             Extension(
                 "_pyawaitable",

src/_pyawaitable/awaitable.c

@@ -38,26 +38,18 @@ PyObject *
 awaitable_next(PyObject *self)
 {
     PyAwaitableObject *aw = (PyAwaitableObject *)self;
-    aw->aw_awaited = true;
-
-    if (aw->aw_done)
+    if (aw->aw_awaited)
     {
         PyErr_SetString(
             PyExc_RuntimeError,
             "pyawaitable: cannot reuse awaitable"
         );
         return NULL;
     }
-
+    aw->aw_awaited = true;
     PyObject *gen = genwrapper_new(aw);
-
-    if (gen == NULL)
-    {
-        return NULL;
-    }
-
-    aw->aw_gen = gen;
-    return Py_NewRef(gen);
+    aw->aw_gen = Py_XNewRef(gen);
+    return gen;
 }
 
 static void
@@ -76,13 +68,22 @@ awaitable_dealloc(PyObject *self)
 
     for (int i = 0; i < CALLBACK_ARRAY_SIZE; ++i)
     {
-        pyawaitable_callback *cb = aw->aw_callbacks[i];
+        pyawaitable_callback *cb = &aw->aw_callbacks[i];
         if (cb == NULL)
             break;
 
-        if (!cb->done)
-            Py_DECREF(cb->coro);
-        PyMem_Free(cb);
+        if (cb->done)
+        {
+            if (cb->coro != NULL)
+            {
+                PyErr_SetString(
+                    PyExc_SystemError,
+                    "sanity check: coro was not cleared"
+                );
+                PyErr_WriteUnraisable(self);
+            }
+        } else
+            Py_XDECREF(cb->coro);
     }
 
     if (!aw->aw_done && aw->aw_used)
@@ -106,23 +107,20 @@ void
 pyawaitable_cancel_impl(PyObject *aw)
 {
     assert(aw != NULL);
-    Py_INCREF(aw);
-
     PyAwaitableObject *a = (PyAwaitableObject *) aw;
 
     for (int i = 0; i < CALLBACK_ARRAY_SIZE; ++i)
     {
-        pyawaitable_callback *cb = a->aw_callbacks[i];
+        pyawaitable_callback *cb = &a->aw_callbacks[i];
         if (!cb)
             break;
 
-        if (!cb->done)
-            Py_DECREF(cb->coro);
-
-        a->aw_callbacks[i] = NULL;
+        // Reset the callback
+        Py_CLEAR(cb->coro);
+        cb->done = false;
+        cb->callback = NULL;
+        cb->err_callback = NULL;
     }
-
-    Py_DECREF(aw);
 }
 
 int
@@ -133,10 +131,6 @@ pyawaitable_await_impl(
     awaitcallback_err err
 )
 {
-    assert(aw != NULL);
-    assert(coro != NULL);
-    Py_INCREF(coro);
-    Py_INCREF(aw);
     PyAwaitableObject *a = (PyAwaitableObject *) aw;
     if (a->aw_callback_index == CALLBACK_ARRAY_SIZE)
     {
@@ -147,31 +141,18 @@ pyawaitable_await_impl(
         return -1;
     }
 
-    pyawaitable_callback *aw_c = PyMem_Malloc(sizeof(pyawaitable_callback));
-    if (aw_c == NULL)
-    {
-        Py_DECREF(aw);
-        Py_DECREF(coro);
-        PyErr_NoMemory();
-        return -1;
-    }
-
-    aw_c->coro = coro; // Steal our own reference
+    pyawaitable_callback *aw_c = &a->aw_callbacks[a->aw_callback_index++];
+    aw_c->coro = Py_NewRef(coro);
     aw_c->callback = cb;
     aw_c->err_callback = err;
     aw_c->done = false;
-    a->aw_callbacks[a->aw_callback_index++] = aw_c;
-    Py_DECREF(aw);
 
     return 0;
 }
 
 int
 pyawaitable_set_result_impl(PyObject *awaitable, PyObject *result)
 {
-    assert(awaitable != NULL);
-    assert(result != NULL);
-
     PyAwaitableObject *aw = (PyAwaitableObject *) awaitable;
     aw->aw_result = Py_NewRef(result);
     return 0;

src/_pyawaitable/backport.c

@@ -41,6 +41,8 @@ PyErr_GetRaisedException(void)
 void
 PyErr_SetRaisedException(PyObject *err)
 {
+    // NOTE: We need to incref the type object here, even though
+    // this function steals a reference to err.
     PyErr_Restore(Py_NewRef((PyObject *) Py_TYPE(err)), err, NULL);
 }
 

src/_pyawaitable/coro.c

@@ -14,6 +14,7 @@ awaitable_send_with_arg(PyObject *self, PyObject *value)
         if (gen == NULL)
             return NULL;
 
+        Py_DECREF(gen);
         Py_RETURN_NONE;
     }
 
@@ -82,7 +83,7 @@ awaitable_throw(PyObject *self, PyObject *args)
     if ((aw->aw_gen != NULL) && (aw->aw_state != 0))
     {
         GenWrapperObject *gw = (GenWrapperObject *)aw->aw_gen;
-        pyawaitable_callback *cb = aw->aw_callbacks[aw->aw_state - 1];
+        pyawaitable_callback *cb = &aw->aw_callbacks[aw->aw_state - 1];
         if (cb == NULL)
             return NULL;
 
@@ -101,10 +102,11 @@ awaitable_am_send(PyObject *self, PyObject *arg, PyObject **presult)
     PyObject *send_res = awaitable_send_with_arg(self, arg);
     if (send_res == NULL)
     {
-        PyObject *occurred = PyErr_Occurred();
-        if (PyErr_GivenExceptionMatches(occurred, PyExc_StopIteration))
+        if (PyErr_ExceptionMatches(PyExc_StopIteration))
         {
+            PyObject *occurred = PyErr_GetRaisedException();
             PyObject *item = PyObject_GetAttrString(occurred, "value");
+            Py_DECREF(occurred);
 
             if (item == NULL)
             {