gh-125543: Add an internal C API for dynamic arrays

Include/internal/pycore_dynarray.h

@@ -0,0 +1,143 @@
+#ifndef Py_INTERNAL_DYNARRAY_H
+#define Py_INTERNAL_DYNARRAY_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Python.h" // Py_ssize_t
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#define _PyDynArray_DEFAULT_SIZE 16
+
+typedef void (*_PyDynArray_Deallocator)(void *);
+
+typedef struct {
+    void **items;
+    Py_ssize_t capacity;
+    Py_ssize_t length;
+    _PyDynArray_Deallocator deallocator;
+} _PyDynArray;
+
+static inline void
+_PyDynArray_ASSERT_VALID(_PyDynArray *array)
+{
+    assert(array != NULL);
+    assert(array->items != NULL);
+}
+
+/*
+ * Initialize a dynamic array with an initial size.
+ *
+ * Returns -1 upon failure, 0 otherwise.
+ */
+PyAPI_FUNC(int)
+_PyDynArray_InitWithSize(_PyDynArray *array,
+                         _PyDynArray_Deallocator deallocator,
+                         Py_ssize_t initial);
+
+/*
+ * Append to a dynamic array.
+ *
+ * Returns -1 upon failure, 0 otherwise.
+ * If this fails, the deallocator is not ran on *item*.
+ */
+PyAPI_FUNC(int) _PyDynArray_Append(_PyDynArray *array, void *item);
+
+/*
+ * Clear all the fields on a dynamic array.
+ *
+ * Note that this does *not* free the actual dynamic array
+ * structure--use _PyDynArray_Free() for that.
+ *
+ * It's safe to call _PyDynArray_Init() or InitWithSize() again
+ * on the array after calling this.
+ */
+PyAPI_FUNC(void) _PyDynArray_Clear(_PyDynArray *array);
+
+/*
+ * Clear all the fields on a dynamic array, and then
+ * free the dynamic array structure itself.
+ *
+ * The passed array must have been created by _PyDynArray_New()
+ */
+static inline void
+_PyDynArray_Free(_PyDynArray *array)
+{
+    _PyDynArray_ASSERT_VALID(array);
+    _PyDynArray_Clear(array);
+    PyMem_RawFree(array);
+}
+
+/*
+ * Equivalent to _PyDynArray_InitWithSize() with a size of 16.
+ *
+ * Returns -1 upon failure, 0 otherwise.
+ */
+static inline int
+_PyDynArray_Init(_PyDynArray *array, _PyDynArray_Deallocator deallocator)
+{
+    return _PyDynArray_InitWithSize(array, deallocator, _PyDynArray_DEFAULT_SIZE);
+}
+
+/*
+ * Allocate and create a new dynamic array on the heap.
+ *
+ * The returned pointer should be freed with _PyDynArray_Free()
+ * If this function fails, it returns NULL.
+ */
+static inline _PyDynArray *
+_PyDynArray_NewWithSize(_PyDynArray_Deallocator deallocator, Py_ssize_t initial)
+{
+    _PyDynArray *array = PyMem_RawMalloc(sizeof(_PyDynArray));
+    if (array == NULL)
+    {
+        return NULL;
+    }
+
+    if (_PyDynArray_InitWithSize(array, deallocator, initial) < 0)
+    {
+        PyMem_RawFree(array);
+        return NULL;
+    }
+
+    _PyDynArray_ASSERT_VALID(array); // Sanity check
+    return array;
+}
+
+/*
+ * Equivalent to _PyDynArray_NewWithSize() with a size of 16.
+ */
+static inline _PyDynArray *
+_PyDynArray_New(_PyDynArray_Deallocator deallocator)
+{
+    return _PyDynArray_NewWithSize(deallocator, _PyDynArray_DEFAULT_SIZE);
+}
+
+/*
+ * Get an item from the array.
+ *
+ * If the index is not valid, this is undefined behavior.
+ */
+static inline void *
+_PyDynArray_GET_ITEM(_PyDynArray *array, Py_ssize_t index)
+{
+    _PyDynArray_ASSERT_VALID(array);
+    assert(index >= 0);
+    assert(index < array->length);
+    return array->items[index];
+}
+
+static inline Py_ssize_t
+_PyDynArray_LENGTH(_PyDynArray *array)
+{
+    _PyDynArray_ASSERT_VALID(array);
+    return array->length;
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif   /* !Py_INTERNAL_DYNARRAY_H */

Makefile.pre.in

@@ -435,6 +435,7 @@ PYTHON_OBJS=	\
 		Python/critical_section.o \
 		Python/crossinterp.o \
 		Python/dynamic_annotations.o \
+		Python/dynarray.o \
 		Python/errors.o \
 		Python/flowgraph.o \
 		Python/frame.o \
@@ -1197,6 +1198,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_dict.h \
 		$(srcdir)/Include/internal/pycore_dict_state.h \
 		$(srcdir)/Include/internal/pycore_dtoa.h \
+		$(srcdir)/Include/internal/pycore_dynarray.h \
 		$(srcdir)/Include/internal/pycore_exceptions.h \
 		$(srcdir)/Include/internal/pycore_faulthandler.h \
 		$(srcdir)/Include/internal/pycore_fileutils.h \

Modules/_testinternalcapi.c

@@ -17,6 +17,7 @@
 #include "pycore_compile.h"       // _PyCompile_CodeGen()
 #include "pycore_context.h"       // _PyContext_NewHamtForTests()
 #include "pycore_dict.h"          // _PyManagedDictPointer_GetValues()
+#include "pycore_dynarray.h"      // _PyDynArray
 #include "pycore_fileutils.h"     // _Py_normpath()
 #include "pycore_flowgraph.h"     // _PyCompile_OptimizeCfg()
 #include "pycore_frame.h"         // _PyInterpreterFrame
@@ -2048,6 +2049,104 @@ identify_type_slot_wrappers(PyObject *self, PyObject *Py_UNUSED(ignored))
     return _PyType_GetSlotWrapperNames();
 }
 
+static int
+test_dynarray_common(_PyDynArray *array)
+{
+#define APPEND(ptr) do {                    \
+    if (_PyDynArray_Append(array, ptr) < 0) \
+    {                                       \
+        return -1;                          \
+    }                                       \
+} while (0);
+
+    // Some dummy pointers
+    APPEND(Py_None);
+    APPEND(Py_True);
+    APPEND(Py_False);
+
+    // Make sure that indexes work
+    assert(_PyDynArray_GET_ITEM(array, 0) == Py_None);
+    assert(_PyDynArray_GET_ITEM(array, 1) == Py_True);
+    assert(_PyDynArray_GET_ITEM(array, 2) == Py_False);
+
+    // Now, let's make sure that resizing works.
+    for (int i = 0; i < (_PyDynArray_DEFAULT_SIZE * 2); ++i)
+    {
+        APPEND(NULL);
+    }
+
+#undef APPEND
+
+    // Make sure that nothing got corrupted
+    assert(_PyDynArray_GET_ITEM(array, 0) == Py_None);
+    assert(_PyDynArray_GET_ITEM(array, 1) == Py_True);
+    assert(_PyDynArray_GET_ITEM(array, 2) == Py_False);
+    return 0;
+}
+
+static PyObject *
+test_dynarray(PyObject *self, PyObject *unused)
+{
+    _PyDynArray array;
+    // In a loop to make sure that reinitialization works
+    for (int i = 0; i < 3; ++i)
+    {
+        if (_PyDynArray_Init(&array, NULL) < 0)
+        {
+            PyErr_NoMemory();
+            return NULL;
+        }
+
+        if (test_dynarray_common(&array) < 0)
+        {
+            PyErr_NoMemory();
+            _PyDynArray_Clear(&array);
+            return NULL;
+        }
+
+        _PyDynArray_Clear(&array);
+    }
+
+    // Array allocated on the heap
+    _PyDynArray *heap_array = _PyDynArray_New(NULL);
+    if (test_dynarray_common(heap_array) < 0)
+    {
+        _PyDynArray_Free(heap_array);
+        PyErr_NoMemory();
+        return NULL;
+    }
+    _PyDynArray_Free(heap_array);
+
+    // Still on the heap, but now the fields are too
+    _PyDynArray *array_with_deallocator = _PyDynArray_New(PyMem_Free);
+    if (array_with_deallocator == NULL)
+    {
+        PyErr_NoMemory();
+        return NULL;
+    }
+#define SILLY_STRING "My hovercraft is full of eels"
+    char *my_string = PyMem_Malloc(sizeof(SILLY_STRING));
+    if (my_string == NULL)
+    {
+        _PyDynArray_Free(array_with_deallocator);
+        PyErr_NoMemory();
+        return NULL;
+    }
+    strcpy(my_string, SILLY_STRING);
+    if (_PyDynArray_Append(array_with_deallocator, my_string) < 0)
+    {
+        PyMem_Free(my_string);
+        _PyDynArray_Free(array_with_deallocator);
+        PyErr_NoMemory();
+        return NULL;
+    }
+
+    assert(!strcmp(_PyDynArray_GET_ITEM(array_with_deallocator, 0), SILLY_STRING));
+    _PyDynArray_Free(array_with_deallocator);
+#undef SILLY_STRING
+
+    Py_RETURN_NONE;
+}
 
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
@@ -2145,6 +2244,7 @@ static PyMethodDef module_functions[] = {
     GH_119213_GETARGS_METHODDEF
     {"get_static_builtin_types", get_static_builtin_types, METH_NOARGS},
     {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS},
+    {"test_dynarray", test_dynarray, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };
 

PCbuild/_freeze_module.vcxproj

@@ -201,6 +201,7 @@
     <ClCompile Include="..\Python\crossinterp.c" />
     <ClCompile Include="..\Python\dtoa.c" />
     <ClCompile Include="..\Python\dynamic_annotations.c" />
+    <ClCompile Include="..\Python\dynarray.c" />
     <ClCompile Include="..\Python\dynload_win.c" />
     <ClCompile Include="..\Python\errors.c" />
     <ClCompile Include="..\Python\fileutils.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -130,6 +130,9 @@
     <ClCompile Include="..\Python\dynamic_annotations.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\dynarray.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\dynload_win.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -231,6 +231,7 @@
     <ClInclude Include="..\Include\internal\pycore_dict.h" />
     <ClInclude Include="..\Include\internal\pycore_dict_state.h" />
     <ClInclude Include="..\Include\internal\pycore_dtoa.h" />
+    <ClInclude Include="..\Include\internal\pycore_dynarray.h" />
     <ClInclude Include="..\Include\internal\pycore_exceptions.h" />
     <ClInclude Include="..\Include\internal\pycore_faulthandler.h" />
     <ClInclude Include="..\Include\internal\pycore_fileutils.h" />
@@ -587,6 +588,7 @@
     <ClCompile Include="..\Python\critical_section.c" />
     <ClCompile Include="..\Python\crossinterp.c" />
     <ClCompile Include="..\Python\dynamic_annotations.c" />
+    <ClCompile Include="..\Python\dynarray.c" />
     <ClCompile Include="..\Python\dynload_win.c" />
     <ClCompile Include="..\Python\errors.c" />
     <ClCompile Include="..\Python\fileutils.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -615,6 +615,9 @@
     <ClInclude Include="..\Include\internal\pycore_dtoa.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal/pycore_dynarray.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_exceptions.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
@@ -1316,6 +1319,9 @@
     <ClCompile Include="..\Python\dynload_win.c">
       <Filter>Python</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\dynarray.c">
+      <Filter>Python</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\errors.c">
       <Filter>Python</Filter>
     </ClCompile>