hg merge default to release-1.15

Author: nitzmahone

.github/workflows/ci.yaml

@@ -33,6 +33,7 @@ jobs:
         - spec: cp38-manylinux_x86_64
         - spec: cp39-manylinux_x86_64
         - spec: cp310-manylinux_x86_64
+        - spec: cp311-manylinux_x86_64
         - spec: cp27-manylinux_i686
           cibw_version: cibuildwheel<2.0  # py2.7 is not supported on CIBW 2.0+
           manylinux_img: manylinux1  # build really old Pythons on manylinux1
@@ -42,32 +43,41 @@ jobs:
         - spec: cp38-manylinux_i686
         - spec: cp39-manylinux_i686
         - spec: cp310-manylinux_i686
+        - spec: cp311-manylinux_i686
+        - spec: cp39-musllinux_x86_64
+        - spec: cp310-musllinux_x86_64
+        - spec: cp311-musllinux_x86_64
+        - spec: cp39-musllinux_i686
+        - spec: cp310-musllinux_i686
+        - spec: cp311-musllinux_i686
     steps:
     - name: clone repo
       uses: actions/checkout@v2
 
     - name: build/test wheels
       env:
+        CFLAGS: -Dffi_call=cffistatic_ffi_call  # override name for ffi_call to break hard if we linked against someone else's libffi
         CIBW_ARCHS_LINUX: auto
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_BEFORE_BUILD: |
           set -eux && \
           curl -L -O https://github.com/libffi/libffi/archive/v3.4.2.tar.gz && \
           tar zxf v3.4.2.tar.gz && cd libffi-3.4.2 && \
           ./autogen.sh && \
-          ./configure --without-gcc-arch --disable-docs && \
+          ./configure --without-gcc-arch --disable-docs --with-pic --enable-shared=no && \
           make && \
           make install && \
-          ldconfig
-        # TODO: update default to '' once CIBW 2.1.3 ships: https://github.com/pypa/cibuildwheel/pull/829
-        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || 'manylinux2010' }}
-        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || 'manylinux2010' }}
+          ldconfig || true
+        CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS  # ensure that the build container can see our overridden build config
+        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_TEST_REQUIRES: pytest
-        CIBW_TEST_COMMAND: python -m pytest {project}/c {project}/testing
+        CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest {project}
       run: |
         python -m pip install --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}"
 
-        # actually build libyaml + wheel (using env tweaks above)
+        # actually build libffi + wheel (using env tweaks above)
         python -m cibuildwheel --output-dir dist .
 
 
@@ -94,6 +104,7 @@ jobs:
         - spec: cp38-macosx_x86_64
         - spec: cp39-macosx_x86_64
         - spec: cp310-macosx_x86_64
+        - spec: cp311-macosx_x86_64
         # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
         # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel)
         - spec: cp39-macosx_arm64
@@ -107,6 +118,14 @@ jobs:
           runs_on: [self-hosted, macOS]
           run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
           sdkroot: macosx11.3
+
+        - spec: cp311-macosx_arm64
+          deployment_target: '11.0'
+          runs_on: [self-hosted, macOS]
+          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
+          sdkroot: macosx11.3
+
+
     steps:
     - name: clone repo
       uses: actions/checkout@v2
@@ -119,8 +138,9 @@ jobs:
     - name: build/test wheels
       env:
         CIBW_BUILD: ${{ matrix.spec }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_TEST_REQUIRES: pytest
-        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; pytest
+        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest
       run: |
         if [[ -n "${{ matrix.deployment_target || '' }}" ]]
         then
@@ -141,7 +161,7 @@ jobs:
         if-no-files-found: error
 
   windows:
-    runs-on: windows-2016
+    runs-on: windows-2019
     strategy:
       matrix:
         include:
@@ -152,13 +172,15 @@ jobs:
         - spec: cp38-win_amd64
         - spec: cp39-win_amd64
         - spec: cp310-win_amd64
+        - spec: cp311-win_amd64
         - spec: cp27-win32
           cibw_version: cibuildwheel==1.10  # last release with proper py2.7 Windows support
         - spec: cp36-win32
         - spec: cp37-win32
         - spec: cp38-win32
         - spec: cp39-win32
         - spec: cp310-win32
+        - spec: cp311-win32
     steps:
     - name: clone repo
       uses: actions/checkout@v2
@@ -180,6 +202,7 @@ jobs:
     - name: build/test wheels
       env:
         CIBW_BUILD: ${{ matrix.spec }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
       run: |
         python -m pip install --upgrade pip
         pip install "${{ matrix.cibw_version || 'cibuildwheel'}}"

c/_cffi_backend.c

@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.15.0"
+#define CFFI_VERSION  "1.15.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
@@ -2197,7 +2197,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name)
     }
 }
 
-static PyObject *cdataowning_repr(CDataObject *cd)
+static Py_ssize_t cdataowning_size_bytes(CDataObject *cd)
 {
     Py_ssize_t size = _cdata_var_byte_size(cd);
     if (size < 0) {
@@ -2208,6 +2208,12 @@ static PyObject *cdataowning_repr(CDataObject *cd)
         else
             size = cd->c_type->ct_size;
     }
+    return size;
+}
+
+static PyObject *cdataowning_repr(CDataObject *cd)
+{
+    Py_ssize_t size = cdataowning_size_bytes(cd);
     return PyText_FromFormat("<cdata '%s' owning %zd bytes>",
                              cd->c_type->ct_name, size);
 }
@@ -4603,7 +4609,7 @@ static PyObject *get_unique_type(CTypeDescrObject *x,
            array      [ctype, length]
            funcptr    [ctresult, ellipsis+abi, num_args, ctargs...]
     */
-    PyObject *key, *y;
+    PyObject *key, *y, *res;
     void *pkey;
 
     key = PyBytes_FromStringAndSize(NULL, keylength * sizeof(void *));
@@ -4635,8 +4641,9 @@ static PyObject *get_unique_type(CTypeDescrObject *x,
     /* the 'value' in unique_cache doesn't count as 1, but don't use
        Py_DECREF(x) here because it will confuse debug builds into thinking
        there was an extra DECREF in total. */
-    ((PyObject *)x)->ob_refcnt--;
-    return (PyObject *)x;
+    res = (PyObject *)x;
+    Py_SET_REFCNT(res, Py_REFCNT(res) - 1);
+    return res;
 
  error:
     Py_DECREF(x);
@@ -5665,7 +5672,8 @@ static ffi_type *fb_fill_type(struct funcbuilder_s *fb, CTypeDescrObject *ct,
     }
 }
 
-#define ALIGN_ARG(n)  ((n) + 7) & ~7
+#define ALIGN_TO(n, a)  ((n) + ((a)-1)) & ~((a)-1)
+#define ALIGN_ARG(n)    ALIGN_TO(n, 8)
 
 static int fb_build(struct funcbuilder_s *fb, PyObject *fargs,
                     CTypeDescrObject *fresult)
@@ -5690,10 +5698,12 @@ static int fb_build(struct funcbuilder_s *fb, PyObject *fargs,
         /* exchange data size */
         /* first, enough room for an array of 'nargs' pointers */
         exchange_offset = nargs * sizeof(void*);
+        /* then enough room for the result --- which means at least
+           sizeof(ffi_arg), according to the ffi docs, but we also
+           align according to the result type, for issue #531 */
+        exchange_offset = ALIGN_TO(exchange_offset, fb->rtype->alignment);
         exchange_offset = ALIGN_ARG(exchange_offset);
         cif_descr->exchange_offset_arg[0] = exchange_offset;
-        /* then enough room for the result --- which means at least
-           sizeof(ffi_arg), according to the ffi docs */
         i = fb->rtype->size;
         if (i < (Py_ssize_t)sizeof(ffi_arg))
             i = sizeof(ffi_arg);
@@ -5721,6 +5731,7 @@ static int fb_build(struct funcbuilder_s *fb, PyObject *fargs,
         if (fb->atypes != NULL) {
             fb->atypes[i] = atype;
             /* exchange data size */
+            exchange_offset = ALIGN_TO(exchange_offset, atype->alignment);
             exchange_offset = ALIGN_ARG(exchange_offset);
             cif_descr->exchange_offset_arg[1 + i] = exchange_offset;
             exchange_offset += atype->size;
@@ -5737,6 +5748,7 @@ static int fb_build(struct funcbuilder_s *fb, PyObject *fargs,
 }
 
 #undef ALIGN_ARG
+#undef ALIGN_TO
 
 static void fb_cat_name(struct funcbuilder_s *fb, const char *piece,
                         int piecelen)
@@ -7010,12 +7022,14 @@ b_buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     /* this is the constructor of the type implemented in minibuffer.h */
     CDataObject *cd;
     Py_ssize_t size = -1;
+    int explicit_size;
     static char *keywords[] = {"cdata", "size", NULL};
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|n:buffer", keywords,
                                      &CData_Type, &cd, &size))
         return NULL;
 
+    explicit_size = size >= 0;
     if (size < 0)
         size = _cdata_var_byte_size(cd);
 
@@ -7039,6 +7053,20 @@ b_buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
                      cd->c_type->ct_name);
         return NULL;
     }
+
+    if (explicit_size && CDataOwn_Check(cd)) {
+        Py_ssize_t size_max = cdataowning_size_bytes(cd);
+        if (size > size_max) {
+            char msg[256];
+            sprintf(msg, "ffi.buffer(cdata, bytes): creating a buffer of %llu "
+                         "bytes over a cdata that owns only %llu bytes.  This "
+                         "will crash if you access the extra memory",
+                         (unsigned PY_LONG_LONG)size,
+                         (unsigned PY_LONG_LONG)size_max);
+            if (PyErr_WarnEx(PyExc_UserWarning, msg, 1))
+                return NULL;
+        }
+    }
     /*WRITE(cd->c_data, size)*/
     return minibuffer_new(cd->c_data, size, (PyObject *)cd);
 }

c/libffi_arm64/README

@@ -1,5 +1,5 @@
-Libffi package for ARM64 is copied from cpython binary dependencies
+Please run build_libffi.bat to generate the libffi static library and header files
 
-https://github.com/python/cpython-bin-deps/archive/libffi.zip
+Environment variable LIBFFI_SOURCE needs to be set to libffi source before invoking the script.
 
-The library file has been renamed from libffi-7.lib to ffi.lib to avoid special casing
+Libffi source can be downloaded from https://github.com/libffi/libffi

c/libffi_arm64/build_libffi.bat

@@ -0,0 +1,71 @@
+@echo off
+goto :Run
+ 
+:Usage
+echo.
+echo Before running prepare_libffi.bat
+echo .
+echo   LIBFFI_SOURCE environment variable must be set to the location of libffi source
+echo   Source can be checked out from https://github.com/libffi/libffi
+echo.
+echo   Cygwin needs to be installed (Invoke with --install-cygwin to install)
+echo.  
+echo.  Visual Studio 2017 or newer with ARM64 toolchain needs to be installed
+:Run
+set INSTALL_CYGWIN=
+ 
+:CheckOpts
+if "%1"=="" goto :CheckOptsDone
+if /I "%1"=="-?" goto :Usage
+if /I "%1"=="--install-cygwin" (set INSTALL_CYGWIN=1) & shift & goto :CheckOpts
+goto :Usage
+ 
+:CheckOptsDone
+ 
+if "%INSTALL_CYGWIN%"=="1" call :InstallCygwin
+ 
+REM Set build variables
+ 
+set BUILD=i686-pc-cygwin
+set HOST=aarch64-w64-cygwin
+if NOT DEFINED SH if exist c:\cygwin\bin\sh.exe set SH=c:\cygwin\bin\sh.exe
+ 
+REM Initialise ARM64 build environment
+ 
+if NOT DEFINED VCVARSALL (
+  for /F "tokens=*" %%i in ('"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -property installationPath -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64') DO @(set VCVARSALL="%%i\VC\Auxiliary\Build\vcvarsall.bat")
+)
+if ^%VCVARSALL:~0,1% NEQ ^" SET VCVARSALL="%VCVARSALL%"
+call %VCVARSALL% x86_arm64
+pushd %LIBFFI_SOURCE%
+%SH% --login -lc "cygcheck -dc cygwin"
+set GET_MSVCC=%SH% -lc "cd $LIBFFI_SOURCE; export MSVCC=`/usr/bin/find $PWD -name msvcc.sh`; echo ${MSVCC};"
+FOR /F "usebackq delims==" %%i IN (`%GET_MSVCC%`) do @set MSVCC=%%i
+set MSVCC=%MSVCC% -marm64
+
+echo Configuring and building libffi for ARM64
+ 
+%SH% -lc "(cd $LIBFFI_SOURCE; ./autogen.sh)"
+%SH% -lc "(cd $LIBFFI_SOURCE; ./configure CC='%MSVCC%' CXX='%MSVCC%' LD='link' CPP='cl -nologo -EP' CXXCPP='cl -nologo -EP' CPPFLAGS='-DFFI_BUILDING_DLL'  NM='dumpbin -symbols' STRIP=':' --build=$BUILD --host=$HOST --enable-static --disable-shared)"
+%SH% -lc "(cd $LIBFFI_SOURCE; cp src/aarch64/ffitarget.h include)"
+%SH% -lc "(cd $LIBFFI_SOURCE; make)"
+
+set LIBFFI_OUT=%~dp0
+
+echo copying files to %LIBFFI_OUT%
+if not exist %LIBFFI_OUT%\include (md %LIBFFI_OUT%\include)
+copy %LIBFFI_SOURCE%\%HOST%\.libs\libffi.lib %LIBFFI_OUT%\ffi.lib || exit /B 1
+copy %LIBFFI_SOURCE%\%HOST%\fficonfig.h %LIBFFI_OUT%\include || exit /B 1
+copy %LIBFFI_SOURCE%\%HOST%\include\*.h %LIBFFI_OUT%\include || exit /B 1
+popd
+exit /B
+ 
+:InstallCygwin
+setlocal
+set CYG_ROOT=C:/cygwin
+set CYG_CACHE=C:/cygwin/var/cache/setup
+set CYG_MIRROR=http://mirrors.kernel.org/sourceware/cygwin/
+powershell -c "Invoke-WebRequest https://cygwin.com/setup-x86.exe -OutFile setup.exe"
+setup.exe -qgnNdO -R "%CYG_ROOT%" -s "%CYG_MIRROR%" -l "%CYG_CACHE%" -P dejagnu -P autoconf -P automake -P libtool -P make
+endlocal
+exit /B