Merge branch 'release'

Author: thewtex

scripts/dockcross-manylinux-build-module-wheels.sh

@@ -6,10 +6,10 @@
 # Versions can be restricted by passing them in as arguments to the script
 # For example,
 #
-#   scripts/dockcross-manylinux-build-module-wheels.sh cp35
+#   scripts/dockcross-manylinux-build-module-wheels.sh cp39
 
 # Generate dockcross scripts
-docker run --rm dockcross/manylinux1-x64:20200416-a6b6635 > /tmp/dockcross-manylinux-x64
+docker run --rm dockcross/manylinux1-x64:20201015-96d8741 > /tmp/dockcross-manylinux-x64
 chmod u+x /tmp/dockcross-manylinux-x64
 
 script_dir=$(cd $(dirname $0) || exit 1; pwd)

scripts/dockcross-manylinux-build-wheels.sh

@@ -8,7 +8,7 @@
 #   scripts/dockcross-manylinux-build-wheels.sh cp35
 
 # Generate dockcross scripts
-docker run --rm dockcross/manylinux1-x64:20200416-a6b6635 > /tmp/dockcross-manylinux-x64
+docker run --rm dockcross/manylinux1-x64:20201015-96d8741 > /tmp/dockcross-manylinux-x64
 chmod u+x /tmp/dockcross-manylinux-x64
 
 script_dir=$(cd $(dirname $0) || exit 1; pwd)

scripts/internal/manylinux-build-common.sh

@@ -14,7 +14,7 @@ if [[ $# -eq 0 ]]; then
   PYBIN=(/opt/python/*/bin)
   PYBINARIES=()
   for version in "${PYBIN[@]}"; do
-    if [[  ${version} == *"cp35"* || ${version} == *"cp36"* || ${version} == *"cp37"* || ${version} == *"cp38"*  ]]; then
+    if [[  ${version} == *"cp35"* || ${version} == *"cp36"* || ${version} == *"cp37"* || ${version} == *"cp38"* || ${version} == *"cp39"* ]]; then
       PYBINARIES+=(${version})
     fi
   done

scripts/internal/windows_build_common.py

@@ -2,8 +2,9 @@
 
 from subprocess import check_call
 import os
+import shutil
 
-DEFAULT_PY_ENVS = ["35-x64", "36-x64", "37-x64", "38-x64"]
+DEFAULT_PY_ENVS = ["35-x64", "36-x64", "37-x64", "38-x64", "39-x64"]
 
 SCRIPT_DIR = os.path.dirname(__file__)
 ROOT_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
@@ -32,6 +33,8 @@ def venv_paths(python_version):
     pip = os.path.join(venv_dir, "Scripts", "pip.exe")
 
     ninja_executable = os.path.join(venv_dir, "Scripts", "ninja.exe")
+    if not os.path.exists(ninja_executable):
+        ninja_executable = shutil.which("ninja.exe")
     print("NINJA_EXECUTABLE:%s" % ninja_executable)
 
     # Update PATH

scripts/macpython-build-common.sh

@@ -9,7 +9,7 @@ MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
 
 # Versions can be restricted by passing them in as arguments to the script
 # For example,
-# macpython-build-wheels.sh 3.5
+# macpython-build-wheels.sh 3.9
 if [[ $# -eq 0 ]]; then
   PYBINARIES=(${MACPYTHON_PY_PREFIX}/*)
 else

scripts/macpython-build-wheels.sh

@@ -5,7 +5,7 @@
 # Versions can be restricted by passing them in as arguments to the script
 # For example,
 #
-#   scripts/macpython-build-wheels.sh 3.5
+#   scripts/macpython-build-wheels.sh 3.9
 
 # -----------------------------------------------------------------------
 # These variables are set in common script:

scripts/macpython-install-python.sh

@@ -43,14 +43,15 @@ GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
 DOWNLOADS_SDIR=downloads
 WORKING_SDIR=working
 
-# As of 24 Oct 2019 - latest Python of each version with binary download
+# As of 24 September 2020 - latest Python of each version with binary download
 # available.
 # See: https://www.python.org/downloads/mac-osx/
-LATEST_2p7=2.7.17
+LATEST_2p7=2.7.18
 LATEST_3p5=3.5.4
 LATEST_3p6=3.6.8
 LATEST_3p7=3.7.6
-LATEST_3p8=3.8.0
+LATEST_3p8=3.8.6
+LATEST_3p9=3.9.0
 
 
 function check_python {
@@ -107,20 +108,52 @@ function fill_pyver {
         echo $ver
     elif [ $ver == 2 ] || [ $ver == "2.7" ]; then
         echo $LATEST_2p7
-    elif [ $ver == 3 ] || [ $ver == "3.7" ]; then
+    elif [ $ver == 3 ] || [ $ver == "3.9" ]; then
+        echo $LATEST_3p9
+    elif [ $ver == "3.8" ]; then
+        echo $LATEST_3p8
+    elif [ $ver == "3.7" ]; then
         echo $LATEST_3p7
     elif [ $ver == "3.6" ]; then
         echo $LATEST_3p6
     elif [ $ver == "3.5" ]; then
         echo $LATEST_3p5
-    elif [ $ver == "3.8" ]; then
-        echo $LATEST_3p8
     else
         echo "Can't fill version $ver" 1>&2
         exit 1
     fi
 }
 
+function macpython_sdk_list_for_version {
+    # return a list of SDK targets supported for a given CPython version
+    # Parameters
+    #   $py_version (python version in major.minor.extra format)
+    # eg
+    #  macpython_sdks_for_version 2.7.15
+    #  >> 10.6 10.9
+    local _ver=$(fill_pyver $1)
+    local _major=${_ver%%.*}
+    local _return
+
+    if [ "$_major" -eq "2" ]; then
+        [ $(lex_ver $_ver) -lt $(lex_ver 2.7.18) ] && _return="10.6"
+        [ $(lex_ver $_ver) -ge $(lex_ver 2.7.15) ] && _return="$_return 10.9"
+    elif [ "$_major" -eq "3" ]; then
+        [ $(lex_ver $_ver) -lt $(lex_ver 3.8)    ] && _return="10.6"
+        [ $(lex_ver $_ver) -ge $(lex_ver 3.6.5)  ] && _return="$_return 10.9"
+    else
+        echo "Error version=${_ver}, expecting 2.x or 3.x" 1>&2
+        exit 1
+    fi
+    echo $_return
+}
+
+function macpython_sdk_for_version {
+    # assumes the output of macpython_sdk_list_for_version is a list
+    # of SDK versions XX.Y in sorted order, eg "10.6 10.9" or "10.9"
+    echo $(macpython_sdk_list_for_version $1) | awk -F' ' '{print $NF}'
+}
+
 function pyinst_ext_for_version {
     # echo "pkg" or "dmg" depending on the passed Python version
     # Parameters
@@ -148,13 +181,11 @@ function pyinst_fname_for_version {
     # Parameters
     #   $py_version (Python version in major.minor.extra format)
     #   $py_osx_ver: {major.minor | not defined}
-    #       if defined, the macOS version that Python is built for, e.g.
-    #       "10.6" or "10.9", if not defined, uses the default
-    #       MACPYTHON_DEFAULT_OSX
-    #       Note: this is the version the Python is built for, and hence
-    #       the min version supported, NOT the version of the current system
+    #       if defined, the minimum macOS SDK version that Python is
+    #       built for, eg: "10.6" or "10.9", if not defined, infers
+    #       this from $py_version using macpython_sdk_for_version
     local py_version=$1
-    local py_osx_ver=${2:-$MACPYTHON_DEFAULT_OSX}
+    local py_osx_ver=${2:-$(macpython_sdk_for_version $py_version)}
     local inst_ext=$(pyinst_ext_for_version $py_version)
     echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
 }
@@ -229,7 +260,7 @@ function macpython_impl_for_version {
     #         "pypy-5.4" for PyPy
     local version=$1
     check_var $1
-    if [[ "$version" =~ pypy-([0-9\.]+) ]]; then
+    if [[ "$version" =~ ^pypy ]]; then
         echo pp
     elif [[ "$version" =~ ([0-9\.]+) ]]; then
         echo cp
@@ -257,17 +288,17 @@ function install_macpython {
     # Install Python and set $PYTHON_EXE to the installed executable
     # Parameters:
     #     $version : [implementation-]major[.minor[.patch]]
-    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
+    #         The Python implementation to install, e.g. "3.6", "pypy-5.4" or "pypy3.6-7.2"
     #     $py_osx_ver: {major.minor | not defined}
     #       if defined, the macOS version that CPython is built for, e.g.
     #       "10.6" or "10.9". Ignored for PyPy
     local version=$1
     local py_osx_ver=$2
     local impl=$(macpython_impl_for_version $version)
-    local stripped_ver=$(strip_macpython_ver_prefix $version)
     if [[ "$impl" == "pp" ]]; then
-        install_mac_pypy $stripped_ver
+        install_pypy $version
     elif [[ "$impl" == "cp" ]]; then
+        local stripped_ver=$(strip_macpython_ver_prefix $version)
         install_mac_cpython $stripped_ver $py_osx_ver
     else
         echo "Unexpected Python impl: ${impl}"
@@ -304,43 +335,9 @@ function install_mac_cpython {
     if [ -e "$inst_cmd" ]; then
         sh "$inst_cmd"
     fi
-}
-
-function install_mac_pypy {
-    # Installs pypy.org PyPy
-    # Parameter $version
-    # Version given in major or major.minor or major.minor.micro e.g
-    # "3" or "3.7" or "3.7.1".
-    # sets $PYTHON_EXE variable to python executable
-    local py_version=$(fill_pypy_ver $1)
-    local py_build=$(get_pypy_build_prefix $py_version)$py_version-osx64
-    local py_zip=$py_build.tar.bz2
-    local zip_path=$DOWNLOADS_SDIR/$py_zip
-    mkdir -p $DOWNLOADS_SDIR
-    wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR
-    untar $zip_path
-    PYTHON_EXE=$(realpath $py_build/bin/pypy)
-}
-
-function install_pip {
-    # Generic install pip
-    # Gets needed version from version implied by $PYTHON_EXE
-    # Installs pip into python given by $PYTHON_EXE
-    # Assumes pip will be installed into same directory as $PYTHON_EXE
-    check_python
-    mkdir -p $DOWNLOADS_SDIR
-    local py_mm=`get_py_mm`
-    local get_pip_path=$DOWNLOADS_SDIR/get-pip.py
-    curl $GET_PIP_URL > $get_pip_path
-    # Travis VMS now install pip for system python by default - force install
-    # even if installed already.
-    sudo $PYTHON_EXE $get_pip_path --ignore-installed $pip_args
-    PIP_CMD="sudo $(dirname $PYTHON_EXE)/pip$py_mm"
-    # Append pip_args if present (avoiding trailing space cf using variable
-    # above).
-    if [ -n "$pip_args" ]; then
-        PIP_CMD="$PIP_CMD $pip_args"
-    fi
+    PIP_CMD="$MACPYTHON_PY_PREFIX/$py_mm/bin/python$py_mm -m pip"
+    $PIP_CMD install --upgrade pip
+    export PIP_CMD
 }
 
 function install_virtualenv {
@@ -375,14 +372,12 @@ function make_workon_venv {
 # Remove previous versions
 sudo rm -rf ${MACPYTHON_FRAMEWORK}
 
-for pyversion in $LATEST_3p5 $LATEST_3p6 $LATEST_3p7; do
+for pyversion in $LATEST_3p6 $LATEST_3p7; do
   install_macpython $pyversion 10.6
-  install_pip
   install_virtualenv
 done
 
-for pyversion in $LATEST_3p8; do
+for pyversion in $LATEST_3p8 $LATEST_3p9; do
   install_macpython $pyversion 10.9
-  install_pip
   install_virtualenv
 done

scripts/windows-download-cache-and-build-module-wheels.ps1

@@ -15,4 +15,4 @@ sz x doxygen-1.8.11.windows.bin.zip -oC:\P\doxygen -aoa -r
 Invoke-WebRequest -Uri "https://data.kitware.com/api/v1/file/5bbf87ba8d777f06b91f27d6/download/grep-win.zip" -OutFile "grep-win.zip"
 sz x grep-win.zip -oC:\P\grep -aoa -r
 $env:Path += ";C:\P\grep"
-C:\Python35-x64\python.exe C:\P\IPP\scripts\windows_build_module_wheels.py
+C:\Python37-x64\python.exe C:\P\IPP\scripts\windows_build_module_wheels.py

scripts/windows_build_wheels.py

@@ -251,8 +251,10 @@ def build_wheels(py_envs=DEFAULT_PY_ENVS, single_wheel=False,
 
         cmake_executable = "cmake.exe"
         tools_venv = os.path.join(ROOT_DIR, "venv-" + py_envs[0])
-        pip_install(tools_venv, "ninja")
-        ninja_executable = os.path.join(tools_venv, "Scripts", "ninja.exe")
+        ninja_executable = shutil.which('ninja.exe')
+        if ninja_executable is None:
+            pip_install(tools_venv, "ninja")
+            ninja_executable = os.path.join(tools_venv, "Scripts", "ninja.exe")
 
         # Build standalone project and populate archive cache
         check_call([
@@ -268,7 +270,9 @@ def build_wheels(py_envs=DEFAULT_PY_ENVS, single_wheel=False,
     # Compile wheels re-using standalone project and archive cache
     for py_env in py_envs:
         tools_venv = os.path.join(ROOT_DIR, "venv-" + py_env)
-        pip_install(tools_venv, "ninja")
+        ninja_executable = shutil.which('ninja.exe')
+        if ninja_executable is None:
+            pip_install(tools_venv, "ninja")
         build_wheel(py_env, single_wheel=single_wheel,
             cleanup=cleanup, wheel_names=wheel_names,
             cmake_options=cmake_options)