Fix CVE-2022-0391 for Python's urlparse. (#169)

* Fixed CVE-2022-0391 for Python's urlparse.

* Bash checks for the chevahbs scripts.

* Removed some unused variables from chevahbs scripts.

* Try generic musl build.

* Fixed musl version check for 1.2 and newer.

* Updated OpenSSL sources to 1.1.1q

* Updated OpenSSL 1.1.1 version in our scripts and docs.

* Actually updated the docs for OpenSSL 1.1.1q.

* Updated own tests for generic musl Linux build.

* Backported OpenSSL build fix for macOS.

* Ignore dparse issue for now.

* Updated cffi and psutil to the latest versions.

* Updated cffi sources to 1.15.1.

* Try a different psutil check.

* Pin psutil to version 5.9.0 on generic Linux builds.

* Build generic musl version on Alpine 3.12.

* Use a saved paxctl on Alpine 3.12.

* Save paxctl on Alpine 3.12 in an already existing path dir.

* Changes after own review.

* Updated and reorganized external deps sheets.

* Updated list of RHEL clones from server repo.

* Temporarily disabled ARM64 builds.

* More Alpine-related cleanups and fixes.

* One more Alpine-related fix.

Author: dumol

.github/workflows/bare.yaml

@@ -30,8 +30,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # ARM64 is currently our Pine64 board running Ubuntu 16.04.
-        runs-on: [ ubuntu-20.04, ubuntu-18.04, ARM64 ]
+        # ARM64 is currently our virtualized Ubuntu 16.04 image.
+        #runs-on: [ ubuntu-20.04, ubuntu-18.04, ARM64 ]
+        runs-on: [ ubuntu-20.04, ubuntu-18.04 ]
     timeout-minutes: 90
     steps:
     - name: Prepare OS

.github/workflows/docker.yaml

@@ -32,7 +32,7 @@ jobs:
       matrix:
         # CentOS 5.11 setup was saved as an image pushed to Docker Hub. See the
         # Overview section at https://hub.docker.com/r/proatria/centos for details.
-        container: [ 'alpine:3.14', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ]
+        container: [ 'alpine:3.12', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ]
     timeout-minutes: 30
     steps:
 
@@ -41,7 +41,9 @@ jobs:
       if: startsWith(matrix.container, 'alpine')
       run: |
         apk upgrade -U
-        apk add git curl bash gcc make m4 automake libtool patch libffi-dev zlib-dev openssl-dev musl-dev lddtree shadow sudo openssh-client paxctl
+        apk add git curl bash gcc make m4 automake libtool patch libffi-dev zlib-dev openssl-dev musl-dev lddtree shadow sudo openssh-client
+        curl --output /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12
+        chmod +x /usr/local/bin/paxctl
 
     # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g.
     - name: CentOS 8.2 setup

brink.sh

@@ -33,10 +33,11 @@
 # command used to execute Python inside the newly virtual environment.
 #
 
-# Script initialization.
-set -o nounset
-set -o errexit
-set -o pipefail
+# Bash checks
+set -o nounset    # always check if variables exist
+set -o errexit    # always exit on error
+set -o errtrace   # trap errors in functions as well
+set -o pipefail   # don't ignore exit codes when piping output
 
 # Initialize default value.
 COMMAND=${1-''}
@@ -536,7 +537,7 @@ install_dependencies(){
 # If it's too old, exit with a nice informative message.
 # If it's supported, return through eval the version numbers to be used for
 # naming the package, for example: '8' for RHEL 8.2, '2004' for Ubuntu 20.04,
-# '314' for Alpine Linux 3.14, '71' for AIX 7.1, '114' for Solaris 11.4.
+# '71' for AIX 7.1, '114' for Solaris 11.4.
 #
 check_os_version() {
     # First parameter should be the human-readable name for the current OS.
@@ -584,7 +585,7 @@ check_os_version() {
         if [ "$OS" = "Linux" ]; then
             # For old and/or unsupported Linux distros there's a second chance!
             (>&2 echo "the generic Linux runtime is used, if possible.")
-            check_linux_glibc
+            check_linux_libc
         else
             (>&2 echo "there is currently no support.")
             exit 13
@@ -597,16 +598,42 @@ check_os_version() {
 
 #
 # For old unsupported Linux distros (some with no /etc/os-release) and for other
-# unsupported Linux distros (eg. Arch), we check if the system is glibc-based.
+# unsupported Linux distros, we check if the system is based on glibc or musl.
 # If so, we use a generic code path that builds everything statically,
-# including OpenSSL, thus only requiring glibc 2.X, where X differs by arch.
+# including OpenSSL, thus only requiring glibc or musl.
 #
-check_linux_glibc() {
+check_linux_libc() {
+    local ldd_output_file=".chevah_libc_version"
+    set +o errexit
+
+    command -v ldd > /dev/null
+    if [ $? -ne 0 ]; then
+        (>&2 echo "No ldd binary found, can't check for glibc!")
+        exit 18
+    fi
+
+    ldd --version > $ldd_output_file 2>&1
+    egrep "GNU\ libc|GLIBC" $ldd_output_file > /dev/null
+    if [ $? -eq 0 ]; then
+        check_glibc_version
+    else
+        egrep ^"musl\ libc" $ldd_output_file > /dev/null
+        if [ $? -eq 0 ]; then
+            check_musl_version
+        else
+            (>&2 echo "Unknown libc reported by ldd... Unsupported Linux.")
+            rm $ldd_output_file
+            exit 19
+        fi
+    fi
+
+    set -o errexit
+}
+
+check_glibc_version(){
     local glibc_version
     local glibc_version_array
     local supported_glibc2_version
-    # Output to a file to avoid "write error: Broken pipe" with grep/head.
-    local ldd_output_file=".chevah_glibc_version"
 
     # Supported minimum minor glibc 2.X versions for various arches.
     # For x64, we build on CentOS 5.11 (Final) with glibc 2.5.
@@ -628,21 +655,6 @@ check_linux_glibc() {
     echo "No specific runtime for the current distribution / version / arch."
     echo "Minimum glibc version for this arch: 2.${supported_glibc2_version}."
 
-    set +o errexit
-
-    command -v ldd > /dev/null
-    if [ $? -ne 0 ]; then
-        (>&2 echo "No ldd binary found, can't check for glibc!")
-        exit 18
-    fi
-
-    ldd --version > $ldd_output_file
-    egrep "GNU\ libc|GLIBC" $ldd_output_file > /dev/null
-    if [ $? -ne 0 ]; then
-        (>&2 echo "No glibc reported by ldd... Unsupported Linux libc?")
-        exit 19
-    fi
-
     # Tested with glibc 2.5/2.11.3/2.12/2.23/2.28-31 and eglibc 2.13/2.19.
     glibc_version=$(head -n 1 $ldd_output_file | rev | cut -d\  -f1 | rev)
     rm $ldd_output_file
@@ -668,22 +680,59 @@ check_linux_glibc() {
         echo "All is good. Detected glibc version: ${glibc_version}."
     fi
 
-    set -o errexit
-
-    # glibc 2 detected, we set $OS for a generic Linux build.
+    # Supported glibc version detected, set $OS for a generic glibc Linux build.
     OS="lnx"
 }
 
+check_musl_version(){
+    local musl_version
+    local musl_version_array
+    local supported_musl11_version=24
+
+    echo "No specific runtime for the current distribution / version / arch."
+    echo "Minimum musl version for this arch: 1.1.${supported_musl11_version}."
+
+    # Tested with musl 1.1.24/1.2.2.
+    musl_version=$(egrep ^Version $ldd_output_file | cut -d\  -f2)
+    rm $ldd_output_file
+
+    if [[ $musl_version =~ [^[:digit:]\.] ]]; then
+        (>&2 echo "Musl version should only have numbers and periods, but:")
+        (>&2 echo "    \$musl_version=$musl_version")
+        exit 25
+    fi
+
+    IFS=. read -a musl_version_array <<< "$musl_version"
+
+    if [ ${musl_version_array[0]} -lt 1 -o ${musl_version_array[1]} -lt 1 ];then
+        (>&2 echo "Only musl 1.1 or greater supported! Detected: $musl_version")
+        exit 26
+    fi
+
+    # Decrement supported_musl11_version if building against an older musl.
+    if [ ${musl_version_array[0]} -eq 1 -a ${musl_version_array[1]} -eq 1 \
+        -a ${musl_version_array[2]} -lt ${supported_musl11_version} ]; then
+        (>&2 echo "NOT good. Detected version is older: ${musl_version}!")
+        exit 27
+    else
+        echo "All is good. Detected musl version: ${musl_version}."
+    fi
+
+    # Supported musl version detected, set $OS for a generic musl Linux build.
+    OS="lnx_musl"
+}
+
 #
-# For glibc-based Linux distros, after checking if current version is
-# supported with check_os_version(), $OS might already be set to "lnx"
-# if current version is too old, through check_linux_glibc().
+# For Linux distros with a supported libc, after checking if current version is
+# supported with check_os_version(), $OS might be set to something like "lnx"
+# if current version is too old, through check_linux_libc() and its subroutines.
 #
 set_os_if_not_generic() {
     local distro_name="$1"
     local distro_version="$2"
 
-    if [ "$OS" != "lnx" ]; then
+    # Check if OS starts with "lnx", to match "lnx_musl" too, just in case.
+    if [ "${OS#lnx}" = "$OS" ]; then
         OS="${distro_name}${distro_version}"
     fi
 }
@@ -705,15 +754,15 @@ detect_os() {
             if [ ! -f /etc/os-release ]; then
                 # No /etc/os-release file present, so we don't support this
                 # distro, but check for glibc, the generic build should work.
-                check_linux_glibc
+                check_linux_libc
             else
                 source /etc/os-release
                 linux_distro="$ID"
                 distro_fancy_name="$NAME"
                 # Some rolling-release distros (eg. Arch Linux) have
                 # no VERSION_ID here, so don't count on it unconditionally.
                 case "$linux_distro" in
-                    rhel|centos|ol)
+                    rhel|centos|almalinux|rocky|ol)
                         os_version_raw="$VERSION_ID"
                         check_os_version "Red Hat Enterprise Linux" 8 \
                             "$os_version_raw" os_version_chevah
@@ -728,23 +777,17 @@ detect_os() {
                         # 04 or first two digits are uneven, use generic build.
                         if [ ${os_version_chevah%%04} == ${os_version_chevah} \
                             -o $(( ${os_version_chevah:0:2} % 2 )) -ne 0 ]; then
-                            check_linux_glibc
+                            check_linux_libc
                         elif [ ${os_version_chevah} == "2204" ]; then
                             # OpenSSL 3.0.x not supported by cryptography 3.3.x.
-                            check_linux_glibc
+                            check_linux_libc
                         fi
                         set_os_if_not_generic "ubuntu" $os_version_chevah
                         ;;
-                    alpine)
-                        os_version_raw="$VERSION_ID"
-                        check_os_version "$distro_fancy_name" 3.12 \
-                            "$os_version_raw" os_version_chevah
-                        set_os_if_not_generic "alpine" $os_version_chevah
-                        ;;
                     *)
                         # Supported distros with unsupported OpenSSL versions or
                         # distros not specifically supported: SLES, Debian, etc.
-                        check_linux_glibc
+                        check_linux_libc
                         ;;
                 esac
             fi

chevah_build

@@ -6,9 +6,11 @@
 # test
 # compat (for the compat repo tests)
 
-set -o nounset
-set -o errexit
-set -o pipefail
+# Bash checks
+set -o nounset    # always check if variables exist
+set -o errexit    # always exit on error
+set -o errtrace   # trap errors in functions as well
+set -o pipefail   # don't ignore exit codes when piping output
 
 PYTHON_BUILD_VERSION="2.7.18"
 LIBFFI_VERSION="3.4.2"
@@ -17,14 +19,14 @@ BZIP2_VERSION="1.0.8"
 # We statically build the BSD libedit on selected platforms to get the
 # readline module available without linking to the GPL-only readline libs.
 LIBEDIT_VERSION="20170329-3.1"
-OPENSSL_VERSION="1.1.1n"
+OPENSSL_VERSION="1.1.1q"
 SQLITE_VERSION="3.37.2"
 
 # Python modules versions to be used everywhere possible.
 PYSQLITE_VERSION="2.8.3"
-CFFI_VERSION="1.15.0"
+CFFI_VERSION="1.15.1"
 SCANDIR_VERSION="1.10.0"
-PSUTIL_VERSION="5.9.0"
+PSUTIL_VERSION="5.9.2"
 SUBPROCESS32_VERSION="3.5.4"
 
 # Versions no longer upgradable because of Python 2 deprecation.
@@ -33,8 +35,8 @@ PYOPENSSL_VERSION="21.0.0"
 # Backported fix for https://github.com/pypa/pip/issues/9827
 # at https://github.com/chevah/pip/tree/20.3.4chevah.
 PIP_VERSION="20.3.4chevah1"
-# For pip <21.1 and click <8.0.0.
-SAFETY_IGNORED_OPTS="-i 40291 -i 47833"
+# For pip <21.1, click <8.0.0, dparse <0.5.2.
+SAFETY_IGNORED_OPTS="-i 40291 -i 47833 -i 50571"
 # setuptools 44.x is the last series to support Python 2.7.
 # More at https://github.com/pypa/setuptools/pull/1955.
 SETUPTOOLS_VERSION="44.1.1"
@@ -124,7 +126,6 @@ BUILD_FOLDER='build'
 COMMON_PKGS="gcc make m4 automake libtool patch"
 DEB_PKGS="$COMMON_PKGS git libffi-dev zlib1g-dev libncurses5-dev libssl-dev"
 RPM_PKGS="$COMMON_PKGS git libffi-devel zlib-devel ncurses-devel openssl-devel"
-APK_PKGS="$COMMON_PKGS git zlib-dev musl-dev openssl-dev linux-headers lddtree"
 # No automated Windows package management, but here's what it's needed.
 CHOCO_PKGS="vcpython27 make git StrawberryPerl nasm 7zip"
 
@@ -279,7 +280,7 @@ case $OS in
         # libffi not available in the base system, only as port/package.
         export BUILD_LIBFFI="yes"
         ;;
-    lnx)
+    lnx*)
         export CC="gcc"
         export CXX="g++"
         export MAKE="make"
@@ -294,18 +295,19 @@ case $OS in
         export PATH="/usr/local/bin:$PATH"
         # In particular, Perl's Test::Simple and its deps are required.
         execute perl -MTest::Simple -e 1
-        ;;
-    alpine*)
-        # Do not depend on libffi and ncurses-libs Alpine packages.
-        # It's better to run on minimal Alpine containers.
-        export BUILD_LIBFFI="yes"
-        export BUILD_LIBEDIT="no"
-        export CC="gcc"
-        export CXX="g++"
-        export MAKE="make"
+        # Version 5.9.2 of psutil not working properly on CentOS 5.
+        PIP_LIBRARIES="\
+            cryptography==${CRYPTOGRAPHY_VERSION} \
+            pyOpenSSL==${PYOPENSSL_VERSION} \
+            scandir==${SCANDIR_VERSION} \
+            subprocess32==${SUBPROCESS32_VERSION} \
+            bcrypt==${BCRYPT_VERSION} \
+            psutil=="5.9.0" \
+            setproctitle==${SETPROCTITLE_VERSION}
+            "
         ;;
     *)
-        # Only supported Linux distributions other than Alpine should be left.
+        # Only supported Linux distributions should be left.
         export CC="gcc"
         export CXX="g++"
         export MAKE="make"
@@ -367,13 +369,8 @@ check_dependencies() {
             packages=$RPM_PKGS
             check_command="rpm --query"
             ;;
-        alpine*)
-            # Alpine 3.9 switched back to OpenSSL as default.
-            packages=$APK_PKGS
-            check_command="apk info -q -e"
-            ;;
         # On remaining OS'es we just check for some of the needed commands.
-        lnx)
+        lnx*)
             # Generic Linux builds need Perl 5.10.0+ for building OpenSSL.
             # For testing OpenSSL, Test::More 0.96 or newer is needed.
             packages="$CC make m4 git patch perl"
@@ -485,8 +482,7 @@ command_build() {
                 ${INSTALL_FOLDER}/lib/pkgconfig/:${PKG_CONFIG_PATH}"
         elif [ "${OS%lnx*}" = "" -o "${OS%macos*}" = "" ]; then
             # Passing -rpath to ldd is needed for building cryptography w/ pip.
-            export LDFLAGS="-L${INSTALL_FOLDER}/lib64/ \
-                -Wl,-rpath,${INSTALL_FOLDER}/lib64/ ${LDFLAGS}"
+            export LDFLAGS="-Wl,-rpath,${INSTALL_FOLDER}/lib/ ${LDFLAGS}"
             export PKG_CONFIG_PATH="\
                 ${INSTALL_FOLDER}/lib64/pkgconfig/:${PKG_CONFIG_PATH}"
         fi