AIX fixes for building on IBM Cloud. (#151)

* Install OpenSSL manually under AIX.

* Check for more commands under AIX.

* Limit build parallelism on AIX.

* Use bin.chevah.com instead of binary.chevah.com.

* Updated deps spreadsheets.

* Testing Start-BitsTransfer.

* Work around the 500 error on Windows.

* Use the real curl on Windows.

* Reenabled compat tests under Docker.

* Moved chevah user creation back to its place under Docker.

* Minor improvements inspired by chevah/compat.

* Unbreak package uploading under Windows.

Author: dumol

.github/workflows/bare.yaml

@@ -58,25 +58,25 @@ jobs:
 
     - name: Detect OS and build Python
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./brink.sh detect_os
         ./chevah_build build
 
     - name: Own tests
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./chevah_build test
 
     - name: Compat tests
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./chevah_build compat
 
     # Upload using a (per-OS selected) sftp command, then show final links.
     - name: Upload testing package
       run: |
         mkdir -pv ~/.ssh/
-        cd python-package/
+        cd $CHEVAH_REPO
         touch priv_key
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
@@ -122,7 +122,7 @@ jobs:
 
     - name: Detect OS and build Python
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./brink.sh detect_os
         ./chevah_build build
 
@@ -135,18 +135,18 @@ jobs:
 
     - name: Own tests
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./chevah_build test
 
     - name: Compat tests
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./chevah_build compat
 
     - name: Upload testing package
       run: |
         mkdir -pv ~/.ssh/
-        cd python-package/
+        cd $CHEVAH_REPO
         touch priv_key
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
@@ -174,9 +174,11 @@ jobs:
     - name: Prepare OS
       shell: powershell
       run: |
-        chocolatey install --yes --no-progress make nasm 7zip
+        chocolatey install --yes --no-progress make nasm 7zip curl
         # There's no vcpython27 choco pkg since Microsoft removed the installer.
-        start-bitstransfer https://binary.chevah.com/third-party-stuff/VCForPython27.msi
+        #Start-BitsTransfer https://bin.chevah.com:20443/third-party-stuff/VCForPython27.msi
+        # Pending upstream fix for SFTPPlus, we use the real curl for now.
+        curl.exe -O https://bin.chevah.com:20443/third-party-stuff/VCForPython27.msi
         msiexec /quiet /i VCForPython27.msi
 
     - name: Clone sources independently
@@ -196,20 +198,20 @@ jobs:
     - name: Detect OS and build Python
       shell: bash
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./brink.sh detect_os
         ./chevah_build build
 
     - name: Own tests
       shell: bash
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./chevah_build test
 
     - name: Compat tests
       shell: bash
       run: |
-        cd python-package/
+        cd $CHEVAH_REPO
         ./chevah_build compat
 
     # To use an RSA key with SFTPPlus, install upstream OpenSSH package,
@@ -229,7 +231,7 @@ jobs:
     - name: Upload testing package
       shell: bash
       run: |
-        cd python-package
+        cd $CHEVAH_REPO
         ./publish_dist.sh
         rm priv_key
 

.github/workflows/docker.yaml

@@ -40,7 +40,7 @@ jobs:
       run: |
         apk upgrade -U
         apk add git curl bash gcc make m4 automake libtool patch zlib-dev libffi-dev ncurses-dev linux-headers musl-dev openssl-dev lddtree shadow sudo openssh-client
-        curl -o /usr/local/bin/paxctl https://binary.chevah.com/third-party-stuff/alpine/paxctl-3.12
+        curl -o /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12
         chmod +x /usr/local/bin/paxctl
 
     # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g.
@@ -59,13 +59,13 @@ jobs:
         sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo
         sed -i s@^#baseurl=http://mirror.centos.org/centos/\$releasever/@baseurl=http://vault.centos.org/5.11/@ /etc/yum.repos.d/*.repo
         yum -y upgrade
-        # Use http://binary.chevah.com/third-party-stuff/centos5/tuxad/
+        # Use https://bin.chevah.com:20443/third-party-stuff/centos5/tuxad/
         # when tuxad.de dissapears, it has the minimum required stuff.
         rpm -i http://www.tuxad.de/rpms/tuxad-release-5-1.noarch.rpm
         yum -y install wget curl gcc44 make m4 automake libtool patch sudo which openssh-clients
         ln -s /usr/bin/gcc44 /usr/local/bin/gcc
-        wget --mirror --no-parent https://binary.chevah.com/third-party-stuff/centos5/endpoint/
-        cd binary.chevah.com/third-party-stuff/centos5/endpoint/
+        wget --mirror --no-parent https://bin.chevah.com:20443/third-party-stuff/centos5/endpoint/
+        cd bin.chevah.com\:20443/third-party-stuff/centos5/endpoint/
         rpm -i local-perl-*.rpm
         rpm -i --nodeps git{-core,}-2.5.0-1.ep.x86_64.rpm
 
@@ -97,27 +97,27 @@ jobs:
 
     - name: Detect OS and build Python
       run: |
-        cd /home/chevah/python-package
+        cd /home/chevah/$CHEVAH_REPO
         ./brink.sh detect_os
         ./chevah_build build
 
     - name: Own tests
       run: |
-        cd /home/chevah/python-package
+        cd /home/chevah/$CHEVAH_REPO
         ./chevah_build test
 
-    # And compat tests must run as regular user with sudo rights.
-    # Compat tests are disabled pending https://github.com/chevah/compat/pull/113.
-    #- name: Compat tests
-    #  run: |
-    #    chown -R chevah /home/chevah/python-package
-    #    su chevah -lc "umask 002; cd /home/chevah/python-package; ./chevah_build compat"
+    # Compat tests must run as regular user with sudo rights.
+    - name: Compat tests
+      run: |
+        chown -R chevah /home/chevah/$CHEVAH_REPO
+        cd /home/chevah/$CHEVAH_REPO
+        su chevah -c "./chevah_build compat"
 
     # Using `~/` is problematic under Docker, use `/root/`.
     - name: Upload testing package
       run: |
         mkdir -pv /root/.ssh/
-        cd /home/chevah/python-package
+        cd /home/chevah/$CHEVAH_REPO
         touch priv_key
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key

README.rst

@@ -1,6 +1,12 @@
 Python Portable Package
 =======================
 
+.. image:: https://github.com/chevah/python-package/workflows/Bare/badge.svg
+  :target: https://github.com/chevah/python-package/actions/workflows/bare.yaml
+
+.. image:: https://github.com/chevah/python-package/workflows/Docker/badge.svg
+  :target: https://github.com/chevah/python-package/actions/workflows/docker.yaml
+
 Build system for a portable Python distribution.
 
 Used by Chevah's Buildbot setup to generate and publish binary Python
@@ -10,7 +16,7 @@ Please check https://github.com/chevah/python-package/releases for
 latest packages. For older releases or platforms no longer supported,
 please check https://binary.chevah.com/production/python/.
 
-Building steps as used by Buildbot:
+Building steps:
 
 * Login to a system running on the desired platform (e.g. Ubuntu Server 20.04).
 * Get the code for this repository from GitHub.
@@ -21,19 +27,20 @@ Building steps as used by Buildbot:
 
 You can try the above steps on your own to build and test a new Python package.
 
-To have new Python packages uploaded (typically for all supported platforms),
-you should try the following, after securing access to Chevah infrastructure:
+Use ``./chevah_build help`` to discover all available commands.
 
-* ``./brink.sh test_remote group-all --properties=force_upload_production=yes
+New testing packages are uploaded automatically for green GitHub builds at
+https://bin.chevah.com:20443/testing/.
 
-Use ``./chevah_build help`` to discover all available commands.
+Production packages are available both through GitHub releases and at
+https://bin.chevah.com:20443/production/.
 
 
 Patching upstream code
 ----------------------
 
 This repository contains a lot of imported code from upstream repos:
-Python, OpenSSL, SQLite, gmp, libffi, PyCrypto, etc.
+Python, OpenSSL, SQLite, libffi, etc.
 
 You can find those sources in two sub-directories:
 
@@ -64,10 +71,10 @@ An example for creating a patch for src/python/Python-2.7.13/Lib/site.py::
 External dependencies and associated vulnerabilities
 ----------------------------------------------------
 
-This Python package builds on and links to third-party libraries.
+This Python package builds and/or links to third-party libraries.
 
-To document them for all OS'es and to list their known vulnerabilities,
-the LibreOffice spreadsheet ``external_deps.fods`` is used.
+Their known vulnerabilities are documented for all OS'es in the
+LibreOffice spreadsheet ``external_deps.fods``.
 
-For your convenience, this flat ODS file is also exported in CSV format in
+For your convenience, this flat ODS file is also exported in CSV format as
 ``external_deps.csv``.

brink.conf

@@ -1,5 +1,5 @@
 BASE_REQUIREMENTS='pip==20.2.4 chevah-brink==0.79.0 paver==1.2.4'
-PYTHON_CONFIGURATION='default@2.7.18.e26e753c'
+PYTHON_CONFIGURATION='default@2.7.18.c3cdaec'
 # For production packages there are 2 options:
 BINARY_DIST_URI='https://github.com/chevah/python-package/releases/download'
 #BINARY_DIST_URI='https://bin.chevah.com:20443/production'

chevah_build

@@ -197,6 +197,8 @@ case $OS in
         export BUILD_OPENSSL="yes"
         # 1.1.1 tests fail on AIX, use 1.0.2 with patches from Ubuntu 16.04 LTS.
         OPENSSL_VERSION="1.0.2v-chevah2"
+        # Perl's Test::Simple and its deps are required for building OpenSSL.
+        execute perl -MTest::Simple -e 1
         # cryptography 3.2.x, last version to support OpenSSL 1.0.2.
         export CRYPTOGRAPHY_ALLOW_OPENSSL_102="yes"
         PIP_LIBRARIES=$PIP_LIBRARIES_OPENSSL_102
@@ -244,6 +246,8 @@ case $OS in
         # But 10.13 has version 2.2.7, while cryptography 2.9 requires 2.7.
         # Therefore, we build OpenSSL for both stdlib and cryptography.
         export BUILD_OPENSSL="yes"
+        # Perl's Test::Simple and its deps are required for building OpenSSL.
+        execute perl -MTest::Simple -e 1
         ;;
     fbsd*)
         export CC="clang"
@@ -366,14 +370,19 @@ check_dependencies() {
             check_command="apk info -q -e"
             ;;
         # On remaining OS'es we just check for some of the needed commands.
-        macos)
-            packages="$CC make m4 libtool git patch"
-            ;;
         lnx)
             # Generic Linux builds need Perl 5.10.0+ for building OpenSSL.
             # For testing OpenSSL, Test::More 0.96 or newer is needed.
             packages="$CC make m4 git patch perl"
             ;;
+        aix*)
+            # /usr/bin/patch is not good enough, GNU patch is needed.
+            # seq is from coreutils RPM.
+            packages="git gmake patch xlc_r seq sudo"
+            ;;
+        macos)
+            packages="$CC make m4 libtool git patch"
+            ;;
         win)
             # To not get confused by MSYS2's perl, we check for wperl.
             packages="git patch wperl nasm 7z"