linux: support LibreSSL

The impetus for this is that I couldn't get OpenSSL 1.1.1c building
against musl libc. But LibreSSL is a compelling replacement for OpenSSL
and I think it is a good idea to support it. We'll want this eventually
for BSD support anyway.

This commit teaches the Linux build system to build LibreSSL and to
have Python use it. AFAICT things "just work."

Author: indygreg

LICENSE.libressl.txt

@@ -0,0 +1,132 @@
+
+  LibReSSL files are retained under the copyright of the authors. New
+  additions are ISC licensed as per OpenBSD's normal licensing policy,
+  or are placed in the public domain.
+
+  The OpenSSL code is distributed under the terms of the original OpenSSL
+  licenses which follow:
+
+  LICENSE ISSUES
+  ==============
+
+  The OpenSSL toolkit stays under a dual license, i.e. both the conditions of
+  the OpenSSL License and the original SSLeay license apply to the toolkit.
+  See below for the actual license texts.  In case of any license issues
+  related to OpenSSL please contact [email protected].
+
+  OpenSSL License
+  ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2011 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    [email protected].
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * ([email protected]).  This product includes software written by Tim
+ * Hudson ([email protected]).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young ([email protected])
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young ([email protected]).
+ * The implementation was written so as to conform with Netscapes SSL.
+ *
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to.  The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson ([email protected]).
+ *
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    "This product includes cryptographic software written by
+ *     Eric Young ([email protected])"
+ *    The word 'cryptographic' can be left out if the rouines from the library
+ *    being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from
+ *    the apps directory (application code) you must include an acknowledgement:
+ *    "This product includes software written by Tim Hudson ([email protected])"
+ *
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed.  i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */

README.rst

@@ -50,6 +50,10 @@ To build a Python distribution for Linux x64::
     # With profile-guided optimizations (generated code should be faster):
     $ ./build-linux.py --optimized
 
+To build a Python distribution for Linux x64 using musl libc::
+
+    $ ./build-linux.py --musl --libressl
+
 To build a Python distribution for macOS::
 
     $ ./build-macos.py
@@ -224,6 +228,19 @@ system, typically in ``/usr/lib``.
 On Linux, we statically link a ``libedit`` we compile ourselves, which
 is compiled against a ``libncurses`` we build ourselves.
 
+OpenSSL / LibreSSL
+------------------
+
+By default we compile with OpenSSL. We have some support for compiling
+against LibreSSL.
+
+LibreSSL is currently required for musl libc builds because
+https://github.com/openssl/openssl/commit/38023b87f037f4b832c236dfce2a76272be08763
+broke OpenSSL in our build environment. Projects like Alpine Linux appear
+to still be able to build OpenSSL 1.1.1c. It requires certain headers
+to be in place though. When we tried to work around this, it turned out to
+be easier to compile with LibreSSL than with OpenSSL.
+
 Distribution Format
 ===================
 

build-linux.py

@@ -25,6 +25,7 @@
 def bootstrap():
     parser = argparse.ArgumentParser()
     parser.add_argument('--debug', action='store_true')
+    parser.add_argument('--libressl', action='store_true')
     parser.add_argument('--musl', action='store_true')
     parser.add_argument('--optimized', action='store_true')
 
@@ -44,6 +45,8 @@ def bootstrap():
 
     if args.debug:
         os.environ['PYBUILD_DEBUG'] = '1'
+    if args.libressl:
+        os.environ['PYBUILD_LIBRESSL'] = '1'
     if args.musl:
         os.environ['PYBUILD_MUSL'] = '1'
     if args.optimized:

cpython-linux/Makefile

@@ -79,6 +79,9 @@ $(OUTDIR)/openssl-%.tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-openssl.sh
 $(OUTDIR)/libedit-%.tar: $(PYTHON_DEP_DEPENDS) $(OUTDIR)/ncurses-%.tar $(HERE)/build-libedit.sh
 	$(BUILD) --platform $* libedit
 
+$(OUTDIR)/libressl-%.tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libressl.sh
+	$(BUILD) --platform $* libressl
+
 $(OUTDIR)/readline-%.tar: $(PYTHON_DEP_DEPENDS) $(OUTDIR)/ncurses-$(PLATFORM).tar $(HERE)/build-readline.sh
 	$(BUILD) --platform $* readline
 
@@ -104,7 +107,6 @@ PYTHON_DEPENDS := \
     $(OUTDIR)/libedit-$(PLATFORM).tar \
     $(OUTDIR)/libffi-$(PLATFORM).tar \
     $(OUTDIR)/ncurses-$(PLATFORM).tar \
-    $(OUTDIR)/openssl-$(PLATFORM).tar \
     $(OUTDIR)/readline-$(PLATFORM).tar \
     $(OUTDIR)/sqlite-$(PLATFORM).tar \
     $(OUTDIR)/uuid-$(PLATFORM).tar \
@@ -113,6 +115,12 @@ PYTHON_DEPENDS := \
     $(HERE)/static-modules \
     $(NULL)
 
+ifdef PYBUILD_LIBRESSL
+PYTHON_DEPENDS += $(OUTDIR)/libressl-$(PLATFORM).tar
+else
+PYTHON_DEPENDS += $(OUTDIR)/openssl-$(PLATFORM).tar
+endif
+
 $(OUTDIR)/cpython-$(PLATFORM).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-cpython.sh $(PYTHON_DEPENDS)
 	$(BUILD) --platform $(PLATFORM) cpython
 

cpython-linux/build-libressl.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+cd /build
+
+export PATH=/tools/${TOOLCHAIN}/bin:/tools/host/bin:$PATH
+
+tar -xf libressl-${LIBRESSL_VERSION}.tar.gz
+
+pushd libressl-${LIBRESSL_VERSION}
+
+# Backport of https://github.com/libressl-portable/portable/pull/529 for MUSL support.
+patch -p1 << EOF
+diff --git a/crypto/compat/getprogname_linux.c b/crypto/compat/getprogname_linux.c
+index 2c89743..4e7e31f 100644
+--- a/crypto/compat/getprogname_linux.c
++++ b/crypto/compat/getprogname_linux.c
+@@ -26,9 +26,7 @@ getprogname(void)
+ #if defined(__ANDROID_API__) && __ANDROID_API__ < 21
+ 	extern const char *__progname;
+ 	return __progname;
+-#elif defined(__GLIBC__)
+-	return program_invocation_short_name;
+ #else
+-#error "Cannot emulate getprogname"
++	return program_invocation_short_name;
+ #endif
+ }
+EOF
+
+CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" ./configure \
+    --build=x86_x64-unknown-linux-gnu \
+    --host=${TARGET} \
+    --prefix=/tools/deps \
+    --disable-shared
+
+make -j `nproc`
+make -j `nproc` install DESTDIR=/build/out

cpython-linux/build.py

@@ -456,7 +456,7 @@ def build_tcltk(client, image, platform, musl=False):
         download_tools_archive(container, BUILD / dest_path)
 
 
-def python_build_info(container, config_c_in, setup_dist, setup_local):
+def python_build_info(container, config_c_in, setup_dist, setup_local, libressl=False):
     """Obtain build metadata for the Python distribution."""
 
     bi = {
@@ -559,7 +559,12 @@ def process_setup_line(line, variant=None):
             'variant': d['variant'],
         }
 
-        add_licenses_to_extension_entry(entry)
+        if libressl:
+            ignore_keys = {'openssl'}
+        else:
+            ignore_keys = {'libressl'}
+
+        add_licenses_to_extension_entry(entry, ignore_keys=ignore_keys)
 
         bi['extensions'].setdefault(extension, []).append(entry)
 
@@ -629,7 +634,8 @@ def process_setup_line(line, variant=None):
     return bi
 
 
-def build_cpython(client, image, platform, debug=False, optimized=False, musl=False):
+def build_cpython(client, image, platform, debug=False, optimized=False, musl=False,
+                  libressl=False):
     """Build CPython in a Docker image'"""
     python_archive = download_entry('cpython-3.7', BUILD)
     setuptools_archive = download_entry('setuptools', BUILD)
@@ -659,7 +665,12 @@ def build_cpython(client, image, platform, debug=False, optimized=False, musl=Fa
         install_tools_archive(container, BUILD / ('libedit-%s.tar' % dep_platform))
         install_tools_archive(container, BUILD / ('libffi-%s.tar' % dep_platform))
         install_tools_archive(container, BUILD / ('ncurses-%s.tar' % dep_platform))
-        install_tools_archive(container, BUILD / ('openssl-%s.tar' % dep_platform))
+
+        if libressl:
+            install_tools_archive(container, BUILD / ('libressl-%s.tar' % dep_platform))
+        else:
+            install_tools_archive(container, BUILD / ('openssl-%s.tar' % dep_platform))
+
         install_tools_archive(container, BUILD / ('readline-%s.tar' % dep_platform))
         install_tools_archive(container, BUILD / ('sqlite-%s.tar' % dep_platform))
         # tk requires a bunch of X11 stuff.
@@ -729,7 +740,8 @@ def build_cpython(client, image, platform, debug=False, optimized=False, musl=Fa
             'python_include': 'install/include/%s' % fully_qualified_name,
             'python_stdlib': 'install/lib/python3.7',
             'build_info': python_build_info(container, config_c_in,
-                                            setup_dist_content, setup_local_content),
+                                            setup_dist_content, setup_local_content,
+                                            libressl=libressl),
             'licenses': DOWNLOADS['cpython-3.7']['licenses'],
             'license_path': 'licenses/LICENSE.cpython.txt',
         }
@@ -824,7 +836,7 @@ def main():
             build_readline(client, get_image(client, 'build'), platform=platform,
                            musl=musl)
 
-        elif action in ('bdb', 'bzip2', 'gdbm', 'libffi', 'ncurses', 'openssl', 'sqlite', 'uuid', 'xz', 'zlib'):
+        elif action in ('bdb', 'bzip2', 'gdbm', 'libffi', 'libressl', 'ncurses', 'openssl', 'sqlite', 'uuid', 'xz', 'zlib'):
             simple_build(client, get_image(client, 'build'), action, platform=platform,
                          musl=musl)
 
@@ -834,7 +846,8 @@ def main():
 
         elif action == 'cpython':
             build_cpython(client, get_image(client, 'build'), platform=platform,
-                          musl=musl, debug=args.debug, optimized=args.optimized)
+                          musl=musl, debug=args.debug, optimized=args.optimized,
+                          libressl='PYBUILD_LIBRESSL' in os.environ)
 
         else:
             print('unknown build action: %s' % action)

pythonbuild/downloads.py

@@ -141,6 +141,15 @@
         'licenses': ['MIT'],
         'license_file': 'LICENSE.libffi.txt',
     },
+    'libressl': {
+        'url': 'https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.9.2.tar.gz',
+        'size': 3607549,
+        'sha256': 'c4c78167fae325b47aebd8beb54b6041d6f6a56b3743f4bd5d79b15642f9d5d4',
+        'version': '2.9.2',
+        'library_names': ['crypto', 'ssl'],
+        'licenses': ['OpenSSL'],
+        'license_file': 'LICENSE.libressl.txt',
+    },
     'libx11': {
         'url': 'https://www.x.org/releases/X11R7.7/src/lib/libX11-1.5.0.tar.gz',
         'size': 3073820,

pythonbuild/utils.py

@@ -159,7 +159,7 @@ def compress_python_archive(source_path: pathlib.Path,
     return dest_path
 
 
-def add_licenses_to_extension_entry(entry):
+def add_licenses_to_extension_entry(entry, ignore_keys=None):
     """Add licenses keys to a ``extensions`` entry for JSON distribution info."""
 
     have_licenses = False
@@ -170,7 +170,10 @@ def add_licenses_to_extension_entry(entry):
     for link in entry['links']:
         name = link['name']
 
-        for value in DOWNLOADS.values():
+        for key, value in DOWNLOADS.items():
+            if ignore_keys and key in ignore_keys:
+                continue
+
             if name not in value.get('library_names', []):
                 continue