linux: add license into to PYTHON.json

Now that we've annotated licenses for libraries, it is time to add
that annotation to the PYTHON.json file.

We bump the version of PYTHON.json to 2 and define fields to hold
license info.

We annotate libraries with their library names so we can find
licenses from link requirements.

Then we hook it all together and add the license annotations to
PYTHON.json and add the license files to the distribution.

Author: indygreg

README.rst

@@ -239,9 +239,6 @@ PYTHON.json
 
    See the ``PYTHON.json File`` section for the format of this file.
 
-LICENSE.rst
-   Contains license information of software contained in the distribution.
-
 By convention, the ``build/`` directory contains artifacts from building
 this distribution (object files, libraries, etc) and the ``install/`` directory
 contains a working, self-contained Python installation of this distribution.
@@ -259,7 +256,7 @@ without having to resort to heuristics.
 The file contains a JSON map. This map has the following keys:
 
 version
-   Version number of the file format. Currently ``1``.
+   Version number of the file format. Currently ``2``.
 
 os
    Target operating system for the distribution. e.g. ``linux``, ``macos``,
@@ -291,6 +288,17 @@ build_info
 
    See the ``build_info Data`` section below.
 
+licenses
+   Array of strings containing the license shortname identifiers from the
+   SPDX license list (https://spdx.org/licenses/) for the Python distribution.
+
+  (Version 2 or above only.)
+
+license_path
+   Path to a text file containing the license for this Python distribution.
+
+   (Version 2 or above only.)
+
 build_info Data
 ---------------
 
@@ -391,3 +399,25 @@ system
 
    System libraries are typically passed into the linker by name only and
    found using default library search paths.
+
+licenses
+   Array of strings containing the license shortname identifiers from the
+   SPDX license list (https://spdx.org/licenses/).
+
+   If this field is missing, licenses are unknown. Empty array denotes no known
+   licenses.
+
+   (Version 2 or above only.)
+
+license_path
+   Path to a text file containing the license for this library.
+
+   (Version 2 or above only.)
+
+license_public_domain
+   Bool indicating that the library is in the public domain.
+
+   There is no SPDX identifier for public domain. And we want to be explicit
+   about something being in the public domain because of the legal implications.
+
+   (Version 2 or above only.)

cpython-linux/build-cpython.sh

@@ -171,4 +171,6 @@ cp -av Modules/config.c.in /build/out/python/build/Modules/
 cp -av Python/frozen.c /build/out/python/build/Python/
 cp -av Modules/Setup.dist /build/out/python/build/Modules/
 cp -av Modules/Setup.local /build/out/python/build/Modules/
-cp /build/python-licenses.rst /build/out/python/LICENSE.rst
+
+mkdir /build/out/python/licenses
+cp /build/LICENSE.*.txt /build/out/python/licenses/

cpython-linux/build.py

@@ -25,6 +25,7 @@
     DOWNLOADS,
 )
 from pythonbuild.utils import (
+    add_license_to_link_entry,
     download_entry,
 )
 
@@ -538,10 +539,14 @@ def process_setup_line(line, variant=None):
             log('adding library %s for extension %s' % (libname, extension))
 
             if libname in libraries:
-                links.append({
+                entry = {
                     'name': libname,
                     'path_static': 'build/lib/lib%s.a' % libname,
-                })
+                }
+
+                add_license_to_link_entry(entry)
+
+                links.append(entry)
             else:
                 links.append({
                     'name': libname,
@@ -667,7 +672,10 @@ def build_cpython(client, image, platform, optimized=False, musl=False):
         copy_file_to_container(pip_archive, container, '/build')
         copy_file_to_container(SUPPORT / 'build-cpython.sh', container,
                                '/build')
-        copy_file_to_container(ROOT / 'python-licenses.rst', container, '/build')
+
+        for f in sorted(os.listdir(ROOT)):
+            if f.startswith('LICENSE.') and f.endswith('.txt'):
+                copy_file_to_container(ROOT / f, container, '/build')
 
         # TODO copy latest pip/setuptools.
 
@@ -705,7 +713,7 @@ def build_cpython(client, image, platform, optimized=False, musl=False):
 
         # Create PYTHON.json file describing this distribution.
         python_info = {
-            'version': '1',
+            'version': '2',
             'os': 'linux',
             'arch': 'x86_64',
             'python_flavor': 'cpython',
@@ -715,6 +723,8 @@ def build_cpython(client, image, platform, optimized=False, musl=False):
             'python_stdlib': 'install/lib/python3.7',
             'build_info': python_build_info(container, config_c_in,
                                             setup_dist_content, setup_local_content),
+            'licenses': DOWNLOADS['cpython-3.7']['licenses'],
+            'license_path': 'licenses/LICENSE.cpython.txt',
         }
 
         with tempfile.NamedTemporaryFile('w') as fh:

pythonbuild/downloads.py

@@ -9,6 +9,7 @@
         'size': 36541923,
         'sha256': '2917c28f60903908c2ca4587ded1363b812c4e830a5326aaa77c9879d13ae18e',
         'version': '6.0.19',
+        'library_names': ['db'],
         'licenses': ['Sleepycat'],
         'license_file': 'LICENSE.bdb.txt',
     },
@@ -23,6 +24,7 @@
         'size': 782025,
         'sha256': 'a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd',
         'version': '1.0.6',
+        'library_names': ['bz2'],
         'licenses': ['bzip2-1.0.6'],
         'license_file': 'LICENSE.bzip2.txt',
     },
@@ -81,6 +83,7 @@
         'size': 941863,
         'sha256': '86e613527e5dba544e73208f42b78b7c022d4fa5a6d5498bf18c8d6f745b91dc',
         'version': '1.18.1',
+        'library_names': ['gdbm'],
         'licenses': ['GPL-3.0-or-later'],
         'license_file': 'LICENSE.gdbm.txt',
     },
@@ -125,6 +128,7 @@
         'size': 521931,
         'sha256': '2811d70c0b000f2ca91b7cb1a37203134441743c4fcc9c37b0b687f328611064',
         'version': '20181209-3.1',
+        'library_names': ['edit'],
         'licenses': ['BSD-3-Clause'],
         'license_file': 'LICENSE.libedit.txt',
     },
@@ -133,6 +137,7 @@
         'size': 940837,
         'sha256': 'd06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37',
         'version': '3.2.1',
+        'library_names': ['ffi'],
         'licenses': ['MIT'],
         'license_file': 'LICENSE.libffi.txt',
     },
@@ -188,6 +193,7 @@
         'size': 3365395,
         'sha256': 'aa057eeeb4a14d470101eff4597d5833dcef5965331be3528c08d99cebaa0d17',
         'version': '6.1',
+        'library_names': ['ncurses', 'panel'],
         'licenses': ['X11'],
         'license_file': 'LICENSE.ncurses.txt',
     },
@@ -208,6 +214,7 @@
         'size': 8213737,
         'sha256': '5c557b023230413dfb0756f3137a13e6d726838ccd1430888ad15bfb2b43ea4b',
         'version': '1.1.1b',
+        'library_names': ['crypto', 'ssl'],
         'licenses': ['OpenSSL'],
         'license_file': 'LICENSE.openssl.txt',
     },
@@ -228,6 +235,7 @@
         'size': 2468560,
         'sha256': '56ba6071b9462f980c5a72ab0023893b65ba6debb4eeb475d7a563dc65cafd43',
         'version': '6.3',
+        'library_names': ['readline'],
         'licenses': ['GPL-3.0'],
         'license_file': 'LICENSE.readline.txt',
     },
@@ -248,7 +256,10 @@
         'sha256': 'd61b5286f062adfce5125eaf544d495300656908e61fca143517afcc0a89b7c3',
         'version': '3280000',
         'actual_version': '3.28.0.0',
+        'library_names': ['sqlite3'],
+        'licenses': [],
         'license_file': 'LICENSE.sqlite.txt',
+        'license_public_domain': True,
     },
     'strawberryperl': {
         'url': 'http://strawberryperl.com/download/5.28.1.1/strawberry-perl-5.28.1.1-32bit-portable.zip',
@@ -278,6 +289,7 @@
         'size': 318256,
         'sha256': '46af3275291091009ad7f1b899de3d0cea0252737550e7919d17237997db5644',
         'version': '1.0.3',
+        'library_names': ['uuid'],
         'licenses': ['BSD-3-Clause'],
         'license_file': 'LICENSE.libuuid.txt',
     },
@@ -286,15 +298,19 @@
         'size': 1572354,
         'sha256': 'b512f3b726d3b37b6dc4c8570e137b9311e7552e8ccbab4d39d47ce5f4177145',
         'version': '5.2.4',
+        'library_names': ['lzma'],
         # liblzma is in the public domain. Other parts of code have licenses. But
         # we only use liblzma.
+        'licenses': [],
         'license_file': 'LICENSE.liblzma.txt',
+        'license_public_domain': True,
     },
     'zlib': {
         'url': 'https://zlib.net/zlib-1.2.11.tar.gz',
         'size': 607698,
         'sha256': 'c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1',
         'version': '1.2.11',
+        'library_names': ['z'],
         'licenses': ['Zlib'],
         'license_file': 'LICENSE.zlib.txt',
     },

pythonbuild/utils.py

@@ -157,3 +157,23 @@ def compress_python_archive(source_path: pathlib.Path,
     print('%s has SHA256 %s' % (dest_path, hash_path(dest_path)))
 
     return dest_path
+
+
+def add_license_to_link_entry(entry):
+    """Add licenses keys to a ``link`` entry for JSON distribution info."""
+    name = entry['name']
+
+    for value in DOWNLOADS.values():
+        if name not in value.get('library_names', []):
+            continue
+
+        # Don't add licenses annotations if they aren't defined. This leaves
+        # things as "unknown" to consumers.
+        if 'licenses' not in value:
+            continue
+
+        entry['licenses'] = value['licenses']
+        entry['license_path'] = 'licenses/%s' % value['license_file']
+        entry['license_public_domain'] = value.get('license_public_domain', False)
+
+        return