Merge pull request #2469 from nexB/release-21.3.3x-prep

Release 21.3.30

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

.gitignore

@@ -85,3 +85,6 @@ tcl
 
 *.orig
 /release*
+/00-*.txt
+/z-todo-licenses-*
+

CHANGELOG.rst

@@ -1,16 +1,109 @@
 Changelog
 =========
 
+v21.4.x (next)
+--------------
 
-v21.3.x (next)
-------------
+Breaking API changes:
 
+ - The data structure of the JSON output has changed for copyrights, authors
+   and holders: we now use proper name for attributes and not a generic "value".
 
-Misc.:
+ - The data structure of the JSON output has changed for licenses: we now
+   return match details once for each matched license expression rather than
+   once for each license in a matched expression. There is a new top-level
+   "licenses" attributes that contains the data details for each detected
+   licenses only once. This data can contain the reference license text
+   as an option.
+
+ - The data structure of the JSON output has changed for packages: we now
+   return "package_manifests" package information at the manifest file-level
+   rather than "packages". There is a a new top-level "packages" attribute
+   that contains each package instace that can be aggregating data from
+   multiple manifests for a single package instance.
+
+
+Ouputs:
+
+ - Add new YAML-formatted output. This is exactly the same data structure as for
+   the JSON output
+
+
+License scanning:
+
+ - Add new command line option to filter ignorable copyrights when included
+   in licenses.
+
+
+
+v21.3.30
+--------
+
+This is a mjor version with no breaking API changes. Heads-up: the next version
+will bring up some significant API changes.
+
+
+Security:
+
+ - Update dependency versions for security.
+
+
+License scanning:
+
+ - Add 22 new and update 71 existing reference licenses
+
+ - Update licenses to include the SPDX license list 3.12
+
+ - Improve license detection accuracy with over 2300 new and improved license
+   detection rules
+
+ - Undeprecate the regexp license and deprecate the hs-regexp-orig license
+
+ - Improve license db initial load time with caching for faster scancode
+   start time
+
+ - Ensure that license short names are no more than 50 characters long
+
+ - Thank you to Chin-Yeung Li @chinyeungli, Armijn Hemmel @jelmer
+
+
+Copyright scanning:
+
+ - Detect SPDX-FileCopyrightText as defined by the FSFE Reuse project
 
  - Fix bug when using the --filter-clues command line option
    Thank you to Van Lindberg @VanL
 
+ - Allow calling copyright detection from text lines to ease integration
+   Thank you to Jelmer Vernooij @jelmer
+   
+
+Package scanning:
+
+ - Add support for installed RPMs detection internally (not wired to scans)
+   Thank you to Chin-Yeung Li @chinyeungli
+
+ - Improve handling of Debian copyright files with faster and more
+   accurate license detection
+   Thank you to Thomas Druez @tdruez 
+   
+ - Add new built-in support for installed_files report. Only available when
+   used as a library.
+
+ - Improve support for RPM, npm, Debian, build scripts (Bazel) and Go packages
+ 
+ - Add new support to collect information from semi-structured Readme files
+
+
+Ouputs:
+
+ - Add new Debian copyright-formatted output.
+   Thank you to Jelmer Vernooij @jelmer
+   
+ - Fix bug in --ignore where directories where not skipped correctly
+   Thank you to Pierre Tardy @tardyp 
+
+
 
 v21.2.25
 --------

NOTICE

@@ -34,6 +34,7 @@ including:
 - https://github.com/nexB/thirdparty-packages/pypi/
 - https://github.com/nexB/scancode-plugins/
 - https://github.com/nexB/scancode-thirdparty-src/
+- https://github.com/nexB/thirdparty-packages/
 
 You may also contact us to request the source code by email at [email protected] or
 by postal mail at:

configure

@@ -52,7 +52,7 @@ export THIRDPARTY_LINKS="https://thirdparty.aboutcode.org/pypi"
 # requirements used by default or with --dev.
 # note the use of constraints with -c 
 REQUIREMENTS="--editable . --constraint requirements.txt"
-DEV_REQUIREMENTS="--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt"
+DEV_REQUIREMENTS="--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt"
 
 # default supported Python version
 if [[ "$CONFIGURE_SUPPORTED_PYTHON" == "" ]]; then

etc/release/utils_thirdparty.py

@@ -1745,7 +1745,13 @@ def get_remote_repo(remote_links_url=REMOTE_LINKS_URL):
 
 
 def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL):
-    return get_remote_repo(remote_links_url).get_package(name, version)
+    """
+    Return a PypiPackage or None.
+    """
+    try:
+        return get_remote_repo(remote_links_url).get_package(name, version)
+    except RemoteNotFetchedException as e:
+        print(f'Failed to fetch remote package info: {e}')
 
 
 _PYPI_REPO = None
@@ -1759,7 +1765,13 @@ def get_pypi_repo(pypi_simple_url=PYPI_SIMPLE_URL):
 
 
 def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL):
-    return get_pypi_repo(pypi_simple_url).get_package(name, version)
+    """
+    Return a PypiPackage or None.
+    """
+    try:
+        return get_pypi_repo(pypi_simple_url).get_package(name, version)
+    except RemoteNotFetchedException as e:
+        print(f'Failed to fetch remote package info: {e}')
 
 ################################################################################
 #
@@ -2003,20 +2015,25 @@ def fetch_missing_sources(dest_dir=THIRDPARTY_DIR):
     for package in local_packages:
         if not package.sdist:
             print(f'Finding sources for: {package.name}=={package.version}: ', end='')
-            pypi_package = pypi_repo.get_package(
-                name=package.name, version=package.version)
-
-            if pypi_package and pypi_package.sdist:
-                print(f'Fetching sources from Pypi')
-                pypi_package.fetch_sdist(dest_dir=dest_dir)
-            else:
-                remote_package = remote_repo.get_package(
+            try:
+                pypi_package = pypi_repo.get_package(
                     name=package.name, version=package.version)
 
-                if remote_package and remote_package.sdist:
-                    print(f'Fetching sources from Remote')
-                    remote_package.fetch_sdist(dest_dir=dest_dir)
+                if pypi_package and pypi_package.sdist:
+                    print(f'Fetching sources from Pypi')
+                    pypi_package.fetch_sdist(dest_dir=dest_dir)
                     continue
+                else:
+                    remote_package = remote_repo.get_package(
+                        name=package.name, version=package.version)
+
+                    if remote_package and remote_package.sdist:
+                        print(f'Fetching sources from Remote')
+                        remote_package.fetch_sdist(dest_dir=dest_dir)
+                        continue
+
+            except RemoteNotFetchedException as e:
+                print(f'Failed to fetch remote package info: {e}')
 
             print(f'No sources found')
             not_found.append((package.name, package.version,))
@@ -2542,7 +2559,6 @@ def add_or_upgrade_built_wheels(
             name=name, version=version, environment=environment, dest_dir=dest_dir)
         if wheel_filename:
             wheel_filenames.append(wheel_filename)
-            continue
 
         # the wheel is not available locally, remotely or in Pypi
         # we need to build binary from sources

etc/scripts/licenses/buildrules.py

@@ -139,7 +139,7 @@ def rule_exists(text):
     if len(matches) > 1:
         return False
     match = matches[0]
-    if match.matcher == match_hash.MATCH_HASH:
+    if match.matcher == match_hash.MATCH_HASH and match.score() == 100:
         return match.rule.identifier
 
 

etc/scripts/licenses/synclic.py

@@ -37,13 +37,11 @@
 Run python synclic.py -h for help.
 """
 
-
 TRACE = True
 TRACE_ADD = True
 TRACE_FETCH = True
 TRACE_DEEP = False
 
-
 # may be useful to change for testing
 SPDX_DEFAULT_REPO = 'spdx/license-list-data'
 
@@ -164,15 +162,15 @@ def __init__(self, external_base_dir):
         if not exists(self.new_dir):
             mkdir(self.new_dir)
 
-    def get_licenses(self, scancode_licenses):
+    def get_licenses(self, scancode_licenses, **kwargs):
         """
         Return a mapping of key -> ScanCode License objects either fetched
         externally or loaded from the existing `self.original_dir`
         """
         print('Fetching and storing external licenses in:', self.original_dir)
 
         licenses = []
-        for lic, text in self.fetch_licenses(scancode_licenses):
+        for lic, text in self.fetch_licenses(scancode_licenses, **kwargs):
             try:
                 with io.open(lic.text_file, 'w', encoding='utf-8')as tf:
                     tf.write(text)
@@ -193,7 +191,7 @@ def get_licenses(self, scancode_licenses):
 
         return load_licenses(self.update_dir, with_deprecated=True)
 
-    def fetch_licenses(self, scancode_licenses):
+    def fetch_licenses(self, scancode_licenses, **kwargs):
         """
         Yield tuples of (License object, license text) fetched from this external source.
         """
@@ -338,14 +336,18 @@ class SpdxSource(ExternalLicensesSource):
         'notes',
     )
 
-    def fetch_licenses(self, scancode_licenses, from_repo=SPDX_DEFAULT_REPO):
+    def fetch_licenses(self, scancode_licenses, commitish=None, from_repo=SPDX_DEFAULT_REPO):
         """
         Yield License objects fetched from the latest SPDX license list.
+        Use the latest tagged version or the `commitish` is provided.
         """
-        # get latest tag
-        tags_url = 'https://api.github.com/repos/{from_repo}/tags'.format(**locals())
-        tags = get_response(tags_url, headers={}, params={})
-        tag = tags[0]['name']
+        if not commitish:
+            # get latest tag
+            tags_url = 'https://api.github.com/repos/{from_repo}/tags'.format(**locals())
+            tags = get_response(tags_url, headers={}, params={})
+            tag = tags[0]['name']
+        else:
+            tag = commitish
 
         # fetch licenses and exceptions
         # note that exceptions data have -- weirdly enough -- a different schema
@@ -385,11 +387,15 @@ def build_license(self, mapping, scancode_licenses):
 
         # these keys have a complicated history
         if key in set([
-                'gpl-1.0', 'gpl-2.0', 'gpl-3.0',
-                'lgpl-2.0', 'lgpl-2.1', 'lgpl-3.0',
-                'agpl-1.0', 'agpl-2.0', 'agpl-3.0',
-                'gfdl-1.1', 'gfdl-1.2', 'gfdl-1.3',
-                'nokia-qt-exception-1.1', 'bzip2-1.0.5']):
+            'gpl-1.0', 'gpl-2.0', 'gpl-3.0',
+            'lgpl-2.0', 'lgpl-2.1', 'lgpl-3.0',
+            'agpl-1.0', 'agpl-2.0', 'agpl-3.0',
+            'gfdl-1.1', 'gfdl-1.2', 'gfdl-1.3',
+            'nokia-qt-exception-1.1',
+            'bzip2-1.0.5',
+            'bsd-2-clause-freebsd',
+            'bsd-2-clause-netbsd',
+        ]):
             return
 
         deprecated = mapping.get('isDeprecatedLicenseId', False)
@@ -484,7 +490,7 @@ def __init__(self, external_base_dir, api_base_url=None, api_key=None):
 
         super(DejaSource, self).__init__(external_base_dir)
 
-    def fetch_licenses(self, scancode_licenses):
+    def fetch_licenses(self, scancode_licenses, **kwargs):
         api_url = '/'.join([self.api_base_url.rstrip('/'), 'licenses/'])
         for licenses in call_deja_api(api_url, self.api_key, paginate=100):
             for lic in licenses:
@@ -815,7 +821,9 @@ def update_external(_attrib, _sc_val, _ext_val):
         scancode_key = scancode_license.spdx_license_key
         external_key = external_license.spdx_license_key
         if scancode_key != external_key:
-            raise Exception('Non mergeable licenses with different SPDX keys: %(scancode_key)s <> %(external_key)s' % locals())
+            raise Exception(
+                f'Non mergeable licenses with different SPDX keys: scancode_license.spdx_license_key {scancode_key} <>  external_license.spdx_license_key {external_key}'
+            )
     else:
         scancode_key = scancode_license.key
         external_key = external_license.key
@@ -914,7 +922,7 @@ def update_external(_attrib, _sc_val, _ext_val):
 
 
 def synchronize_licenses(scancode_licenses, external_source, use_spdx_key=False,
-                         match_text=False, match_approx=False):
+                         match_text=False, match_approx=False, commitish=None):
     """
     Update the `scancode_licenses` ScanCodeLicenses licenses and texts in-place
     (e.g. in their current storage directory) from an `external_source`
@@ -949,7 +957,7 @@ def synchronize_licenses(scancode_licenses, external_source, use_spdx_key=False,
 
     # mappings of key -> License
     scancodes_by_key = scancode_licenses.by_key
-    externals_by_key = external_source.get_licenses(scancode_licenses)
+    externals_by_key = external_source.get_licenses(scancode_licenses, commitish=commitish)
 
     if use_spdx_key:
         scancodes_by_key = scancode_licenses.by_spdx_key
@@ -1123,8 +1131,9 @@ def synchronize_licenses(scancode_licenses, external_source, use_spdx_key=False,
 @click.option('-a', '--match-approx', is_flag=True, default=False, help='Include approximate license detection matches when matching ScanCode license.')
 @click.option('-t', '--trace', is_flag=True, default=False, help='Print execution trace.')
 @click.option('--create-ext', is_flag=True, default=False, help='Create new external licenses in the external source if possible.')
+@click.option('--commitish', type=str, default=None, help='An optional commitish to use for SPDX license data instead of the latest release.')
 @click.help_option('-h', '--help')
-def cli(license_dir, source, match_text, match_approx, trace, create_ext):
+def cli(license_dir, source, match_text, match_approx, trace, create_ext, commitish=None):
     """
     Synchronize ScanCode licenses with an external license source.
 
@@ -1143,10 +1152,13 @@ def cli(license_dir, source, match_text, match_approx, trace, create_ext):
     scancode_licenses = ScanCodeLicenses()
 
     use_spdx_key = source == 'spdx'
-    added_to_external = synchronize_licenses(scancode_licenses, external_source,
-                         use_spdx_key=use_spdx_key,
-                         match_text=match_text,
-                         match_approx=match_approx)
+    added_to_external = synchronize_licenses(
+        scancode_licenses, external_source,
+        use_spdx_key=use_spdx_key,
+        match_text=match_text,
+        match_approx=match_approx,
+        commitish=commitish,
+    )
     print()
     if create_ext and isinstance(external_source, DejaSource):
         api_url = external_source.api_base_url

requirements-dev.txt

@@ -1,4 +1,4 @@
-aboutcode-toolkit==5.1.0
+aboutcode-toolkit==6.0.0
 apipkg==1.5
 codecov==2.1.11
 coverage==5.3.1