Merge branch 'nexB:develop' into add_new_flag

Author: akugarg

CHANGELOG.rst

@@ -31,6 +31,7 @@ Outputs:
    the JSON output
  - Add new Debian machine readable copyright output.
  - The CSV output "Resource" column has been renamed to "path".
+ - The SPDX output now has the mandatory DocumentNamespace attribut per SPDX specs #2344
 
 
 Copyright detection:
@@ -39,6 +40,11 @@ Copyright detection:
  - The data structure in the JSON is now using consistently named attributes as
    opposed to a plain value.
 
+ - The copyright detection speed has been significantly improved with the tests
+   taking roughly 1/2 of the time to run. This is achieved mostly by replacing
+   NLTK with a the minimal and simplified subset we need in a new library named
+   pygmars.
+
 
 Package detection:
 ~~~~~~~~~~~~~~~~~~

CONTRIBUTING.rst

@@ -22,7 +22,7 @@ with your name and email::
     Signed-off-by: Jane Doe <[email protected]>
 
 Please try to write a good commit message, see `good commit message wiki
-<https://github.com/nexB/aboutcode/wiki/Writing-good-commit-messages>`_ for
+<https://aboutcode.readthedocs.io/en/latest/contributing/writing_good_commit_messages.html>` for
 details. In particular use the imperative for your commit subject: think that
 you are giving an order to the codebase to update itself.
 

etc/release/gen_pypi_simple.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# SPDX-License-Identifier: BSD-2-Clause-Views AND MIT
+# Copyright (c) 2010 David Wolever <[email protected]>. All rights reserved.
+# originally from https://github.com/wolever/pip2pi
+
+import os
+import re
+import shutil
+
+from html import escape
+from pathlib import Path
+
+"""
+name: pip compatibility tags
+version: 20.3.1
+download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py
+copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
+license_expression: mit
+notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py
+
+Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+get_wheel_from_filename = re.compile(
+    r"""^(?P<namever>(?P<name>.+?)-(?P<version>.*?))
+    ((-(?P<build>\d[^-]*?))?-(?P<pyvers>.+?)-(?P<abis>.+?)-(?P<plats>.+?)
+    \.whl)$""",
+    re.VERBOSE
+).match
+
+sdist_exts = ".tar.gz", ".tar.bz2", ".zip", ".tar.xz",
+wheel_ext = ".whl"
+app_ext = ".pyz"
+dist_exts = sdist_exts + (wheel_ext, app_ext)
+
+
+class InvalidDistributionFilename(Exception):
+    pass
+
+
+def get_package_name_from_filename(filename, normalize=True):
+    """
+    Return the package name extracted from a package ``filename``.
+    Optionally ``normalize`` the name according to distribution name rules.
+    Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid::
+
+    >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz")
+    'foo'
+    >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl")
+    'foo-bar'
+    >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl")
+    'cython'
+    >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl")
+    'python-ldap'
+    >>> get_package_name_from_filename("foo.whl")
+    Traceback (most recent call last):
+        ...
+    InvalidDistributionFilename: ...
+    >>> get_package_name_from_filename("foo.png")
+    Traceback (most recent call last):
+        ...
+    InvalidFilePackageName: ...
+    """
+    if not filename or not filename.endswith(dist_exts):
+        raise InvalidDistributionFilename(filename)
+
+    filename = os.path.basename(filename)
+
+    if filename.endswith(sdist_exts):
+        name_ver = None
+        extension = None
+
+        for ext in sdist_exts:
+            if filename.endswith(ext):
+                name_ver, extension, _ = filename.rpartition(ext)
+                break
+
+        if not extension or not name_ver:
+            raise InvalidDistributionFilename(filename)
+
+        name, _, version = name_ver.rpartition('-')
+
+        if not (name and version):
+            raise InvalidDistributionFilename(filename)
+
+    elif filename.endswith(wheel_ext):
+
+        wheel_info = get_wheel_from_filename(filename)
+
+        if not wheel_info:
+            raise InvalidDistributionFilename(filename)
+
+        name = wheel_info.group('name')
+        version = wheel_info.group('version')
+
+        if not (name and version):
+            raise InvalidDistributionFilename(filename)
+
+    elif filename.endswith(app_ext):
+        name_ver, extension, _ = filename.rpartition(".pyz")
+
+        if "-" in filename:
+            name, _, version = name_ver.rpartition('-')
+        else:
+            name = name_ver
+
+        if not name:
+            raise InvalidDistributionFilename(filename)
+
+    if normalize:
+        name = name.lower().replace('_', '-')
+    return name
+
+
+def build_pypi_index(directory, write_index=False):
+    """
+    Using a ``directory`` directory of wheels and sdists, create the a PyPI simple
+    directory index at ``directory``/simple/ populated with the proper PyPI simple
+    index directory structure crafted using symlinks.
+
+    WARNING: The ``directory``/simple/ directory is removed if it exists.
+    """
+
+    directory = Path(directory)
+
+    index_dir = directory / "simple"
+    if index_dir.exists():
+        shutil.rmtree(str(index_dir), ignore_errors=True)
+
+    index_dir.mkdir(parents=True)
+
+    if write_index:
+        simple_html_index = [
+            "<html><head><title>PyPI Simple Index</title>",
+            "<meta name='api-version' value='2' /></head><body>",
+        ]
+
+    package_names = set()
+    for pkg_file in directory.iterdir():
+
+        pkg_filename = pkg_file.name
+
+        if (
+            not pkg_file.is_file()
+            or not pkg_filename.endswith(dist_exts)
+            or pkg_filename.startswith(".")
+        ):
+            continue
+
+        pkg_name = get_package_name_from_filename(pkg_filename)
+        pkg_index_dir = index_dir / pkg_name
+        pkg_index_dir.mkdir(parents=True, exist_ok=True)
+        pkg_indexed_file = pkg_index_dir / pkg_filename
+        link_target = Path("../..") / pkg_filename
+        pkg_indexed_file.symlink_to(link_target)
+
+        if write_index and pkg_name not in package_names:
+            esc_name = escape(pkg_name)
+            simple_html_index.append(f'<a href="{esc_name}/">{esc_name}</a><br/>')
+            package_names.add(pkg_name)
+
+    if write_index:
+        simple_html_index.append("</body></html>")
+        index_html = index_dir / "index.html"
+        index_html.write_text("\n".join(simple_html_index))
+
+
+if __name__ == "__main__":
+    import sys
+    pkg_dir = sys.argv[1]
+    build_pypi_index(pkg_dir)

etc/release/gen_pypi_simple.py.ABOUT

@@ -0,0 +1,8 @@
+about_resource: gen_pypi_simple.py
+name: gen_pypi_simple.py
+license_expression: bsd-2-clause-views and mit
+copyright: Copyright (c) nexB Inc.
+   Copyright (c) 2010 David Wolever <[email protected]>
+   Copyright (c) The pip developers
+notes: Originally from https://github.com/wolever/pip2pi and modified extensivley
+ Also partially derived from pip code

etc/release/gen_pypi_simple.py.NOTICE

@@ -0,0 +1,56 @@
+SPDX-License-Identifier: BSD-2-Clause-Views AND mit
+
+Copyright (c) nexB Inc.
+Copyright (c) 2010 David Wolever <[email protected]>
+Copyright (c) The pip developers
+
+
+Original code: copyright 2010 David Wolever <[email protected]>. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those
+of the authors and should not be interpreted as representing official policies,
+either expressed or implied, of David Wolever.
+
+
+Original code: Copyright (c) 2008-2020 The pip developers
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

etc/release/utils_thirdparty.py

@@ -975,7 +975,7 @@ class Sdist(Distribution):
     @classmethod
     def from_filename(cls, filename):
         """
-        Return a sdist object built from a filename.
+        Return a Sdist object built from a filename.
         Raise an exception if this is not a valid sdist filename
         """
         name_ver = None
@@ -1048,12 +1048,12 @@ class Wheel(Distribution):
     WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     """
 
-    wheel_file_re = re.compile(
+    get_wheel_from_filename = re.compile(
         r"""^(?P<namever>(?P<name>.+?)-(?P<ver>.*?))
         ((-(?P<build>\d[^-]*?))?-(?P<pyvers>.+?)-(?P<abis>.+?)-(?P<plats>.+?)
         \.whl)$""",
         re.VERBOSE
-    )
+    ).match
 
     build = attr.ib(
         type=str,
@@ -1092,7 +1092,7 @@ def from_filename(cls, filename):
         Return a wheel object built from a filename.
         Raise an exception if this is not a valid wheel filename
         """
-        wheel_info = cls.wheel_file_re.match(filename)
+        wheel_info = cls.get_wheel_from_filename(filename)
         if not wheel_info:
             raise InvalidDistributionFilename(filename)
 

requirements.txt

@@ -9,7 +9,7 @@ cffi==1.14.5
 chardet==4.0.0
 click==8.0.1
 colorama==0.4.4
-commoncode==21.5.25
+commoncode==21.6.11
 cryptography==3.4.7
 debian-inspector==21.5.25
 dparse==0.5.1
@@ -34,7 +34,6 @@ license-expression==1.2
 lxml==4.6.3
 MarkupSafe==2.0.1
 more-itertools==8.8.0
-nltk==3.4.5
 normality==2.2.2
 packageurl-python==0.9.4
 packaging==20.9
@@ -50,6 +49,7 @@ publicsuffix2==2.20191221
 pyahocorasick==1.4.2
 pycparser==2.20
 Pygments==2.9.0
+pygmars==0.5.0
 pymaven-patch==0.3.0
 pyparsing==2.4.7
 PyYAML==5.4.1

setup-mini.cfg

@@ -76,7 +76,6 @@ install_requires =
     license_expression >= 1.0
     lxml >= 4.6.3, < 5.0.0
     MarkupSafe >= 1.0
-    nltk >= 3.2, !=3.6, < 4.0
     packageurl_python >= 0.9.0
     packaging > 20
     pdfminer.six >= 20200101
@@ -86,6 +85,7 @@ install_requires =
     plugincode >= 21.1.21
     publicsuffix2
     pyahocorasick >= 1.4.0, < 1.5
+    pygmars
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0, < 3.0.0
@@ -234,6 +234,6 @@ python_functions=test
 
 addopts =
     -rfExXw
-    --strict
+    --strict-markers
     --ignore setup.py
     --doctest-modules