Merge pull request #3757 from nexB/improve-pypi-package-detection

Upgrade python package detection

Author: AyanSinhaMahapatra

.github/workflows/scancode-release.yml

@@ -235,7 +235,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-22.04, ubuntu-22.04, macos-11, macos-12]
+        os: [ubuntu-22.04, ubuntu-22.04, macos-12, macos-13]
         pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
@@ -367,7 +367,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [macos-11, macos-12]
+        os: [macos-12, macos-13]
         pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:

CHANGELOG.rst

@@ -46,6 +46,20 @@ v33.0.0 (next next, roadmap)
   from swift `swift-show-dependencies.deplock` generated by DepLock.
   See https://github.com/nexB/scancode-toolkit/pull/3829
 
+- Add support for `pip-inspect.deplock` files to parse and store
+  resolved packages and dependency relationships, to statically
+  resolve a python dependency graph.
+  See https://github.com/nexB/scancode.io/issues/1262
+
+- Add support for poetry packages, with poetry specific pyproject.toml
+  support, poetry.lock and package assembly support. Also add support
+  for parsing and storing resolved packages and dependency relationships
+  required to statically resolve poetry dependecy graphs.
+  See https://github.com/nexB/scancode-toolkit/issues/2109
+
+- Add support for pyproject.toml files in python projects.
+  See https://github.com/nexB/scancode-toolkit/issues/3753
+
 v32.2.0 - 2024-06-19
 ----------------------
 

azure-pipelines.yml

@@ -111,21 +111,20 @@ jobs:
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos11_cpython
-          python_architecture: x64
-          image_name: macOS-11
+          job_name: macos12_cpython
+          image_name: macOS-12
           python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          python_architecture: x64
           test_suites:
               all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos12_cpython
-          image_name: macOS-12
+          job_name: macos13_cpython
+          image_name: macOS-13
           python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
-          python_architecture: x64
           test_suites:
-              all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
@@ -204,19 +203,19 @@ jobs:
           python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
           test_suites:
               all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
-
+  
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos11_cpython_latest_from_pip
-          image_name: macos-11
+          job_name: macos12_cpython_latest_from_pip
+          image_name: macos-12
           python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
           test_suites:
               all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos12_cpython_latest_from_pip
-          image_name: macos-12
+          job_name: macos13_cpython_latest_from_pip
+          image_name: macos-13
           python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
           test_suites:
               all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py

docs/source/reference/available_package_parsers.rst

@@ -587,6 +587,12 @@ parsers in scancode-toolkit during documentation builds.
      - ``nuget_nupsec``
      - None
      - https://docs.microsoft.com/en-us/nuget/reference/nuspec
+   * - NuGet packages.lock.json file
+     - ``*packages.lock.json``
+     - ``nuget``
+     - ``nuget_packages_lock``
+     - None
+     - https://learn.microsoft.com/en-us/nuget/reference/cli-reference/cli-ref-restore
    * - Ocaml Opam file
      - ``*opam``
      - ``opam``
@@ -661,12 +667,30 @@ parsers in scancode-toolkit during documentation builds.
      - ``pypi_egg_pkginfo``
      - Python
      - https://peps.python.org/pep-0376/
+   * - Python poetry pyproject.toml
+     - ``*pip-inspect.deplock``
+     - ``pypi``
+     - ``pypi_inspect_deplock``
+     - Python
+     - https://pip.pypa.io/en/stable/cli/pip_inspect/
+   * - Python poetry lockfile
+     - ``*poetry.lock``
+     - ``pypi``
+     - ``pypi_poetry_lock``
+     - Python
+     - https://python-poetry.org/docs/basic-usage/#installing-with-poetrylock
+   * - Python poetry pyproject.toml
+     - ``*pyproject.toml``
+     - ``pypi``
+     - ``pypi_poetry_pyproject_toml``
+     - Python
+     - https://packaging.python.org/en/latest/specifications/pyproject-toml/
    * - Python pyproject.toml
      - ``*pyproject.toml``
      - ``pypi``
      - ``pypi_pyproject_toml``
      - Python
-     - https://peps.python.org/pep-0621/
+     - https://packaging.python.org/en/latest/specifications/pyproject-toml/
    * - PyPI extracted sdist PKG-INFO
      - ``*/PKG-INFO``
      - ``pypi``

src/packagedcode/__init__.py

@@ -173,6 +173,8 @@
     # pypi.PypiSdistArchiveHandler,
     pypi.PypiWheelHandler,
     pypi.PyprojectTomlHandler,
+    pypi.PoetryPyprojectTomlHandler,
+    pypi.PoetryLockHandler,
     pypi.PythonEditableInstallationPkgInfoFile,
     pypi.PythonEggPkgInfoFile,
     pypi.PythonInstalledWheelMetadataFile,
@@ -208,6 +210,9 @@
     windows.MicrosoftUpdateManifestHandler,
 
     win_pe.WindowsExecutableHandler,
+
+    # These are handlers for deplock generated files
+    pypi.PipInspectDeplockHandler,
 ]
 
 if on_linux:

src/packagedcode/debian.py

@@ -21,6 +21,7 @@
 
 from packagedcode import models
 from packagedcode.utils import get_ancestor
+from packagedcode.utils import parse_maintainer_name_email
 
 """
 Handle Debian package archives, control files and installed databases.
@@ -653,20 +654,20 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
 
     maintainer = debian_data.get('maintainer')
     if maintainer:
-        maintainer_name, maintainer_email = parse_debian_maintainers(maintainer)
+        maintainer_name, maintainer_email = parse_maintainer_name_email(maintainer)
         party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email)
         parties.append(party)
 
     orig_maintainer = debian_data.get('original_maintainer')
     if orig_maintainer:
-        maintainer_name, maintainer_email = parse_debian_maintainers(orig_maintainer)
+        maintainer_name, maintainer_email = parse_maintainer_name_email(orig_maintainer)
         party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email)
         parties.append(party)
 
     uploaders = debian_data.get('uploaders')
     if uploaders:
         for uploader in uploaders.split(", "):
-            uploader_name, uploader_email = parse_debian_maintainers(uploader)
+            uploader_name, uploader_email = parse_maintainer_name_email(uploader)
             party = models.Party(role='uploader', name=uploader_name, email=uploader_email)
             parties.append(party)
 
@@ -736,26 +737,6 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
     return models.PackageData.from_data(package_data, package_only)
 
 
-def parse_debian_maintainers(maintainer):
-    """
-    Get name and email values from a debian maintainer string.
-
-    Example string:
-    Debian systemd Maintainers <[email protected]>
-    """
-    email_wrappers = ["<", ">"]
-    has_email = "@" in maintainer and all([
-        True 
-        for char in email_wrappers
-        if char in maintainer
-    ])
-    if not has_email:
-        return maintainer, None
-
-    name, _, email = maintainer.rpartition("<")
-    return name.rstrip(" "), email.rstrip(">")
-
-
 def populate_debian_namespace(packages):
     """
     For an iterable of debian `packages`, populate the