Merge pull request #4171 from aboutcode-org/fix-pypi-installed-wheel-assembly

Refactor and fix package assembly for installed wheels

Author: AyanSinhaMahapatra

src/packagedcode/pypi.py

@@ -13,6 +13,7 @@
 import json
 import logging
 import os
+import posixpath
 import re
 import sys
 import tempfile
@@ -390,19 +391,23 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
         Assign files to package for an installed wheel. This requires a bit
         of navigation around as the files can be in multiple places.
         """
-        site_packages = resource.parent(codebase).parent(codebase)
+        dist_info_dir = resource.parent(codebase)
+        if not dist_info_dir:
+            return
+
+        # This could be a regular directory too which is not `site-packages`
+        site_packages = dist_info_dir.parent(codebase)
         if not site_packages:
             return
+
         package_data = resource.package_data
         assert len(resource.package_data) == 1, (
             f'Unsupported Pypi METADATA wheel structure: {resource.path!r} '
             f'with multiple {package_data!r}'
         )
 
         package_data = models.PackageData.from_dict(package_data[0])
-
         package_uid = package.package_uid
-
         if package_uid:
             # save thyself!
             package_adder(package_uid, resource, codebase)
@@ -414,76 +419,33 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
                 # relative paths need special treatment
                 # most of thense are references to bin ../../../bin/wheel
                 cannot_resolve = False
-                ref_resource = None
+                ref_resource = site_packages
+                # note that resolving leading ".." always stays in the codebase 
                 while path_ref.startswith('..'):
                     _, _, path_ref = path_ref.partition('../')
-                    ref_resource = site_packages.parent(codebase)
+                    ref_resource = ref_resource.parent(codebase)
                     if not ref_resource:
                         cannot_resolve = True
                         break
+
                 if cannot_resolve or not ref_resource:
                     # TODO:w e should log these kind of things
                     continue
                 else:
-                    if package_uid:
+                    ref_resource = codebase.get_resource(
+                        path=posixpath.join(ref_resource.path, path_ref)
+                    )
+                    if ref_resource and package_uid:
                         package_adder(package_uid, ref_resource, codebase)
             else:
-                ref_resource = get_resource_for_path(
-                    path=path_ref,
-                    root=site_packages,
-                    codebase=codebase,
+                # These are absolute paths from the site-packages directory
+                ref_resource = codebase.get_resource(
+                    path=posixpath.join(site_packages.path, path_ref)
                 )
                 if ref_resource and package_uid:
                     package_adder(package_uid, ref_resource, codebase)
 
 
-def get_resource_for_path(path, root, codebase):
-    """
-    Return a resource in ``codebase`` that has a ``path`` relative to the
-    ``root` Resource
-
-    For example, say we start from this:
-        path: this/is/that therefore segments [this, is, that]
-        root: /usr/foo
-
-    We would have these iterations:
-    iteration1
-        root = /usr/foo
-        segments = [this, is, that]
-        seg  this
-        segments = [is, that]
-        children = [/usr/foo/this]
-        root = /usr/foo/this
-
-    iteration2
-        root = /usr/foo/this
-        segments = [is, that]
-        seg  is
-        segments = [that]
-        children = [/usr/foo/this/is]
-        root = /usr/foo/this/is
-
-    iteration3
-        root = /usr/foo/this/is
-        segments = [that]
-        seg  that
-        segments = []
-        children = [/usr/foo/this/is/that]
-        root = /usr/foo/this/is/that
-
-    finally return root as /usr/foo/this/is/that
-    """
-    segments = path.strip('/').split('/')
-    while segments:
-        seg = segments.pop(0)
-        children = [c for c in root.children(codebase) if c.name == seg]
-        if len(children) != 1:
-            return
-        else:
-            root = children[0]
-    return root
-
-
 class PyprojectTomlHandler(BaseExtractedPythonLayout):
     datasource_id = 'pypi_pyproject_toml'
     path_patterns = ('*pyproject.toml',)
@@ -1680,10 +1642,10 @@ def get_declared_license(metainfo):
     # TODO: We should make the declared license as it is, this should be
     # updated in scancode to parse a pure string
     lic = get_attribute(metainfo, 'License')
-
-    license_file = None
-    if lic and 'file' in lic:
-        license_file = lic.pop('file')
+    license_file = get_attribute(metainfo, 'License-File')
+    if not license_file and lic:
+        if isinstance(lic, dict) and 'file' in lic.keys():
+            license_file = lic.pop('file')
 
     if lic and not lic == 'UNKNOWN':
         if 'text' in lic:

tests/packagedcode/data/conda/assembly-conda-scan.json

@@ -177,7 +177,8 @@
       "is_private": false,
       "is_virtual": false,
       "extra_data": {
-        "Documentation": "https://requests.readthedocs.io"
+        "Documentation": "https://requests.readthedocs.io",
+        "license_file": "LICENSE"
       },
       "repository_homepage_url": "https://pypi.org/project/requests",
       "repository_download_url": "https://pypi.org/packages/source/r/requests/requests-2.32.3.tar.gz",
@@ -491,7 +492,8 @@
           "is_private": false,
           "is_virtual": false,
           "extra_data": {
-            "Documentation": "https://requests.readthedocs.io"
+            "Documentation": "https://requests.readthedocs.io",
+            "license_file": "LICENSE"
           },
           "dependencies": [
             {

tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests-with-license.json

@@ -77,7 +77,8 @@
       "is_private": false,
       "is_virtual": false,
       "extra_data": {
-        "Documentation": "https://setuptools.readthedocs.io/"
+        "Documentation": "https://setuptools.readthedocs.io/",
+        "license_file": "LICENSE"
       },
       "repository_homepage_url": "https://pypi.org/project/setuptools",
       "repository_download_url": "https://pypi.org/packages/source/s/setuptools/setuptools-58.2.0.tar.gz",
@@ -308,7 +309,8 @@
           "is_private": false,
           "is_virtual": false,
           "extra_data": {
-            "Documentation": "https://setuptools.readthedocs.io/"
+            "Documentation": "https://setuptools.readthedocs.io/",
+            "license_file": "LICENSE"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/setuptools",

tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json

@@ -77,7 +77,8 @@
       "is_private": false,
       "is_virtual": false,
       "extra_data": {
-        "Documentation": "https://setuptools.readthedocs.io/"
+        "Documentation": "https://setuptools.readthedocs.io/",
+        "license_file": "LICENSE"
       },
       "repository_homepage_url": "https://pypi.org/project/setuptools",
       "repository_download_url": "https://pypi.org/packages/source/s/setuptools/setuptools-58.2.0.tar.gz",
@@ -194,7 +195,8 @@
           "is_private": false,
           "is_virtual": false,
           "extra_data": {
-            "Documentation": "https://setuptools.readthedocs.io/"
+            "Documentation": "https://setuptools.readthedocs.io/",
+            "license_file": "LICENSE"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/setuptools",

tests/packagedcode/data/pypi/site-packages/codebase/bin/pip

@@ -5,37 +5,4 @@ click-8.0.4.dist-info/RECORD,,
 click-8.0.4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
 click-8.0.4.dist-info/top_level.txt,sha256=J1ZQogalYS4pphY_lPECoNMfw0HzTSrZglC4Yfwo4xA,6
 click/__init__.py,sha256=bOKrvMqmR9rN07vN_ycyrdF-EcTCl-EmuAjq-Fp4yPM,3243
-click/__pycache__/__init__.cpython-36.pyc,,
-click/__pycache__/_compat.cpython-36.pyc,,
-click/__pycache__/_termui_impl.cpython-36.pyc,,
-click/__pycache__/_textwrap.cpython-36.pyc,,
-click/__pycache__/_unicodefun.cpython-36.pyc,,
-click/__pycache__/_winconsole.cpython-36.pyc,,
-click/__pycache__/core.cpython-36.pyc,,
-click/__pycache__/decorators.cpython-36.pyc,,
-click/__pycache__/exceptions.cpython-36.pyc,,
-click/__pycache__/formatting.cpython-36.pyc,,
-click/__pycache__/globals.cpython-36.pyc,,
-click/__pycache__/parser.cpython-36.pyc,,
-click/__pycache__/shell_completion.cpython-36.pyc,,
-click/__pycache__/termui.cpython-36.pyc,,
-click/__pycache__/testing.cpython-36.pyc,,
-click/__pycache__/types.cpython-36.pyc,,
-click/__pycache__/utils.cpython-36.pyc,,
-click/_compat.py,sha256=JIHLYs7Jzz4KT9t-ds4o4jBzLjnwCiJQKqur-5iwCKI,18810
-click/_termui_impl.py,sha256=qK6Cfy4mRFxvxE8dya8RBhLpSC8HjF-lvBc6aNrPdwg,23451
-click/_textwrap.py,sha256=10fQ64OcBUMuK7mFvh8363_uoOxPlRItZBmKzRJDgoY,1353
-click/_unicodefun.py,sha256=JKSh1oSwG_zbjAu4TBCa9tQde2P9FiYcf4MBfy5NdT8,3201
-click/_winconsole.py,sha256=5ju3jQkcZD0W27WEMGqmEP4y_crUVzPCqsX_FYb7BO0,7860
-click/core.py,sha256=MYYmvqVa_dh4x4f-mr7VyNi6197ucYbJ0fsWW5EnCrA,111432
-click/decorators.py,sha256=5ZngOD72Flo8VLN29iarI0A5u_F-dxmqtUqx4KN8hOg,14879
-click/exceptions.py,sha256=7gDaLGuFZBeCNwY9ERMsF2-Z3R9Fvq09Zc6IZSKjseo,9167
-click/formatting.py,sha256=Frf0-5W33-loyY_i9qrwXR8-STnW3m5gvyxLVUdyxyk,9706
-click/globals.py,sha256=TP-qM88STzc7f127h35TD_v920FgfOD2EwzqA0oE8XU,1961
-click/parser.py,sha256=cAEt1uQR8gq3-S9ysqbVU-fdAZNvilxw4ReJ_T1OQMk,19044
-click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-click/shell_completion.py,sha256=HHH0wMwlDW4WQhs8rRcS9M8MAo9gltGBN6V2PEbsTp0,18003
-click/termui.py,sha256=mZ12uc3-loFaV__vr8buxn9uIjAhy7QwVuZOQ8jDdjc,28873
-click/testing.py,sha256=ptpMYgRY7dVfE3UDgkgwayu9ePw98sQI3D7zZXiCpj4,16063
-click/types.py,sha256=rj2g3biAXKkNKV8vlwbIKSUlixhXybH84N84fwCYqUU,35092
-click/utils.py,sha256=M8tuplcFFHROha3vQ60ZRSakSB_ng6w9e8Uc1AugPZ0,18934
+click/core.py,sha256=MYYmvqVa_dh4x4f-mr7VyNi6197ucYbJ0fsWW5EnCrA,111432