Fix license file parsing in pypi metadata files

Reference: #4174
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/packagedcode/pypi.py

@@ -1642,10 +1642,10 @@ def get_declared_license(metainfo):
     # TODO: We should make the declared license as it is, this should be
     # updated in scancode to parse a pure string
     lic = get_attribute(metainfo, 'License')
-
-    license_file = None
-    if lic and 'file' in lic:
-        license_file = lic.pop('file')
+    license_file = get_attribute(metainfo, 'License-File')
+    if not license_file and lic:
+        if isinstance(lic, dict) and 'file' in lic.keys():
+            license_file = lic.pop('file')
 
     if lic and not lic == 'UNKNOWN':
         if 'text' in lic:

tests/packagedcode/data/pypi/site-packages/site-packages-expected.json

@@ -187,7 +187,8 @@
         "Documentation": "https://click.palletsprojects.com/",
         "Changes": "https://click.palletsprojects.com/changes/",
         "Twitter": "https://twitter.com/PalletsTeam",
-        "Chat": "https://discord.gg/pallets"
+        "Chat": "https://discord.gg/pallets",
+        "license_file": "LICENSE.rst"
       },
       "repository_homepage_url": "https://pypi.org/project/click",
       "repository_download_url": "https://pypi.org/packages/source/c/click/click-8.0.4.tar.gz",
@@ -717,7 +718,8 @@
             "Documentation": "https://click.palletsprojects.com/",
             "Changes": "https://click.palletsprojects.com/changes/",
             "Twitter": "https://twitter.com/PalletsTeam",
-            "Chat": "https://discord.gg/pallets"
+            "Chat": "https://discord.gg/pallets",
+            "license_file": "LICENSE.rst"
           },
           "dependencies": [
             {

tests/packagedcode/data/pypi/solo-metadata/expected.json

@@ -93,7 +93,9 @@
       "source_packages": [],
       "is_private": false,
       "is_virtual": false,
-      "extra_data": {},
+      "extra_data": {
+        "license_file": "apache-2.0.LICENSE"
+      },
       "repository_homepage_url": "https://pypi.org/project/scancode-toolkit",
       "repository_download_url": "https://pypi.org/packages/source/s/scancode-toolkit/scancode-toolkit-31.0.0b1.tar.gz",
       "api_data_url": "https://pypi.org/pypi/scancode-toolkit/31.0.0b1/json",
@@ -207,7 +209,9 @@
           "file_references": [],
           "is_private": false,
           "is_virtual": false,
-          "extra_data": {},
+          "extra_data": {
+            "license_file": "apache-2.0.LICENSE"
+          },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/scancode-toolkit",
           "repository_download_url": "https://pypi.org/packages/source/s/scancode-toolkit/scancode-toolkit-31.0.0b1.tar.gz",

tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-expected.json

@@ -103,7 +103,8 @@
       "is_virtual": false,
       "extra_data": {
         "Documentation": "https://pip.pypa.io",
-        "Changelog": "https://pip.pypa.io/en/stable/news/"
+        "Changelog": "https://pip.pypa.io/en/stable/news/",
+        "license_file": "LICENSE.txt"
       },
       "repository_homepage_url": "https://pypi.org/project/pip",
       "repository_download_url": "https://pypi.org/packages/source/p/pip/pip-22.0.4.tar.gz",
@@ -264,7 +265,8 @@
           "is_virtual": false,
           "extra_data": {
             "Documentation": "https://pip.pypa.io",
-            "Changelog": "https://pip.pypa.io/en/stable/news/"
+            "Changelog": "https://pip.pypa.io/en/stable/news/",
+            "license_file": "LICENSE.txt"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/pip",
@@ -1111,7 +1113,8 @@
           "is_virtual": false,
           "extra_data": {
             "Documentation": "https://pip.pypa.io",
-            "Changelog": "https://pip.pypa.io/en/stable/news/"
+            "Changelog": "https://pip.pypa.io/en/stable/news/",
+            "license_file": "LICENSE.txt"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/pip",

tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json

@@ -93,7 +93,8 @@
           "is_virtual": false,
           "extra_data": {
             "Documentation": "https://pip.pypa.io",
-            "Changelog": "https://pip.pypa.io/en/stable/news/"
+            "Changelog": "https://pip.pypa.io/en/stable/news/",
+            "license_file": "LICENSE.txt"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/pip",
@@ -409,12 +410,12 @@
           "notice_text": null,
           "source_packages": [],
           "file_references": [],
+          "is_private": false,
+          "is_virtual": false,
           "extra_data": {
             "Documentation": "https://pip.pypa.io",
             "Changelog": "https://pip.pypa.io/en/stable/news/"
           },
-          "is_private": false,
-          "is_virtual": false,
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/pip",
           "repository_download_url": null,
@@ -793,7 +794,8 @@
           "is_virtual": false,
           "extra_data": {
             "Documentation": "https://pip.pypa.io",
-            "Changelog": "https://pip.pypa.io/en/stable/news/"
+            "Changelog": "https://pip.pypa.io/en/stable/news/",
+            "license_file": "LICENSE.txt"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/pip",

tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-with-license-expected.json

@@ -103,7 +103,8 @@
       "is_virtual": false,
       "extra_data": {
         "Documentation": "https://pip.pypa.io",
-        "Changelog": "https://pip.pypa.io/en/stable/news/"
+        "Changelog": "https://pip.pypa.io/en/stable/news/",
+        "license_file": "LICENSE.txt"
       },
       "repository_homepage_url": "https://pypi.org/project/pip",
       "repository_download_url": "https://pypi.org/packages/source/p/pip/pip-22.0.4.tar.gz",
@@ -518,7 +519,8 @@
           "is_virtual": false,
           "extra_data": {
             "Documentation": "https://pip.pypa.io",
-            "Changelog": "https://pip.pypa.io/en/stable/news/"
+            "Changelog": "https://pip.pypa.io/en/stable/news/",
+            "license_file": "LICENSE.txt"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/pip",
@@ -1660,7 +1662,8 @@
           "is_virtual": false,
           "extra_data": {
             "Documentation": "https://pip.pypa.io",
-            "Changelog": "https://pip.pypa.io/en/stable/news/"
+            "Changelog": "https://pip.pypa.io/en/stable/news/",
+            "license_file": "LICENSE.txt"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/pip",

tests/packagedcode/data/pypi/unpacked_sdist/prefer-egg-info-pkg-info/celery-expected.json

@@ -107,7 +107,8 @@
       "extra_data": {
         "Documentation": "https://docs.celeryproject.org/en/latest/index.html",
         "Changelog": "https://docs.celeryproject.org/en/stable/changelog.html",
-        "Funding": "https://opencollective.com/celery"
+        "Funding": "https://opencollective.com/celery",
+        "license_file": "LICENSE"
       },
       "repository_homepage_url": "https://pypi.org/project/celery",
       "repository_download_url": "https://pypi.org/packages/source/c/celery/celery-5.2.7.tar.gz",
@@ -1336,7 +1337,8 @@
           "extra_data": {
             "Documentation": "https://docs.celeryproject.org/en/latest/index.html",
             "Changelog": "https://docs.celeryproject.org/en/stable/changelog.html",
-            "Funding": "https://opencollective.com/celery"
+            "Funding": "https://opencollective.com/celery",
+            "license_file": "LICENSE"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/celery",
@@ -7339,7 +7341,8 @@
           "extra_data": {
             "Documentation": "https://docs.celeryproject.org/en/latest/index.html",
             "Changelog": "https://docs.celeryproject.org/en/stable/changelog.html",
-            "Funding": "https://opencollective.com/celery"
+            "Funding": "https://opencollective.com/celery",
+            "license_file": "LICENSE"
           },
           "dependencies": [
             {