feat: improve declared licenses detection (#722)

- Add declared licenses from License Troves if not mapped to SPDX
license ID
- CycloneDX 1.6 mark licenses as "declared"

fixes #718

---------

Signed-off-by: Jan Kowalleck <jan.kowalleck@gmail.com>

Author: jkowalleck

cyclonedx_py/_internal/utils/license_trove_classifier.py

@@ -21,9 +21,15 @@
 All in here may have breaking change without notice.
 """
 
-
 from typing import Optional
 
+__LICENSE_TROVE_PREFIX = 'License :: '
+
+
+def is_license_trove(classifier: str) -> bool:
+    return classifier.startswith(__LICENSE_TROVE_PREFIX)
+
+
 """
 Map of trove classifiers to SPDX license ID or SPDX license expression.
 
@@ -73,6 +79,7 @@
     # !! see the ideas and cases of https://peps.python.org/pep-0639/#mapping-license-classifiers-to-spdx-identifiers
     # 'License :: OSI Approved :: Academic Free License (AFL)': which one?
     #   -  AFL-1.1
+    #   -  AFL-...
     #   -  AFL-3.0
     # 'License :: OSI Approved :: Apache Software License': which one?
     #    -  Apache-1.1
@@ -81,6 +88,9 @@
     #    -  APSL-1.0
     #    -  APSL-2.0
     # 'License :: OSI Approved :: Artistic License': which version?
+    #    - Artistic-1.0
+    #    - Artistic-...
+    #    - Artistic-3.0
     'License :: OSI Approved :: Attribution Assurance License': 'AAL',
     # 'License :: OSI Approved :: BSD License': which exactly?
     'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0',

cyclonedx_py/_internal/utils/packaging.py

@@ -21,7 +21,7 @@
 from cyclonedx.exception.model import InvalidUriException
 from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import AttachedText, ExternalReference, ExternalReferenceType, XsUri
-from cyclonedx.model.license import DisjunctiveLicense
+from cyclonedx.model.license import DisjunctiveLicense, LicenseAcknowledgement
 
 from .cdx import url_label_to_ert
 from .pep621 import classifiers2licenses
@@ -39,19 +39,26 @@
 
 def metadata2licenses(metadata: 'PackageMetadata') -> Generator['License', None, None]:
     lfac = LicenseFactory()
+    lack = LicenseAcknowledgement.DECLARED
     if 'Classifier' in metadata:
-        # see https://packaging.python.org/en/latest/specifications/core-metadata/#classifier-multiple-use
+        # see spec: https://packaging.python.org/en/latest/specifications/core-metadata/#classifier-multiple-use
         classifiers: List[str] = metadata.get_all('Classifier')  # type:ignore[assignment]
-        yield from classifiers2licenses(classifiers, lfac)
-    if 'License' in metadata and len(mlicense := metadata['License']) > 0:
-        # see https://packaging.python.org/en/latest/specifications/core-metadata/#license
-        license = lfac.make_from_string(mlicense)
+        yield from classifiers2licenses(classifiers, lfac, lack)
+    for mlicense in metadata.get_all('License', ()):
+        # see spec:  https://packaging.python.org/en/latest/specifications/core-metadata/#license
+        if len(mlicense) <= 0:
+            continue
+        license = lfac.make_from_string(mlicense,
+                                        license_acknowledgement=lack)
         if isinstance(license, DisjunctiveLicense) and license.id is None:
             # per spec, `License` is either a SPDX ID/Expression, or a license text(not name!)
             yield DisjunctiveLicense(name=f"declared license of '{metadata['Name']}'",
+                                     acknowledgement=lack,
                                      text=AttachedText(content=mlicense))
         else:
             yield license
+    # TODO: iterate over "License-File" declarations and read them
+    # for mlfile in  metadata.get_all('License-File'): ...
 
 
 def metadata2extrefs(metadata: 'PackageMetadata') -> Generator['ExternalReference', None, None]:

cyclonedx_py/_internal/utils/pep621.py

@@ -31,34 +31,34 @@
 from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import AttachedText, Encoding, ExternalReference, XsUri
 from cyclonedx.model.component import Component
-from cyclonedx.model.license import DisjunctiveLicense
+from cyclonedx.model.license import DisjunctiveLicense, LicenseAcknowledgement
 from packaging.requirements import Requirement
 
 from .cdx import licenses_fixup, url_label_to_ert
-from .license_trove_classifier import license_trove2spdx
+from .license_trove_classifier import is_license_trove, license_trove2spdx
 
 if TYPE_CHECKING:
     from cyclonedx.model.component import ComponentType
     from cyclonedx.model.license import License
 
 
-def classifiers2licenses(classifiers: Iterable[str], lfac: 'LicenseFactory') -> Generator['License', None, None]:
-    yield from map(lfac.make_from_string,
-                   # `lfac.make_with_id` could be a shortcut,
-                   # but some SPDX ID might not (yet) be known to CDX.
-                   # So better go with `lfac.make_from_string` and be safe.
-                   filter(None,
-                          map(license_trove2spdx,
-                              classifiers)))
+def classifiers2licenses(classifiers: Iterable[str], lfac: 'LicenseFactory',
+                         lack: 'LicenseAcknowledgement'
+                         ) -> Generator['License', None, None]:
+    for c in classifiers:
+        if is_license_trove(c):
+            yield lfac.make_from_string(license_trove2spdx(c) or c,
+                                        license_acknowledgement=lack)
 
 
 def project2licenses(project: Dict[str, Any], lfac: 'LicenseFactory', *,
                      fpath: str) -> Generator['License', None, None]:
+    lack = LicenseAcknowledgement.DECLARED
     if classifiers := project.get('classifiers'):
         # https://packaging.python.org/en/latest/specifications/pyproject-toml/#classifiers
         # https://peps.python.org/pep-0621/#classifiers
         # https://packaging.python.org/en/latest/specifications/core-metadata/#classifier-multiple-use
-        yield from classifiers2licenses(classifiers, lfac)
+        yield from classifiers2licenses(classifiers, lfac, lack)
     if plicense := project.get('license'):
         # https://packaging.python.org/en/latest/specifications/pyproject-toml/#license
         # https://peps.python.org/pep-0621/#license
@@ -73,13 +73,16 @@ def project2licenses(project: Dict[str, Any], lfac: 'LicenseFactory', *,
             # > Tools MUST assume the file’s encoding is UTF-8.
             with open(join(dirname(fpath), plicense['file']), 'rb') as plicense_fileh:
                 yield DisjunctiveLicense(name=f"declared license of '{project['name']}'",
+                                         acknowledgement=lack,
                                          text=AttachedText(encoding=Encoding.BASE_64,
                                                            content=b64encode(plicense_fileh.read()).decode()))
         elif len(plicense_text := plicense.get('text', '')) > 0:
-            license = lfac.make_from_string(plicense_text)
+            license = lfac.make_from_string(plicense_text,
+                                            license_acknowledgement=lack)
             if isinstance(license, DisjunctiveLicense) and license.id is None:
                 # per spec, `License` is either a SPDX ID/Expression, or a license text(not name!)
                 yield DisjunctiveLicense(name=f"declared license of '{project['name']}'",
+                                         acknowledgement=lack,
                                          text=AttachedText(content=plicense_text))
             else:
                 yield license

cyclonedx_py/_internal/utils/poetry.py

@@ -28,6 +28,7 @@
 from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import ExternalReference, ExternalReferenceType, XsUri
 from cyclonedx.model.component import Component
+from cyclonedx.model.license import LicenseAcknowledgement
 from packaging.requirements import Requirement
 
 from .cdx import licenses_fixup, url_label_to_ert
@@ -64,12 +65,14 @@ def poetry2extrefs(poetry: Dict[str, Any]) -> Generator['ExternalReference', Non
 def poetry2component(poetry: Dict[str, Any], *, ctype: 'ComponentType') -> 'Component':
     licenses: List['License'] = []
     lfac = LicenseFactory()
+    lack = LicenseAcknowledgement.DECLARED
     if 'classifiers' in poetry:
-        licenses.extend(classifiers2licenses(poetry['classifiers'], lfac))
+        licenses.extend(classifiers2licenses(poetry['classifiers'], lfac, lack))
     if 'license' in poetry:
         # per spec(https://python-poetry.org/docs/pyproject#license):
         # the `license` is intended to be the name of a license, not the license text itself.
-        licenses.append(lfac.make_from_string(poetry['license']))
+        licenses.append(lfac.make_from_string(poetry['license'],
+                                              license_acknowledgement=lack))
 
     return Component(
         type=ctype,

pyproject.toml

@@ -69,7 +69,7 @@ cyclonedx-py = "cyclonedx_py._internal.cli:run"
 
 [tool.poetry.dependencies]
 python = "^3.8"
-cyclonedx-python-lib = { version = "^7.0.0", extras = ["validation"] }
+cyclonedx-python-lib = { version = "^7.3.0", extras = ["validation"] }
 packageurl-python = ">=0.11, <2"  # keep in sync with same dep in `cyclonedx-python-lib`
 pip-requirements-parser = "^32.0"
 packaging = "^22 || ^23 || ^24"