feat: finalize PEP639

Signed-off-by: Jan Kowalleck <[email protected]>

Author: jkowalleck

cyclonedx_py/_internal/cli_common.py

@@ -30,8 +30,7 @@ def add_argument_pyproject(p: 'ArgumentParser') -> 'Action':
     return p.add_argument('--pyproject',
                           metavar='<file>',
                           help="Path to the root component's `pyproject.toml` file. "
-                               'This should point to a file compliant with PEP 621 '
-                               '(storing project metadata).',
+                               'This should point to a file compliant with PEP 621 (storing project metadata).',
                           dest='pyproject_file',
                           default=None)
 

cyclonedx_py/_internal/environment.py

@@ -27,21 +27,18 @@
 from textwrap import dedent
 from typing import TYPE_CHECKING, Any, Optional
 
+from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import Property
-from cyclonedx.model.component import (  # type:ignore[attr-defined]  # ComponentEvidence was moved, but is still importable - ignore/wont-fix for backwards compatibility # noqa:E501
-    Component,
-    ComponentEvidence,
-    ComponentType,
-)
+from cyclonedx.model.component import Component, ComponentType
 from packageurl import PackageURL
 from packaging.requirements import Requirement
 
 from . import BomBuilder, PropertyName, PurlTypePypi
 from .cli_common import add_argument_mc_type, add_argument_pyproject
-from .utils.cdx import find_LicenseExpression, licenses_fixup, make_bom
+from .utils.cdx import licenses_fixup, make_bom
 from .utils.packaging import metadata2extrefs, metadata2licenses, normalize_packagename
 from .utils.pep610 import PackageSourceArchive, PackageSourceVcs, packagesource2extref, packagesource4dist
-from .utils.pep639 import dist2licenses as dist2licenses_pep639
+from .utils.pep639 import dist2licenses as pep639_dist2licenses
 from .utils.pyproject import pyproject2component, pyproject2dependencies, pyproject_load
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -114,12 +111,6 @@ def make_argument_parser(**kwargs: Any) -> 'ArgumentParser':
                  • Build an SBOM from uv environment:
                      $ %(prog)s "$(uv python find)"
                """)
-        p.add_argument('--PEP-639',
-                       action='store_true',
-                       dest='pep639',
-                       help='Enable license gathering according to PEP 639 '
-                            '(improving license clarity with better package metadata).\n'
-                            'The behavior may change during the draft development of the PEP.')
         p.add_argument('--gather-license-texts',
                        action='store_true',
                        dest='gather_license_texts',
@@ -156,7 +147,7 @@ def __call__(self, *,  # type:ignore[override]
             rc = None
         else:
             pyproject = pyproject_load(pyproject_file)
-            root_c = pyproject2component(pyproject, ctype=mc_type, fpath=pyproject_file)
+            root_c = pyproject2component(pyproject, ctype=mc_type, fpath=pyproject_file, gather_license_texts=False)
             root_c.bom_ref.value = 'root-component'
             root_d = tuple(pyproject2dependencies(pyproject))
             rc = (root_c, root_d)
@@ -189,25 +180,20 @@ def __add_components(self, bom: 'Bom',
                 name=dist_name,
                 version=dist_version,
                 description=dist_meta['Summary'] if 'Summary' in dist_meta else None,
-                licenses=licenses_fixup(metadata2licenses(dist_meta)),
                 external_references=metadata2extrefs(dist_meta),
                 # path of dist-package on disc? naaa... a package may have multiple files/folders on disc
             )
-            if self._pep639:
-                pep639_licenses = list(dist2licenses_pep639(dist, self._gather_license_texts, self._logger))
-                pep639_lexp = find_LicenseExpression(pep639_licenses)
-                if pep639_lexp is not None:
-                    component.licenses = (pep639_lexp,)
-                    pep639_licenses.remove(pep639_lexp)
-                if len(pep639_licenses) > 0:
-                    if find_LicenseExpression(component.licenses) is None:
-                        component.licenses.update(pep639_licenses)
-                    else:
-                        # hack for preventing expressions AND named licenses.
-                        # see https://github.com/CycloneDX/cyclonedx-python/issues/826
-                        # see https://github.com/CycloneDX/specification/issues/454
-                        component.evidence = ComponentEvidence(licenses=pep639_licenses)
-                del pep639_lexp, pep639_licenses
+
+            # region licenses
+            lfac = LicenseFactory()
+            component.licenses.update(pep639_dist2licenses(dist, lfac, self._gather_license_texts, self._logger))
+            if len(component.licenses) == 0:
+                # According to PEP 639 spec, if licenses are declared in the "new" style,
+                # all other license declarations MUST be ignored.
+                # https://peps.python.org/pep-0639/#converting-legacy-metadata
+                component.licenses.update(metadata2licenses(dist_meta, lfac))
+            licenses_fixup(component)
+            # endregion licenses
 
             del dist_meta, dist_name, dist_version
             self.__component_add_extref_and_purl(component, packagesource4dist(dist))

cyclonedx_py/_internal/pipenv.py

@@ -132,7 +132,7 @@ def __call__(self, *,  # type:ignore[override]
             if pyproject_file is None:
                 rc = None
             else:
-                rc = pyproject_file2component(pyproject_file, ctype=mc_type)
+                rc = pyproject_file2component(pyproject_file, ctype=mc_type, gather_license_texts=False)
                 rc.bom_ref.value = 'root-component'
 
             return self._make_bom(rc,

cyclonedx_py/_internal/requirements.py

@@ -116,7 +116,7 @@ def __call__(self, *,  # type:ignore[override]
         if pyproject_file is None:
             rc = None
         else:
-            rc = pyproject_file2component(pyproject_file, ctype=mc_type)
+            rc = pyproject_file2component(pyproject_file, ctype=mc_type, gather_license_texts=False)
             rc.bom_ref.value = 'root-component'
 
         if requirements_file == '-':

cyclonedx_py/_internal/utils/cdx.py

@@ -27,7 +27,11 @@
 from cyclonedx.builder.this import this_component as lib_component
 from cyclonedx.model import ExternalReference, ExternalReferenceType, XsUri
 from cyclonedx.model.bom import Bom
-from cyclonedx.model.component import Component, ComponentType
+from cyclonedx.model.component import (  # type:ignore[attr-defined]  # ComponentEvidence was moved, but is still importable - ignore/wont-fix for backwards compatibility  # noqa:E501
+    Component,
+    ComponentEvidence,
+    ComponentType,
+)
 from cyclonedx.model.license import DisjunctiveLicense, License, LicenseAcknowledgement, LicenseExpression
 
 from ... import __version__ as _THIS_VERSION  # noqa:N812
@@ -95,11 +99,20 @@ def find_LicenseExpression(licenses: Iterable['License']) -> Optional[LicenseExp
     return None
 
 
-def licenses_fixup(licenses: Iterable['License']) -> Iterable['License']:
-    licenses = set(licenses)
-    if (lexp := find_LicenseExpression(licenses)) is not None:
-        return (lexp,)
-    return licenses
+def licenses_fixup(component: 'Component') -> None:
+    """
+    Per CycloneDX spec, there must be EITHER one license expression OR multiple license id/name.
+    If there is an expression, it is used and everything else is moved to evidences, so it is not lost.
+    """
+    licenses = list(component.licenses)
+    lexp = find_LicenseExpression(licenses)
+    if lexp is None:
+        return
+    component.licenses = (lexp,)
+    licenses.remove(lexp)
+    if component.evidence is None:
+        component.evidence = ComponentEvidence()
+    component.evidence.licenses.update(licenses)
 
 
 _MAP_KNOWN_URL_LABELS: dict[str, ExternalReferenceType] = {

cyclonedx_py/_internal/utils/packaging.py

@@ -20,7 +20,6 @@
 from typing import TYPE_CHECKING
 
 from cyclonedx.exception.model import InvalidUriException
-from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import AttachedText, ExternalReference, ExternalReferenceType, XsUri
 from cyclonedx.model.license import DisjunctiveLicense, LicenseAcknowledgement
 
@@ -30,6 +29,7 @@
 if TYPE_CHECKING:  # pragma: no cover
     import sys
 
+    from cyclonedx.factory.license import LicenseFactory
     from cyclonedx.model.license import License
 
     if sys.version_info >= (3, 10):
@@ -38,8 +38,7 @@
         from email.message import Message as PackageMetadata
 
 
-def metadata2licenses(metadata: 'PackageMetadata') -> Generator['License', None, None]:
-    lfac = LicenseFactory()
+def metadata2licenses(metadata: 'PackageMetadata', lfac: 'LicenseFactory') -> Generator['License', None, None]:
     lack = LicenseAcknowledgement.DECLARED
     if 'Classifier' in metadata:
         # see spec: https://packaging.python.org/en/latest/specifications/core-metadata/#classifier-multiple-use

cyclonedx_py/_internal/utils/pep621.py

@@ -29,16 +29,16 @@
 from typing import TYPE_CHECKING, Any
 
 from cyclonedx.exception.model import InvalidUriException
-from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import AttachedText, Encoding, ExternalReference, XsUri
 from cyclonedx.model.component import Component
 from cyclonedx.model.license import DisjunctiveLicense, LicenseAcknowledgement
 from packaging.requirements import Requirement
 
-from .cdx import licenses_fixup, url_label_to_ert
+from .cdx import url_label_to_ert
 from .license_trove_classifier import is_license_trove, license_trove2spdx
 
 if TYPE_CHECKING:
+    from cyclonedx.factory.license import LicenseFactory
     from cyclonedx.model.component import ComponentType
     from cyclonedx.model.license import License
 
@@ -52,7 +52,8 @@ def classifiers2licenses(classifiers: Iterable[str], lfac: 'LicenseFactory',
                                         license_acknowledgement=lack)
 
 
-def project2licenses(project: dict[str, Any], lfac: 'LicenseFactory', *,
+def project2licenses(project: dict[str, Any], lfac: 'LicenseFactory',
+                     gather_text: bool, *,
                      fpath: str) -> Generator['License', None, None]:
     lack = LicenseAcknowledgement.DECLARED
     if classifiers := project.get('classifiers'):
@@ -68,10 +69,11 @@ def project2licenses(project: dict[str, Any], lfac: 'LicenseFactory', *,
             # per spec:
             # > These keys are mutually exclusive, so a tool MUST raise an error if the metadata specifies both keys.
             raise ValueError('`license.file` and `license.text` are mutually exclusive,')
-        if 'file' in plicense:
+        if gather_text and 'file' in plicense:
             # per spec:
             # > [...] a string value that is a relative file path [...].
             # > Tools MUST assume the file’s encoding is UTF-8.
+            # anyway, we don't trust this and assume binary
             with open(join(dirname(fpath), plicense['file']), 'rb') as plicense_fileh:
                 yield DisjunctiveLicense(name=f"declared license of '{project['name']}'",
                                          acknowledgement=lack,
@@ -80,7 +82,7 @@ def project2licenses(project: dict[str, Any], lfac: 'LicenseFactory', *,
         elif len(plicense_text := plicense.get('text', '')) > 0:
             license = lfac.make_from_string(plicense_text,
                                             license_acknowledgement=lack)
-            if isinstance(license, DisjunctiveLicense) and license.id is None:
+            if isinstance(license, DisjunctiveLicense) and license.id is None and gather_text:
                 # per spec, `License` is either a SPDX ID/Expression, or a license text(not name!)
                 yield DisjunctiveLicense(name=f"declared license of '{project['name']}'",
                                          acknowledgement=lack,
@@ -103,15 +105,15 @@ def project2extrefs(project: dict[str, Any]) -> Generator['ExternalReference', N
 
 
 def project2component(project: dict[str, Any], *,
-                      ctype: 'ComponentType', fpath: str) -> 'Component':
+                      ctype: 'ComponentType') -> 'Component':
     dynamic = project.get('dynamic', ())
     return Component(
         type=ctype,
         name=project['name'],
         version=project.get('version', None) if 'version' not in dynamic else None,
         description=project.get('description', None) if 'description' not in dynamic else None,
-        licenses=licenses_fixup(project2licenses(project, LicenseFactory(), fpath=fpath)),
         external_references=project2extrefs(project),
+        # licenses are not gathered here per default, they may be sourced otherwise
         # TODO add more properties according to spec
     )
 

cyclonedx_py/_internal/utils/pep639.py

@@ -23,10 +23,10 @@
 
 from base64 import b64encode
 from collections.abc import Generator
-from os.path import join
-from typing import TYPE_CHECKING
+from glob import glob
+from os.path import dirname, join
+from typing import TYPE_CHECKING, Any
 
-from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import AttachedText, Encoding
 from cyclonedx.model.license import DisjunctiveLicense, LicenseAcknowledgement
 
@@ -37,26 +37,53 @@
     from importlib.metadata import Distribution
     from logging import Logger
 
+    from cyclonedx.factory.license import LicenseFactory
     from cyclonedx.model.license import License
 
+
+def project2licenses(project: dict[str, Any], lfac: 'LicenseFactory',
+                     gather_texts: bool, *,
+                     fpath: str) -> Generator['License', None, None]:
+    lack = LicenseAcknowledgement.DECLARED
+    if isinstance(plicense := project.get('license'), str) \
+            and len(plicense) > 0:
+        # https://peps.python.org/pep-0639/#add-string-value-to-license-key
+        yield lfac.make_from_string(plicense,
+                                    license_acknowledgement=lack)
+    if gather_texts and isinstance(plfiles := project.get('license-files'), list):
+        # https://peps.python.org/pep-0639/#add-license-files-key
+        plfiles_root = dirname(fpath)
+        for plfile_glob in plfiles:
+            for plfile in glob(plfile_glob, root_dir=plfiles_root):
+                # per spec:
+                # > Tools MUST assume that license file content is valid UTF-8 encoded text
+                # anyway, we don't trust this and assume binary
+                with open(join(plfiles_root, plfile), 'rb') as plicense_fileh:
+                    yield DisjunctiveLicense(name=f'declared license file: {plfile}',
+                                             acknowledgement=lack,
+                                             text=AttachedText(encoding=Encoding.BASE_64,
+                                                               content=b64encode(plicense_fileh.read()).decode()))
+    # Silently skip any other types (including string/PEP 621)
+    return None
+
+
 # per spec > license files are stored in the `.dist-info/licenses/` subdirectory of the produced wheel.
 # but in practice, other locations are used, too.
 _LICENSE_LOCATIONS = ('licenses', 'license_files', '')
 
 
 def dist2licenses(
-    dist: 'Distribution',
-    gather_text: bool,
+    dist: 'Distribution', lfac: 'LicenseFactory',
+    gather_texts: bool,
     logger: 'Logger'
 ) -> Generator['License', None, None]:
-    lfac = LicenseFactory()
     lack = LicenseAcknowledgement.DECLARED
     metadata = dist.metadata  # see https://packaging.python.org/en/latest/specifications/core-metadata/
     if (lexp := metadata['License-Expression']) is not None:
         # see spec: https://peps.python.org/pep-0639/#add-license-expression-field
         yield lfac.make_from_string(lexp,
                                     license_acknowledgement=lack)
-    if gather_text:
+    if gather_texts:
         for mlfile in set(metadata.get_all('License-File', ())):
             # see spec: https://peps.python.org/pep-0639/#add-license-file-field
             # latest spec rev: https://discuss.python.org/t/pep-639-round-3-improving-license-clarity-with-better-package-metadata/53020  # noqa: E501

cyclonedx_py/_internal/utils/poetry.py

@@ -37,7 +37,6 @@
 
 if TYPE_CHECKING:
     from cyclonedx.model.component import ComponentType
-    from cyclonedx.model.license import License
 
 
 def poetry2extrefs(poetry: dict[str, Any]) -> Generator['ExternalReference', None, None]:
@@ -64,26 +63,27 @@ def poetry2extrefs(poetry: dict[str, Any]) -> Generator['ExternalReference', Non
 
 
 def poetry2component(poetry: dict[str, Any], *, ctype: 'ComponentType') -> 'Component':
-    licenses: list['License'] = []
-    lfac = LicenseFactory()
-    lack = LicenseAcknowledgement.DECLARED
-    if 'classifiers' in poetry:
-        licenses.extend(classifiers2licenses(poetry['classifiers'], lfac, lack))
-    if 'license' in poetry:
-        # per spec(https://python-poetry.org/docs/pyproject#license):
-        # the `license` is intended to be the name of a license, not the license text itself.
-        licenses.append(lfac.make_from_string(poetry['license'],
-                                              license_acknowledgement=lack))
-
-    return Component(
+    component = Component(
         type=ctype,
         name=poetry['name'],
         version=poetry.get('version'),
         description=poetry.get('description'),
-        licenses=licenses_fixup(licenses),
         external_references=poetry2extrefs(poetry),
         # TODO add more properties according to spec
     )
+    # region licenses
+    lfac = LicenseFactory()
+    lack = LicenseAcknowledgement.DECLARED
+    if 'classifiers' in poetry:
+        component.licenses.update(classifiers2licenses(poetry['classifiers'], lfac, lack))
+    if 'license' in poetry:
+        # per spec(https://python-poetry.org/docs/pyproject#license):
+        # the `license` is intended to be the name of a license, not the license text itself.
+        component.licenses.add(lfac.make_from_string(poetry['license'],
+                                                     license_acknowledgement=lack))
+    licenses_fixup(component)
+    # endregion licenses
+    return component
 
 
 def poetry2dependencies(poetry: dict[str, Any]) -> Generator['Requirement', None, None]: