fix: package name normalization (#652)

use package name normalization everywhere ; https://packaging.python.org/en/latest/specifications/name-normalization/
THis should fix issues with mismatched or wronl/alternative-written(but compatible) package names.

---------

Signed-off-by: Jan Kowalleck <jan.kowalleck@gmail.com>

Author: jkowalleck

cyclonedx_py/_internal/environment.py

@@ -34,7 +34,7 @@
 from . import BomBuilder, PropertyName
 from .cli_common import add_argument_mc_type, add_argument_pyproject
 from .utils.cdx import licenses_fixup, make_bom
-from .utils.packaging import metadata2extrefs, metadata2licenses
+from .utils.packaging import metadata2extrefs, metadata2licenses, normalize_packagename
 from .utils.pep610 import PackageSourceArchive, PackageSourceVcs, packagesource2extref, packagesource4dist
 from .utils.pyproject import pyproject2component, pyproject2dependencies, pyproject_load
 
@@ -129,7 +129,7 @@ def __call__(self, *,  # type:ignore[override]
             pyproject = pyproject_load(pyproject_file)
             root_c = pyproject2component(pyproject, type=mc_type)
             root_c.bom_ref.value = 'root-component'
-            root_d = pyproject2dependencies(pyproject)
+            root_d = tuple(pyproject2dependencies(pyproject))
             rc = (root_c, root_d)
 
         path: List[str]
@@ -166,20 +166,23 @@ def __add_components(self, bom: 'Bom',
             )
             del dist_meta, dist_name, dist_version
             self.__component_add_extred_and_purl(component, packagesource4dist(dist))
-            all_components[component.name.lower()] = component, map(Requirement, dist.requires or ())
+            all_components[normalize_packagename(component.name)] = (
+                component,
+                tuple(map(Requirement, dist.requires or ()))
+            )
 
             self._logger.info('add component for package %r', component.name)
             self._logger.debug('add component: %r', component)
             bom.components.add(component)
 
         if rc is not None:
             root_c = rc[0]
-            root_c_lcname = root_c.name.lower()
-            root_c_existed = all_components.get(root_c_lcname)
+            root_c_nname = normalize_packagename(root_c.name)
+            root_c_existed = all_components.get(root_c_nname)
             if root_c_existed is not None:
                 bom.components.remove(root_c_existed[0])
                 del root_c_existed
-            all_components[root_c_lcname] = rc
+            all_components[root_c_nname] = rc
             bom.metadata.component = root_c
             self._logger.debug('root-component: %r', root_c)
 
@@ -189,7 +192,7 @@ def __finalize_dependencies(self, bom: 'Bom', all_components: 'T_AllComponents')
         for component, requires in all_components.values():
             component_deps: List[Component] = []
             for req in requires:
-                req_component: Optional[Component] = all_components.get(req.name.lower(), (None,))[0]
+                req_component: Optional[Component] = all_components.get(normalize_packagename(req.name), (None,))[0]
                 if req_component is None:
                     continue
                 if req_component is component:

cyclonedx_py/_internal/pipenv.py

@@ -32,6 +32,7 @@
 from .cli_common import add_argument_mc_type, add_argument_pyproject
 from .utils.args import arparse_split
 from .utils.cdx import make_bom
+from .utils.packaging import normalize_packagename
 from .utils.pyproject import pyproject_file2component
 from .utils.secret import redact_auth_from_url
 
@@ -151,16 +152,17 @@ def _make_bom(self, root_c: Optional['Component'],
 
         all_components: Dict[str, Component] = {}
         if root_c:
-            # root for self-installs
-            all_components[root_c.name] = root_c
+            # root for possible self-installs
+            all_components[normalize_packagename(root_c.name)] = root_c
         for group_name in use_groups:
             self._logger.debug('processing group %r ...', group_name)
             for package_name, package_data in locker.get(group_name, {}).items():
-                if package_name in all_components:
-                    component = all_components[package_name]
+                package_name_normalized = normalize_packagename(package_name)
+                if package_name_normalized in all_components:
+                    component = all_components[package_name_normalized]
                     self._logger.info('existing component for package %r', package_name)
                 else:
-                    component = all_components[package_name] = Component(
+                    component = all_components[package_name_normalized] = Component(
                         bom_ref=f'{package_name}{package_data.get("version", "")}',
                         type=ComponentType.LIBRARY,
                         name=package_name,

cyclonedx_py/_internal/poetry.py

@@ -32,6 +32,7 @@
 from . import BomBuilder, PropertyName
 from .cli_common import add_argument_mc_type
 from .utils.cdx import make_bom
+from .utils.packaging import normalize_packagename
 from .utils.poetry import poetry2component
 from .utils.secret import redact_auth_from_url
 from .utils.toml import toml_loads
@@ -229,9 +230,9 @@ def _make_bom(self, project: 'NameDict', locker: 'NameDict',
         ) for extra in use_extras)
         self._logger.debug('root-component: %r', root_c)
 
-        lock_data: Dict[str, _LockEntry] = {le.name.lower(): le for le in self._parse_lock(locker)}
+        lock_data: Dict[str, _LockEntry] = {normalize_packagename(le.name): le for le in self._parse_lock(locker)}
 
-        lock_data[root_c.name] = _LockEntry(  # needed for circle dependencies
+        lock_data[normalize_packagename(root_c.name)] = _LockEntry(  # needed for circle dependencies
             name=root_c.name,
             component=root_c,
             dependencies=set(),
@@ -243,15 +244,10 @@ def _make_bom(self, project: 'NameDict', locker: 'NameDict',
 
         _dep_pattern = re_compile(r'^(?P<name>[^\[]+)(?:\[(?P<extras>.*)\])?$')
 
-        lock_version = self._get_lockfile_version(locker)
-        should_tidy_lock_names = lock_version >= (2,)
-
         def _add_ld(name: str, extras: Set[str]) -> Optional['Component']:
-            name = name.lower()
+            name = normalize_packagename(name)
             if name == 'python':
                 return None
-            if should_tidy_lock_names:
-                name = name.replace('.', '-')
             le = lock_data.get(name)
             if le is None:
                 self._logger.warning('skip unlocked component: %s', name)
@@ -292,9 +288,7 @@ def _add_ld(name: str, extras: Set[str]) -> Optional['Component']:
         for group_name in use_groups:
             self._logger.debug('processing group %r ...', group_name)
             for dep_name, dep_spec in po_cfg['group'][group_name].get('dependencies', {}).items():
-                dep_name = dep_name.lower()
-                if should_tidy_lock_names:
-                    dep_name = dep_name.replace('.', '-')
+                dep_name = normalize_packagename(dep_name)
                 self._logger.debug('root-component depends on %s', dep_name)
                 if dep_name == 'python':
                     continue
@@ -389,7 +383,7 @@ def __purl_qualifiers4lock(self, package: 'NameDict') -> 'NameDict':
             # > For version-controlled files, the VCS location syntax is similar to a URL and has the:
             # > `<vcs_tool>+<transport>://<host_name>[/<path_to_repository>][@<revision_tag_or_branch>][#<sub_path>]`
             qs['vcs_url'] = f'{source["type"]}+{redact_auth_from_url(source["url"])}@' + \
-                source.get('resolved_reference', source.get('reference', ''))
+                            source.get('resolved_reference', source.get('reference', ''))
         elif source_type == 'url':
             if '://files.pythonhosted.org/' not in source['url']:
                 # skip PURL bloat, do not add implicit information

cyclonedx_py/_internal/utils/packaging.py

@@ -15,6 +15,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright (c) OWASP Foundation. All Rights Reserved.
 
+from re import compile as re_compile
 from typing import TYPE_CHECKING, Generator, List
 
 from cyclonedx.exception.model import InvalidUriException
@@ -71,3 +72,12 @@ def metadata2extrefs(metadata: 'PackageMetadata') -> Generator['ExternalReferenc
                 url=XsUri(url.strip()))
         except InvalidUriException:  # pragma: nocover
             pass
+
+
+# see https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
+_NORMALIZE_PN_MATCHER = re_compile(r'[-_.]+')
+_NORMALIZE_PN_REPLACE = '-'
+
+
+def normalize_packagename(name: str) -> str:
+    return _NORMALIZE_PN_MATCHER.sub(_NORMALIZE_PN_REPLACE, name).lower()

tests/_data/infiles/environment/normalize-packagename/init.py

@@ -0,0 +1,59 @@
+"""
+initialize this testbed.
+"""
+
+from os import name as os_name
+from os.path import dirname, join
+from subprocess import PIPE, CompletedProcess, run  # nosec:B404
+from sys import argv, executable
+from typing import Any
+from venv import EnvBuilder
+
+__all__ = ['main']
+
+this_dir = dirname(__file__)
+env_dir = join(this_dir, '.venv')
+constraint_file = join(this_dir, 'pinning.txt')
+
+
+def pip_run(*args: str, **kwargs: Any) -> CompletedProcess:
+    # pip is not API, but a CLI -- call it like that!
+    call = (
+        executable, '-m', 'pip',
+        '--python', env_dir,
+        *args
+    )
+    print('+ ', *call)
+    res = run(call, **kwargs, cwd=this_dir, shell=False)  # nosec:B603
+    if res.returncode != 0:
+        raise RuntimeError('process failed')
+    return res
+
+
+def pip_install(*args: str) -> None:
+    pip_run(
+        'install', '--require-virtualenv', '--no-input', '--progress-bar=off', '--no-color',
+        '-c', constraint_file,  # needed for reproducibility
+        *args
+    )
+
+
+def main() -> None:
+    EnvBuilder(
+        system_site_packages=False,
+        symlinks=os_name != 'nt',
+        with_pip=False,
+    ).create(env_dir)
+
+    pip_install(
+        # https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
+        'ruamel-YAML[jinja2]',  # actually "ruamel.yaml", normalizes to "ruamel-yaml"
+    )
+
+
+if __name__ == '__main__':
+    main()
+    if '--pin' in argv:
+        res = pip_run('freeze', '--all', '--local', stdout=PIPE)
+        with open(constraint_file, 'wb') as cf:
+            cf.write(res.stdout)

tests/_data/infiles/environment/normalize-packagename/pinning.txt

@@ -0,0 +1,3 @@
+ruamel.yaml==0.18.5
+ruamel.yaml.clib==0.2.8
+ruamel.yaml.jinja2==0.2.7

tests/_data/infiles/environment/normalize-packagename/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata
+name = "normalize-packagename"
+version = "0.1.0"
+description = "packages with non-normalized names"
+
+# see https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
+
+dependencies = [
+  "ruamel-YAML[jinja2]"   # actually "ruamel.yaml", normalizes to "ruamel-yaml"
+]

tests/_data/infiles/pipenv/normalize-packagename/Pipfile

@@ -0,0 +1,7 @@
+[pipenv]
+sort_pipfile = true
+
+[packages]
+"ruamel-YAML[jinja2]" = "*"   # actually "ruamel.yaml", normalizes to "ruamel-yaml"
+
+[dev-packages]

tests/_data/infiles/pipenv/normalize-packagename/Pipfile.lock

@@ -0,0 +1,12 @@
+[project]
+# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata
+name = "normalize-packagename"
+version = "0.1.0"
+description = "packages with non-normalized names"
+
+# see https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
+
+dependencies = [
+  "ruamel-YAML[jinja2]"   # actually "ruamel.yaml", normalizes to "ruamel-yaml"
+]
+