Skip to content

Commit 415ba2f

Browse files
Arun Selvamaniclaude
Arun Selvamani
and
claude
committed
feat: data-driven label classification for project URLs (#800)
Move URL-label -> external-reference-type mapping into a dedicated data-only module and add PyPI-style label prefix matching. Adds an optional `url` argument to url_label_to_ert (unused for now) for the upcoming host-based classification. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Signed-off-by: Arun Selvamani <arunselvamani@gmail.com>
1 parent b7abcd5 commit 415ba2f

3 files changed

Lines changed: 128 additions & 27 deletions

File tree

‎cyclonedx_py/_internal/utils/cdx.py‎

Lines changed: 12 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
from cyclonedx.model.license import DisjunctiveLicense, License, LicenseAcknowledgement, LicenseExpression
3636

3737
from ... import __version__ as _THIS_VERSION # noqa:N812
38+
from .url_classifiers import _MAP_KNOWN_URL_LABELS, _MAP_URL_LABEL_PREFIXES
3839

3940

4041
def make_bom(**kwargs: Any) -> Bom:
@@ -119,32 +120,17 @@ def licenses_fixup(component: 'Component') -> None:
119120
component.evidence.licenses.update(licenses)
120121

121122

122-
_MAP_KNOWN_URL_LABELS: dict[str, ExternalReferenceType] = {
123-
# see https://peps.python.org/pep-0345/#project-url-multiple-use
124-
# see https://github.com/pypi/warehouse/issues/5947#issuecomment-699660629
125-
'bugtracker': ExternalReferenceType.ISSUE_TRACKER,
126-
'issuetracker': ExternalReferenceType.ISSUE_TRACKER,
127-
'issues': ExternalReferenceType.ISSUE_TRACKER,
128-
'bugreports': ExternalReferenceType.ISSUE_TRACKER,
129-
'tracker': ExternalReferenceType.ISSUE_TRACKER,
130-
'home': ExternalReferenceType.WEBSITE,
131-
'homepage': ExternalReferenceType.WEBSITE,
132-
'download': ExternalReferenceType.DISTRIBUTION,
133-
'documentation': ExternalReferenceType.DOCUMENTATION,
134-
'docs': ExternalReferenceType.DOCUMENTATION,
135-
'changelog': ExternalReferenceType.RELEASE_NOTES,
136-
'changes': ExternalReferenceType.RELEASE_NOTES,
137-
# 'source': ExternalReferenceType.SOURCE-DISTRIBUTION,
138-
'repository': ExternalReferenceType.VCS,
139-
'github': ExternalReferenceType.VCS,
140-
'chat': ExternalReferenceType.CHAT,
141-
}
142-
143123
_NOCHAR_MATCHER = re_compile('[^a-z]')
144124

145125

146-
def url_label_to_ert(value: str) -> ExternalReferenceType:
147-
return _MAP_KNOWN_URL_LABELS.get(
148-
_NOCHAR_MATCHER.sub('', str(value).lower()),
149-
ExternalReferenceType.OTHER
150-
)
126+
def url_label_to_ert(label: str, url: Optional[str] = None) -> ExternalReferenceType:
127+
norm = _NOCHAR_MATCHER.sub('', str(label).lower())
128+
# 1. exact label
129+
ert = _MAP_KNOWN_URL_LABELS.get(norm)
130+
if ert is not None:
131+
return ert
132+
# 2. label prefix
133+
for prefix, pert in _MAP_URL_LABEL_PREFIXES:
134+
if norm.startswith(prefix):
135+
return pert
136+
return ExternalReferenceType.OTHER

‎cyclonedx_py/_internal/utils/url_classifiers.py‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# This file is part of CycloneDX Python
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
# SPDX-License-Identifier: Apache-2.0
16+
# Copyright (c) OWASP Foundation. All Rights Reserved.
17+
18+
19+
"""
20+
Pure mapping data for URL -> ExternalReferenceType classification.
21+
22+
This module is DATA ONLY -- no logic. To extend classification, add rows here.
23+
Four match styles, applied by ``cdx.url_label_to_ert`` in this precedence order:
24+
25+
1. _MAP_KNOWN_URL_LABELS exact label (normalized: lowercased, non-[a-z] stripped)
26+
2. _MAP_URL_LABEL_PREFIXES label prefix (PyPI '*' semantics); first match wins
27+
3. _MAP_KNOWN_URL_HOST_SUFFIXES host == key OR host endswith '.'+key (domain + subdomains)
28+
4. _MAP_KNOWN_URL_HOST_PREFIXES host == key OR host startswith key+'.' (e.g. docs.*)
29+
30+
Label keys MUST already be normalized (lowercase, only [a-z]).
31+
Host keys MUST be lowercase.
32+
33+
see https://docs.pypi.org/project_metadata/#icons
34+
"""
35+
36+
from cyclonedx.model import ExternalReferenceType
37+
38+
# 1. exact label -> ERT
39+
_MAP_KNOWN_URL_LABELS: dict[str, ExternalReferenceType] = {
40+
# see https://peps.python.org/pep-0345/#project-url-multiple-use
41+
# see https://github.com/pypi/warehouse/issues/5947#issuecomment-699660629
42+
'bugtracker': ExternalReferenceType.ISSUE_TRACKER,
43+
'issuetracker': ExternalReferenceType.ISSUE_TRACKER,
44+
'issues': ExternalReferenceType.ISSUE_TRACKER,
45+
'bugreports': ExternalReferenceType.ISSUE_TRACKER,
46+
'tracker': ExternalReferenceType.ISSUE_TRACKER,
47+
'home': ExternalReferenceType.WEBSITE,
48+
'homepage': ExternalReferenceType.WEBSITE,
49+
'download': ExternalReferenceType.DISTRIBUTION,
50+
'documentation': ExternalReferenceType.DOCUMENTATION,
51+
'docs': ExternalReferenceType.DOCUMENTATION,
52+
'changelog': ExternalReferenceType.RELEASE_NOTES,
53+
'changes': ExternalReferenceType.RELEASE_NOTES,
54+
'releasenotes': ExternalReferenceType.RELEASE_NOTES,
55+
'news': ExternalReferenceType.RELEASE_NOTES,
56+
'whatsnew': ExternalReferenceType.RELEASE_NOTES,
57+
'history': ExternalReferenceType.RELEASE_NOTES,
58+
'repository': ExternalReferenceType.VCS,
59+
'source': ExternalReferenceType.VCS,
60+
'github': ExternalReferenceType.VCS,
61+
'chat': ExternalReferenceType.CHAT,
62+
}
63+
64+
# 2. label prefix -> ERT (ordered; first match wins). normalized prefixes.
65+
_MAP_URL_LABEL_PREFIXES: tuple[tuple[str, ExternalReferenceType], ...] = (
66+
('documentation', ExternalReferenceType.DOCUMENTATION),
67+
('docs', ExternalReferenceType.DOCUMENTATION),
68+
('bug', ExternalReferenceType.ISSUE_TRACKER),
69+
('issue', ExternalReferenceType.ISSUE_TRACKER),
70+
('tracker', ExternalReferenceType.ISSUE_TRACKER),
71+
('report', ExternalReferenceType.ISSUE_TRACKER),
72+
('funding', ExternalReferenceType.OTHER),
73+
('sponsor', ExternalReferenceType.OTHER),
74+
('donation', ExternalReferenceType.OTHER),
75+
('donate', ExternalReferenceType.OTHER),
76+
)

‎tests/unit/test_utils_cdx.py‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@
2323
from cyclonedx.model import ExternalReference, ExternalReferenceType
2424
from cyclonedx.model.component import Component, ComponentType
2525
from cyclonedx.model.license import License, LicenseAcknowledgement
26+
from ddt import data, ddt, unpack
2627

27-
from cyclonedx_py._internal.utils.cdx import make_bom
28+
from cyclonedx_py._internal.utils.cdx import make_bom, url_label_to_ert
2829
from tests import EXPECTED_TOOL_NAME, load_pyproject
2930

3031

@@ -79,3 +80,41 @@ def test_extrefs(self) -> None:
7980
c = self.__get_c_by_name(EXPECTED_TOOL_NAME)
8081
ers: tuple[ExternalReference, ...] = tuple(c.external_references)
8182
self.assertExtRefs(p, ers)
83+
84+
85+
@ddt
86+
class TestUrlLabelToErt(TestCase):
87+
88+
@data(
89+
# exact labels (existing behaviour preserved)
90+
('Homepage', ExternalReferenceType.WEBSITE),
91+
('Home', ExternalReferenceType.WEBSITE),
92+
('Download', ExternalReferenceType.DISTRIBUTION),
93+
('Changelog', ExternalReferenceType.RELEASE_NOTES),
94+
('Change log', ExternalReferenceType.RELEASE_NOTES),
95+
('Release notes', ExternalReferenceType.RELEASE_NOTES),
96+
("What's new", ExternalReferenceType.RELEASE_NOTES),
97+
('History', ExternalReferenceType.RELEASE_NOTES),
98+
('Repository', ExternalReferenceType.VCS),
99+
('Source', ExternalReferenceType.VCS),
100+
('Chat', ExternalReferenceType.CHAT),
101+
# prefix labels (PyPI '*' semantics)
102+
('Documentation', ExternalReferenceType.DOCUMENTATION),
103+
('Documentation for users', ExternalReferenceType.DOCUMENTATION),
104+
('Docs (latest)', ExternalReferenceType.DOCUMENTATION),
105+
('Bug Reports', ExternalReferenceType.ISSUE_TRACKER),
106+
('Issue Tracker', ExternalReferenceType.ISSUE_TRACKER),
107+
('Tracker', ExternalReferenceType.ISSUE_TRACKER),
108+
('Report a bug', ExternalReferenceType.ISSUE_TRACKER),
109+
('Funding', ExternalReferenceType.OTHER),
110+
('Sponsor this project', ExternalReferenceType.OTHER),
111+
('Donate', ExternalReferenceType.OTHER),
112+
# unknown -> OTHER
113+
('Some Random Label', ExternalReferenceType.OTHER),
114+
)
115+
@unpack
116+
def test_label_only(self, label: str, expected: ExternalReferenceType) -> None:
117+
self.assertIs(expected, url_label_to_ert(label))
118+
119+
def test_label_only_url_none_backcompat(self) -> None:
120+
self.assertIs(ExternalReferenceType.WEBSITE, url_label_to_ert('Homepage', None))

0 commit comments

Comments
 (0)