Skip to content

Commit 1cd5d02

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Get debian distro info from clues
We will now deduce and populate debian distro information, i.e. their namespace from clues present in version/maintainer info. Reference: aboutcode-org/scancode.io#899 Signed-off-by: Ayan Sinha Mahapatra <[email protected]>
1 parent f70bbb7 commit 1cd5d02

File tree

11 files changed

+823
-12818
lines changed

11 files changed

+823
-12818
lines changed

‎src/packagedcode/debian.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
305305
package.update(
306306
package_data=package_data,
307307
datafile_path=res.path,
308+
check_compatible=False,
308309
replace=False,
309310
include_version=False,
310311
include_qualifiers=False,
@@ -628,6 +629,16 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
628629
if keyword:
629630
keywords.append(keyword)
630631

632+
# Get distro/namespace information from clues in package data
633+
if not distro:
634+
for clue, namespace in version_clues_for_namespace.items():
635+
if clue in version:
636+
distro = namespace
637+
638+
for clue, namespace in maintainer_clues_for_namespace.items():
639+
if clue in maintainer:
640+
distro = namespace
641+
631642
source_packages = []
632643
source = debian_data.get('source')
633644
if source:
@@ -656,6 +667,18 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
656667
)
657668

658669

670+
version_clues_for_namespace = {
671+
'deb': 'debian',
672+
'ubuntu': 'ubuntu',
673+
}
674+
675+
676+
maintainer_clues_for_namespace = {
677+
'packages.debian.org': 'debian',
678+
'lists.ubuntu.com': 'ubuntu',
679+
}
680+
681+
659682
ignored_root_dirs = {
660683
'/.',
661684
'/bin',

‎src/packagedcode/models.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,10 @@
119119
"""
120120

121121
SCANCODE_DEBUG_PACKAGE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)
122+
SCANCODE_DEBUG_PACKAGE_ASSEMBLY = os.environ.get('SCANCODE_DEBUG_PACKAGE_ASSEMBLY', False)
122123

123124
TRACE = SCANCODE_DEBUG_PACKAGE
124-
TRACE_UPDATE = SCANCODE_DEBUG_PACKAGE
125+
TRACE_UPDATE = SCANCODE_DEBUG_PACKAGE_ASSEMBLY
125126

126127

127128
def logger_debug(*args):
@@ -1537,6 +1538,7 @@ def update(
15371538
self,
15381539
package_data,
15391540
datafile_path,
1541+
check_compatible=True,
15401542
replace=False,
15411543
include_version=True,
15421544
include_qualifiers=False,
@@ -1568,7 +1570,7 @@ def update(
15681570
if isinstance(package_data, dict):
15691571
package_data = PackageData.from_dict(package_data)
15701572

1571-
if not is_compatible(
1573+
if check_compatible and not is_compatible(
15721574
purl1=self,
15731575
purl2=package_data,
15741576
include_version=include_version,

‎tests/packagedcode/data/debian/basic-rootfs-expected.json

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"packages": [
33
{
44
"type": "deb",
5-
"namespace": null,
5+
"namespace": "ubuntu",
66
"name": "libncurses5",
77
"version": "6.1-1ubuntu1.18.04",
88
"qualifiers": {
@@ -134,7 +134,7 @@
134134
"extracted_license_statement": null,
135135
"notice_text": null,
136136
"source_packages": [
137-
"pkg:deb/ncurses"
137+
"pkg:deb/ubuntu/ncurses"
138138
],
139139
"extra_data": {
140140
"multi_arch": "same",
@@ -180,7 +180,7 @@
180180
"repository_homepage_url": null,
181181
"repository_download_url": null,
182182
"api_data_url": null,
183-
"package_uid": "pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
183+
"package_uid": "pkg:deb/ubuntu/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
184184
"datafile_paths": [
185185
"basic-rootfs.tar.gz/var/lib/dpkg/status",
186186
"basic-rootfs.tar.gz/usr/share/doc/libncurses5/copyright",
@@ -191,11 +191,11 @@
191191
"debian_copyright_in_package",
192192
"debian_installed_md5sums"
193193
],
194-
"purl": "pkg:deb/[email protected]?architecture=amd64"
194+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
195195
},
196196
{
197197
"type": "deb",
198-
"namespace": null,
198+
"namespace": "ubuntu",
199199
"name": "libndp0",
200200
"version": "1.4-2ubuntu0.16.04.1",
201201
"qualifiers": {
@@ -291,7 +291,7 @@
291291
"extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
292292
"notice_text": null,
293293
"source_packages": [
294-
"pkg:deb/libndp"
294+
"pkg:deb/ubuntu/libndp"
295295
],
296296
"extra_data": {
297297
"multi_arch": "same",
@@ -319,7 +319,7 @@
319319
"repository_homepage_url": null,
320320
"repository_download_url": null,
321321
"api_data_url": null,
322-
"package_uid": "pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
322+
"package_uid": "pkg:deb/ubuntu/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
323323
"datafile_paths": [
324324
"basic-rootfs.tar.gz/var/lib/dpkg/status",
325325
"basic-rootfs.tar.gz/usr/share/doc/libndp0/copyright",
@@ -330,7 +330,7 @@
330330
"debian_copyright_in_package",
331331
"debian_installed_md5sums"
332332
],
333-
"purl": "pkg:deb/[email protected]?architecture=amd64"
333+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
334334
}
335335
],
336336
"dependencies": [],
@@ -507,7 +507,7 @@
507507
}
508508
],
509509
"for_packages": [
510-
"pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
510+
"pkg:deb/ubuntu/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
511511
],
512512
"scan_errors": []
513513
},
@@ -619,7 +619,7 @@
619619
}
620620
],
621621
"for_packages": [
622-
"pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
622+
"pkg:deb/ubuntu/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
623623
],
624624
"scan_errors": []
625625
},
@@ -738,7 +738,7 @@
738738
}
739739
],
740740
"for_packages": [
741-
"pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
741+
"pkg:deb/ubuntu/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
742742
],
743743
"scan_errors": []
744744
},
@@ -820,7 +820,7 @@
820820
}
821821
],
822822
"for_packages": [
823-
"pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
823+
"pkg:deb/ubuntu/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
824824
],
825825
"scan_errors": []
826826
},
@@ -830,7 +830,7 @@
830830
"package_data": [
831831
{
832832
"type": "deb",
833-
"namespace": null,
833+
"namespace": "ubuntu",
834834
"name": "libncurses5",
835835
"version": "6.1-1ubuntu1.18.04",
836836
"qualifiers": {
@@ -873,7 +873,7 @@
873873
"extracted_license_statement": null,
874874
"notice_text": null,
875875
"source_packages": [
876-
"pkg:deb/ncurses"
876+
"pkg:deb/ubuntu/ncurses"
877877
],
878878
"file_references": [],
879879
"extra_data": {
@@ -884,11 +884,11 @@
884884
"repository_download_url": null,
885885
"api_data_url": null,
886886
"datasource_id": "debian_installed_status_db",
887-
"purl": "pkg:deb/[email protected]?architecture=amd64"
887+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
888888
},
889889
{
890890
"type": "deb",
891-
"namespace": null,
891+
"namespace": "ubuntu",
892892
"name": "libndp0",
893893
"version": "1.4-2ubuntu0.16.04.1",
894894
"qualifiers": {
@@ -931,7 +931,7 @@
931931
"extracted_license_statement": null,
932932
"notice_text": null,
933933
"source_packages": [
934-
"pkg:deb/libndp"
934+
"pkg:deb/ubuntu/libndp"
935935
],
936936
"file_references": [],
937937
"extra_data": {
@@ -942,7 +942,7 @@
942942
"repository_download_url": null,
943943
"api_data_url": null,
944944
"datasource_id": "debian_installed_status_db",
945-
"purl": "pkg:deb/[email protected]?architecture=amd64"
945+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
946946
}
947947
],
948948
"for_packages": [],

‎tests/packagedcode/data/debian/basic/status.expected

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[
22
{
33
"type": "deb",
4-
"namespace": null,
4+
"namespace": "ubuntu",
55
"name": "libncurses5",
66
"version": "6.1-1ubuntu1.18.04",
77
"qualifiers": {
@@ -44,7 +44,7 @@
4444
"extracted_license_statement": null,
4545
"notice_text": null,
4646
"source_packages": [
47-
"pkg:deb/ncurses"
47+
"pkg:deb/ubuntu/ncurses"
4848
],
4949
"file_references": [],
5050
"extra_data": {
@@ -55,11 +55,11 @@
5555
"repository_download_url": null,
5656
"api_data_url": null,
5757
"datasource_id": "debian_installed_status_db",
58-
"purl": "pkg:deb/[email protected]?architecture=amd64"
58+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
5959
},
6060
{
6161
"type": "deb",
62-
"namespace": null,
62+
"namespace": "ubuntu",
6363
"name": "libcom-err2",
6464
"version": "1.44.1-1ubuntu1.1",
6565
"qualifiers": {
@@ -102,7 +102,7 @@
102102
"extracted_license_statement": null,
103103
"notice_text": null,
104104
"source_packages": [
105-
"pkg:deb/e2fsprogs"
105+
"pkg:deb/ubuntu/e2fsprogs"
106106
],
107107
"file_references": [],
108108
"extra_data": {
@@ -113,11 +113,11 @@
113113
"repository_download_url": null,
114114
"api_data_url": null,
115115
"datasource_id": "debian_installed_status_db",
116-
"purl": "pkg:deb/[email protected]?architecture=amd64"
116+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
117117
},
118118
{
119119
"type": "deb",
120-
"namespace": null,
120+
"namespace": "ubuntu",
121121
"name": "libapt-pkg5.0",
122122
"version": "1.6.11",
123123
"qualifiers": {
@@ -160,7 +160,7 @@
160160
"extracted_license_statement": null,
161161
"notice_text": null,
162162
"source_packages": [
163-
"pkg:deb/apt"
163+
"pkg:deb/ubuntu/apt"
164164
],
165165
"file_references": [],
166166
"extra_data": {
@@ -171,11 +171,11 @@
171171
"repository_download_url": null,
172172
"api_data_url": null,
173173
"datasource_id": "debian_installed_status_db",
174-
"purl": "pkg:deb/[email protected]?architecture=amd64"
174+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
175175
},
176176
{
177177
"type": "deb",
178-
"namespace": null,
178+
"namespace": "ubuntu",
179179
"name": "libaudit1",
180180
"version": "1:2.8.2-1ubuntu1",
181181
"qualifiers": {
@@ -218,7 +218,7 @@
218218
"extracted_license_statement": null,
219219
"notice_text": null,
220220
"source_packages": [
221-
"pkg:deb/audit"
221+
"pkg:deb/ubuntu/audit"
222222
],
223223
"file_references": [],
224224
"extra_data": {
@@ -229,11 +229,11 @@
229229
"repository_download_url": null,
230230
"api_data_url": null,
231231
"datasource_id": "debian_installed_status_db",
232-
"purl": "pkg:deb/libaudit1@1:2.8.2-1ubuntu1?architecture=amd64"
232+
"purl": "pkg:deb/ubuntu/libaudit1@1:2.8.2-1ubuntu1?architecture=amd64"
233233
},
234234
{
235235
"type": "deb",
236-
"namespace": null,
236+
"namespace": "ubuntu",
237237
"name": "perl-base",
238238
"version": "5.26.1-6ubuntu0.3",
239239
"qualifiers": {
@@ -276,7 +276,7 @@
276276
"extracted_license_statement": null,
277277
"notice_text": null,
278278
"source_packages": [
279-
"pkg:deb/perl"
279+
"pkg:deb/ubuntu/perl"
280280
],
281281
"file_references": [],
282282
"extra_data": {},
@@ -285,11 +285,11 @@
285285
"repository_download_url": null,
286286
"api_data_url": null,
287287
"datasource_id": "debian_installed_status_db",
288-
"purl": "pkg:deb/[email protected]?architecture=amd64"
288+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
289289
},
290290
{
291291
"type": "deb",
292-
"namespace": null,
292+
"namespace": "ubuntu",
293293
"name": "libudev1",
294294
"version": "237-3ubuntu10.22",
295295
"qualifiers": {
@@ -332,7 +332,7 @@
332332
"extracted_license_statement": null,
333333
"notice_text": null,
334334
"source_packages": [
335-
"pkg:deb/systemd"
335+
"pkg:deb/ubuntu/systemd"
336336
],
337337
"file_references": [],
338338
"extra_data": {
@@ -343,6 +343,6 @@
343343
"repository_download_url": null,
344344
"api_data_url": null,
345345
"datasource_id": "debian_installed_status_db",
346-
"purl": "pkg:deb/[email protected]?architecture=amd64"
346+
"purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
347347
}
348348
]

0 commit comments

Comments
 (0)