Assign debian namespace based on other packages

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/packagedcode/debian.py

@@ -9,6 +9,7 @@
 
 import os
 import logging
+from collections import Counter
 from pathlib import Path
 
 from commoncode import fileutils
@@ -157,15 +158,19 @@ class DebianControlFileInSourceHandler(models.DatafileHandler):
 
     @classmethod
     def parse(cls, location):
-        # TODO: we cannot know the distro from the name only
         # NOTE: a control file in a source repo or debina.tar tarball can contain more than one package
+        debian_packages = []
         for debian_data in get_paragraphs_data_from_file(location=location):
-            yield build_package_data(
-                debian_data,
-                datasource_id=cls.datasource_id,
-                package_type=cls.default_package_type,
+            debian_packages.append(
+                build_package_data(
+                    debian_data=debian_data,
+                    datasource_id=cls.datasource_id,
+                    package_type=cls.default_package_type,
+                )
             )
 
+        yield from populate_debian_namespace(debian_packages)
+
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         # two levels up
@@ -214,13 +219,18 @@ class DebianInstalledStatusDatabaseHandler(models.DatafileHandler):
     def parse(cls, location):
         # note that we do not know yet the distro at this stage
         # we could get it... but we get that later during assemble()
-        for debian_data in get_paragraphs_data_from_file(location):
-            yield build_package_data(
-                debian_data,
-                datasource_id=cls.datasource_id,
-                package_type=cls.default_package_type,
+        debian_packages = []
+        for debian_data in get_paragraphs_data_from_file(location=location):
+            debian_packages.append(
+                build_package_data(
+                    debian_data=debian_data,
+                    datasource_id=cls.datasource_id,
+                    package_type=cls.default_package_type,
+                )
             )
 
+        yield from populate_debian_namespace(debian_packages)
+
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):
         # get the root resource of the rootfs
@@ -380,14 +390,18 @@ def parse(cls, location):
         rootfs installation. distroless is derived from Debian but each package
         has its own status file.
         """
-        for debian_data in get_paragraphs_data_from_file(location):
-            yield build_package_data(
-                debian_data,
-                datasource_id=cls.datasource_id,
-                package_type=cls.default_package_type,
-                distro='distroless',
+        debian_packages = []
+        for debian_data in get_paragraphs_data_from_file(location=location):
+            debian_packages.append(
+                build_package_data(
+                    debian_data=debian_data,
+                    datasource_id=cls.datasource_id,
+                    package_type=cls.default_package_type,
+                )
             )
 
+        yield from populate_debian_namespace(debian_packages)
+
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):
         # get the root resource of the rootfs
@@ -667,6 +681,29 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
     )
 
 
+def populate_debian_namespace(packages):
+    """
+    For an iterable of debian `packages`, populate the
+    most frequently occuring namespace, or the default
+    namespace 'debian' in packages without namespace.
+    """
+    if not packages:
+        return
+
+    namespaces_with_count = Counter([
+        package.namespace
+        for package in packages
+    ])
+    distro = max(namespaces_with_count, key=namespaces_with_count.get)
+    if not distro:
+        distro = 'debian'
+
+    for package in packages:
+        if not package.namespace:
+            package.namespace = distro
+        yield package
+
+
 version_clues_for_namespace = {
     'deb': 'debian',
     'ubuntu': 'ubuntu',

tests/packagedcode/data/debian/ubuntu-var-lib-dpkg/expected.json

@@ -563,6 +563,66 @@
       ],
       "purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
     },
+    {
+      "type": "deb",
+      "namespace": "ubuntu",
+      "name": "e2fsprogs",
+      "version": "1.46.2-2",
+      "qualifiers": {
+        "architecture": "amd64"
+      },
+      "subpath": null,
+      "primary_language": null,
+      "description": "ext2/ext3/ext4 file system utilities\n The ext2, ext3 and ext4 file systems are successors of the original ext\n (\"extended\") file system. They are the main file system types used for\n hard disks on Debian and other Linux systems.\n .\n This package contains programs for creating, checking, and maintaining\n ext2/3/4-based file systems.  It also includes the \"badblocks\" program,\n which can be used to scan for bad blocks on a disk or other storage device.",
+      "release_date": null,
+      "parties": [
+        {
+          "type": null,
+          "role": "maintainer",
+          "name": "Theodore Y. Ts'o <[email protected]>",
+          "email": null,
+          "url": null
+        }
+      ],
+      "keywords": [
+        "admin"
+      ],
+      "homepage_url": "http://e2fsprogs.sourceforge.net",
+      "download_url": null,
+      "size": null,
+      "sha1": null,
+      "md5": null,
+      "sha256": null,
+      "sha512": null,
+      "bug_tracking_url": null,
+      "code_view_url": null,
+      "vcs_url": null,
+      "copyright": null,
+      "holder": null,
+      "declared_license_expression": null,
+      "declared_license_expression_spdx": null,
+      "license_detections": [],
+      "other_license_expression": null,
+      "other_license_expression_spdx": null,
+      "other_license_detections": [],
+      "extracted_license_statement": null,
+      "notice_text": null,
+      "source_packages": [],
+      "extra_data": {
+        "multi_arch": "foreign"
+      },
+      "repository_homepage_url": null,
+      "repository_download_url": null,
+      "api_data_url": null,
+      "package_uid": "pkg:deb/ubuntu/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_paths": [
+        "ubuntu-var-lib-dpkg/var/lib/dpkg/status"
+      ],
+      "datasource_ids": [
+        "debian_installed_status_db"
+      ],
+      "purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
+    },
     {
       "type": "deb",
       "namespace": "ubuntu",
@@ -2699,6 +2759,62 @@
           "datasource_id": "debian_installed_status_db",
           "purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
         },
+        {
+          "type": "deb",
+          "namespace": "ubuntu",
+          "name": "e2fsprogs",
+          "version": "1.46.2-2",
+          "qualifiers": {
+            "architecture": "amd64"
+          },
+          "subpath": null,
+          "primary_language": null,
+          "description": "ext2/ext3/ext4 file system utilities\n The ext2, ext3 and ext4 file systems are successors of the original ext\n (\"extended\") file system. They are the main file system types used for\n hard disks on Debian and other Linux systems.\n .\n This package contains programs for creating, checking, and maintaining\n ext2/3/4-based file systems.  It also includes the \"badblocks\" program,\n which can be used to scan for bad blocks on a disk or other storage device.",
+          "release_date": null,
+          "parties": [
+            {
+              "type": null,
+              "role": "maintainer",
+              "name": "Theodore Y. Ts'o <[email protected]>",
+              "email": null,
+              "url": null
+            }
+          ],
+          "keywords": [
+            "admin"
+          ],
+          "homepage_url": "http://e2fsprogs.sourceforge.net",
+          "download_url": null,
+          "size": null,
+          "sha1": null,
+          "md5": null,
+          "sha256": null,
+          "sha512": null,
+          "bug_tracking_url": null,
+          "code_view_url": null,
+          "vcs_url": null,
+          "copyright": null,
+          "holder": null,
+          "declared_license_expression": null,
+          "declared_license_expression_spdx": null,
+          "license_detections": [],
+          "other_license_expression": null,
+          "other_license_expression_spdx": null,
+          "other_license_detections": [],
+          "extracted_license_statement": null,
+          "notice_text": null,
+          "source_packages": [],
+          "file_references": [],
+          "extra_data": {
+            "multi_arch": "foreign"
+          },
+          "dependencies": [],
+          "repository_homepage_url": null,
+          "repository_download_url": null,
+          "api_data_url": null,
+          "datasource_id": "debian_installed_status_db",
+          "purl": "pkg:deb/ubuntu/[email protected]?architecture=amd64"
+        },
         {
           "type": "deb",
           "namespace": "ubuntu",

tests/packagedcode/data/debian/ubuntu-var-lib-dpkg/var/lib/dpkg/status

@@ -87,6 +87,33 @@ Description: GNU Bourne Again SHell
 Homepage: http://tiswww.case.edu/php/chet/bash/bashtop.html
 Original-Maintainer: Matthias Klose <[email protected]>
 
+Package: e2fsprogs
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 1511
+Maintainer: Theodore Y. Ts'o <[email protected]>
+Architecture: amd64
+Multi-Arch: foreign
+Version: 1.46.2-2
+Depends: logsave
+Pre-Depends: libblkid1 (>= 2.36), libc6 (>= 2.14), libcom-err2 (>= 1.43.9), libext2fs2 (= 1.46.2-2), libss2 (>= 1.38), libuuid1 (>= 2.16)
+Recommends: e2fsprogs-l10n
+Suggests: gpart, parted, fuse2fs, e2fsck-static
+Conffiles:
+ /etc/cron.d/e2scrub_all bc533e09f3b3d96bfe1633ad57eb7026
+ /etc/e2scrub.conf df38534cc670c70a91cf9b035845d244
+ /etc/mke2fs.conf 72b349d890a9b5cca06c7804cd0c8d1d
+Description: ext2/ext3/ext4 file system utilities
+ The ext2, ext3 and ext4 file systems are successors of the original ext
+ ("extended") file system. They are the main file system types used for
+ hard disks on Debian and other Linux systems.
+ .
+ This package contains programs for creating, checking, and maintaining
+ ext2/3/4-based file systems.  It also includes the "badblocks" program,
+ which can be used to scan for bad blocks on a disk or other storage device.
+Homepage: http://e2fsprogs.sourceforge.net
+
 Package: tar
 Essential: yes
 Status: install ok installed