Support --package-only for all package types

Reference: #3464
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/packagedcode/about.py

@@ -47,7 +47,7 @@ class AboutFileHandler(models.DatafileHandler):
     documentation_url = 'https://aboutcode-toolkit.readthedocs.io/en/latest/specification.html'
 
     @classmethod
-    def parse(cls, location):
+    def parse(cls, location, package_only=False):
         """
         Yield one or more Package manifest objects given a file ``location`` pointing to a
         package archive, manifest or similar.
@@ -90,7 +90,7 @@ def parse(cls, location):
             file_references.append(models.FileReference(path=about_resource))
 
         # FIXME: we should put the unprocessed attributes in extra data
-        yield models.PackageData(
+        package_data = dict(
             datasource_id=cls.datasource_id,
             type=package_type,
             namespace=package_ns,
@@ -103,6 +103,7 @@ def parse(cls, location):
             download_url=download_url,
             file_references=file_references,
         )
+        yield models.PackageData.from_data(package_data, package_only)
 
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):

src/packagedcode/alpine.py

@@ -63,11 +63,12 @@ class AlpineInstalledDatabaseHandler(models.DatafileHandler):
     description = 'Alpine Linux installed package database'
 
     @classmethod
-    def parse(cls, location):
+    def parse(cls, location, package_only=False):
         yield from parse_alpine_installed_db(
             location=location,
             datasource_id=cls.datasource_id,
             package_type=cls.default_package_type,
+            package_only=package_only,
         )
 
     @classmethod
@@ -134,9 +135,14 @@ class AlpineApkbuildHandler(models.DatafileHandler):
     documentation_url = 'https://wiki.alpinelinux.org/wiki/APKBUILD_Reference'
 
     @classmethod
-    def parse(cls, location):
-        package_data = parse_apkbuild(location, strict=True)
-        cls.populate_license_fields(package_data)
+    def parse(cls, location, package_only=False):
+        package_data = parse_apkbuild(
+            location=location,
+            strict=True,
+            package_only=package_only
+        )
+        if not package_only:
+            cls.populate_license_fields(package_data)
         if package_data:
             yield package_data
 
@@ -165,7 +171,7 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
         )
 
 
-def parse_alpine_installed_db(location, datasource_id, package_type):
+def parse_alpine_installed_db(location, datasource_id, package_type, package_only=False):
     """
     Yield PackageData objects from an installed database file at `location`
     or None. Typically found at '/lib/apk/db/installed' in an Alpine
@@ -179,6 +185,7 @@ def parse_alpine_installed_db(location, datasource_id, package_type):
             package_fields=package_fields,
             datasource_id=datasource_id,
             package_type=package_type,
+            package_only=package_only,
         )
 
 
@@ -241,7 +248,7 @@ def get_alpine_installed_db_fields(location):
 ])
 
 
-def parse_apkbuild(location, strict=False):
+def parse_apkbuild(location, strict=False, package_only=False):
     """
     Return a PackageData object from an APKBUILD file at ``location`` or None.
 
@@ -256,6 +263,7 @@ def parse_apkbuild(location, strict=False):
         datasource_id=AlpineApkbuildHandler.datasource_id,
         package_type=AlpineApkbuildHandler.default_package_type,
         strict=strict,
+        package_only=package_only,
     )
 
 
@@ -732,7 +740,7 @@ def fix_apkbuild(text):
     return text
 
 
-def parse_apkbuild_text(text, datasource_id, package_type, strict=False):
+def parse_apkbuild_text(text, datasource_id, package_type, strict=False, package_only=False):
     """
     Return a PackageData object from an APKBUILD text context or None. Only
     consider variables with a name listed in the ``names`` set.
@@ -761,7 +769,8 @@ def parse_apkbuild_text(text, datasource_id, package_type, strict=False):
     package = build_package_data(
         variables,
         datasource_id=datasource_id,
-        package_type=package_type
+        package_type=package_type,
+        package_only=package_only,
     )
 
     if package and unresolved:
@@ -800,7 +809,7 @@ def parse_pkginfo(location):
     raise NotImplementedError
 
 
-def build_package_data(package_fields, datasource_id, package_type):
+def build_package_data(package_fields, datasource_id, package_type, package_only=False):
     """
     Return a PackageData object from a ``package_fields`` iterable of (name,
     value) tuples.
@@ -850,7 +859,16 @@ def build_package_data(package_fields, datasource_id, package_type):
 
             converted_fields.update(converted)
 
-    return models.PackageData.from_dict(converted_fields)
+    fields_not_required = ["current_file", "current_dir"]
+    for field in fields_not_required:
+        value = converted_fields.get(field)
+        if value:
+            converted_fields.pop(field)
+
+    return models.PackageData.from_data(
+        package_data=converted_fields,
+        package_only=package_only,
+    )
 
 #####################################
 # Note: all handlers MUST accept **kwargs as they also receive the current data

src/packagedcode/build.py

@@ -55,7 +55,7 @@ class AutotoolsConfigureHandler(models.DatafileHandler):
     documentation_url = 'https://www.gnu.org/software/automake/'
 
     @classmethod
-    def parse(cls, location):
+    def parse(cls, location, package_only=False):
         # we use the parent directory as a package name
         name = fileutils.file_name(fileutils.parent_directory(location))
         # we could use checksums as version in the future
@@ -67,12 +67,13 @@ def parse(cls, location):
         # there are dependencies we could use
         # dependencies = []
 
-        yield models.PackageData(
+        package_data = dict(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
             name=name,
             version=version,
         )
+        yield models.PackageData.from_data(package_data, package_only)
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase, package_adder):
@@ -112,6 +113,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
             package = models.Package.from_package_data(
                 package_data=package_data,
                 datafile_path=resource.path,
+                package_only=True,
             )
 
             if TRACE:
@@ -143,8 +145,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
         yield resource
 
     @classmethod
-    def parse(cls, location):
-
+    def parse(cls, location, package_only=False):
         # Thanks to Starlark being a Python dialect, we can use `ast` to parse it
         with open(location, 'rb') as f:
             tree = ast.parse(f.read())
@@ -196,23 +197,28 @@ def parse(cls, location):
                     if TRACE:
                         logger_debug(f"build: parse: license_files: {license_files}")
 
-                    package_data = models.PackageData(
+                    package_data = dict(
                         datasource_id=cls.datasource_id,
                         type=cls.default_package_type,
                         name=name,
+                        extracted_license_statement=license_files,
+                    )
+                    # `package_only` is True as we do the license detection
+                    # on assembly
+                    yield models.PackageData.from_data(
+                        package_data=package_data,
+                        package_only=True,
                     )
-
-                    package_data.extracted_license_statement = license_files
-                    yield package_data
 
         else:
             # If we don't find anything in the pkgdata file, we yield a Package
             # with the parent directory as the name
-            yield models.PackageData(
+            package_data = dict(
                 datasource_id=cls.datasource_id,
                 type=cls.default_package_type,
                 name=fileutils.file_name(fileutils.parent_directory(location))
             )
+            yield models.PackageData.from_data(package_data, package_only)
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name=None):
@@ -334,7 +340,7 @@ class BuckMetadataBzlHandler(BaseStarlarkManifestHandler):
     documentation_url = 'https://buck.build/'
 
     @classmethod
-    def parse(cls, location):
+    def parse(cls, location, package_only=True):
 
         with open(location, 'rb') as f:
             tree = ast.parse(f.read())
@@ -386,7 +392,7 @@ def parse(cls, location):
         ):
             # TODO: Create function that determines package type from download URL,
             # then create a package of that package type from the metadata info
-            yield models.PackageData(
+            package_data = dict(
                 datasource_id=cls.datasource_id,
                 type=metadata_fields.get('upstream_type', cls.default_package_type),
                 name=metadata_fields.get('name'),
@@ -396,6 +402,7 @@ def parse(cls, location):
                 homepage_url=metadata_fields.get('upstream_address', ''),
                 # TODO: Store 'upstream_hash` somewhere
             )
+            yield models.PackageData.from_data(package_data, package_only=True)
 
         if (
             'package_type'
@@ -409,7 +416,7 @@ def parse(cls, location):
             and 'vcs_commit_hash'
             in metadata_fields
         ):
-            yield models.PackageData(
+            package_data = dict(
                 datasource_id=cls.datasource_id,
                 type=metadata_fields.get('package_type', cls.default_package_type),
                 name=metadata_fields.get('name'),
@@ -422,6 +429,7 @@ def parse(cls, location):
                 sha1=metadata_fields.get('download_archive_sha1', ''),
                 extra_data=dict(vcs_commit_hash=metadata_fields.get('vcs_commit_hash', ''))
             )
+            yield models.PackageData.from_data(package_data, package_only=True)
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase, package_adder):

src/packagedcode/build_gradle.py

@@ -59,9 +59,9 @@ class BuildGradleHandler(models.DatafileHandler):
     description = 'Gradle build script'
 
     @classmethod
-    def parse(cls, location):
+    def parse(cls, location, package_only=False):
         dependencies = get_dependencies(location)
-        return build_package(cls, dependencies)
+        return build_package(cls, dependencies, package_only)
 
     # TODO: handle complex cases of nested builds with many packages
     @classmethod
@@ -328,7 +328,7 @@ def get_dependencies(build_gradle_location):
     return list(get_dependencies_from_parse_tree(parse_tree))
 
 
-def build_package(cls, dependencies):
+def build_package(cls, dependencies, package_only=False):
     """
     Yield PackageData from a ``dependencies`` list of mappings.
     """
@@ -364,10 +364,11 @@ def build_package(cls, dependencies):
             )
         )
 
-    yield models.PackageData(
+    package_data = dict(
         datasource_id=cls.datasource_id,
         type=cls.default_package_type,
         primary_language=BuildGradleHandler.default_primary_language,
         dependencies=package_dependencies,
     )
+    yield models.PackageData.from_data(package_data, package_only)
 

src/packagedcode/cargo.py

@@ -29,7 +29,7 @@ class CargoTomlHandler(models.DatafileHandler):
     documentation_url = 'https://doc.rust-lang.org/cargo/reference/manifest.html'
 
     @classmethod
-    def parse(cls, location):
+    def parse(cls, location, package_only=False):
         package_data = toml.load(location, _dict=dict)
 
         core_package_data = package_data.get('package', {})
@@ -67,7 +67,7 @@ def parse(cls, location):
         repository_download_url = name and version and f'https://crates.io/api/v1/crates/{name}/{version}/download'
         api_data_url = name and f'https://crates.io/api/v1/crates/{name}'
 
-        yield models.PackageData(
+        package_data = dict(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
             name=name,
@@ -83,6 +83,7 @@ def parse(cls, location):
             api_data_url=api_data_url,
             dependencies=dependencies,
         )
+        yield models.PackageData.from_data(package_data, package_only)
 
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):
@@ -116,7 +117,7 @@ class CargoLockHandler(models.DatafileHandler):
     # ]
 
     @classmethod
-    def parse(cls, location):
+    def parse(cls, location, package_only=False):
         cargo_lock = toml.load(location, _dict=dict)
         dependencies = []
         package = cargo_lock.get('package', [])
@@ -137,12 +138,13 @@ def parse(cls, location):
                 )
             )
 
-        yield models.PackageData(
+        package_data = dict(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
             primary_language=cls.default_primary_language,
             dependencies=dependencies,
         )
+        yield models.PackageData.from_data(package_data, package_only)
 
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):