Use SCTK package_only in inspect_packages pipeline (#1118)

* Update inspect_packages pipeline

* Split package/dependencies creation in a seperate step
* Only create packages/dependencies from Assemblable PackageData

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

* Support fast package-only scan in inspect_packages

Support the new only_packages attributes in scancode
get_package_data API, to only scan for package data and
skip license and copyright detection.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

* Add CHANGELOG entry

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

* Make resolve_dependencies integration test faster

Reference: #1087
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

---------

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -23,6 +23,11 @@ v34.1.0 (unreleased)
   https://github.com/nexB/scancode.io/issues/1121
   https://github.com/nexB/scancode.io/issues/1122
 
+- Use the `package_only` option in scancode `get_package_data` API in
+  `inspect_packages` pipeline, to skip license and copyright detection in
+  extracted license and copyright statements found in package metadata.
+  https://github.com/nexB/scancode-toolkit/pull/3689
+
 - Rename the ``match_to_purldb`` pipeline to ``match_to_matchcode``, and add
   MatchCode.io API settings to ScanCode.io settings.
 

scanpipe/pipelines/inspect_packages.py

@@ -49,6 +49,7 @@ def steps(cls):
             cls.flag_empty_files,
             cls.flag_ignored_resources,
             cls.scan_for_application_packages,
+            cls.create_packages_and_dependencies,
         )
 
     def scan_for_application_packages(self):
@@ -60,5 +61,11 @@ def scan_for_application_packages(self):
         # only detect package_data in resources and create
         # Package/Dependency instances directly instead of assembling
         # the packages and assigning files to them
-        scancode.scan_for_application_packages(self.project, assemble=False)
+        scancode.scan_for_application_packages(
+            project=self.project,
+            assemble=False,
+            package_only=True,
+        )
+
+    def create_packages_and_dependencies(self):
         scancode.process_package_data(self.project)

scanpipe/pipes/scancode.py

@@ -219,14 +219,18 @@ def scan_file(location, with_threading=True, min_license_score=0, **kwargs):
     return _scan_resource(location, scanners, with_threading=with_threading)
 
 
-def scan_for_package_data(location, with_threading=True, **kwargs):
+def scan_for_package_data(location, with_threading=True, package_only=False, **kwargs):
     """
     Run a package scan on provided `location` using the scancode-toolkit direct API.
 
     Return a dict of scan `results` and a list of `errors`.
     """
+    scancode_get_packages = partial(
+        scancode_api.get_package_data,
+        package_only=package_only,
+    )
     scanners = [
-        Scanner("package_data", scancode_api.get_package_data),
+        Scanner("package_data", scancode_get_packages),
     ]
     return _scan_resource(location, scanners, with_threading=with_threading)
 
@@ -343,7 +347,9 @@ def scan_for_files(project, resource_qs=None, progress_logger=None):
     )
 
 
-def scan_for_application_packages(project, assemble=True, progress_logger=None):
+def scan_for_application_packages(
+    project, assemble=True, package_only=False, progress_logger=None
+):
     """
     Run a package scan on resources without a status for a `project`,
     and add them in their respective `package_data` attribute.
@@ -359,13 +365,18 @@ def scan_for_application_packages(project, assemble=True, progress_logger=None):
     """
     resource_qs = project.codebaseresources.no_status()
 
+    scan_func_kwargs = {
+        "package_only": package_only,
+    }
+
     # Collect detected Package data and save it to the CodebaseResource it was
     # detected from.
     scan_resources(
         resource_qs=resource_qs,
         scan_func=scan_for_package_data,
         save_func=save_scan_package_results,
         progress_logger=progress_logger,
+        scan_func_kwargs=scan_func_kwargs,
     )
 
     # Iterate through CodebaseResources with Package data and handle them using
@@ -460,15 +471,19 @@ def process_package_data(project):
         logger.info(f"  Processing: {resource.path}")
         for package_mapping in resource.package_data:
             pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping)
+            if not pd.can_assemble:
+                continue
+
             logger.info(f"  Package data: {pd.purl}")
 
             package_data = pd.to_dict()
             dependencies = package_data.pop("dependencies")
-            pipes.update_or_create_package(project, package_data)
-
             for dep in dependencies:
                 pipes.update_or_create_dependency(project, dep)
 
+            if pd.purl:
+                pipes.update_or_create_package(project, package_data)
+
 
 def get_packages_with_purl_from_resources(project):
     """

scanpipe/tests/data/manifests/requirements.txt

@@ -0,0 +1 @@
+click==8.1.3

scanpipe/tests/test_pipelines.py

@@ -922,17 +922,15 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self):
         pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
-        input_location = (
-            self.data_location / "manifests" / "python-inspector-0.10.0.zip"
-        )
+        input_location = self.data_location / "manifests" / "requirements.txt"
         project1.copy_input_from(input_location)
 
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
         exitcode, out = pipeline.execute()
         self.assertEqual(0, exitcode, msg=out)
-        self.assertEqual(26, project1.discoveredpackages.count())
+        self.assertEqual(1, project1.discoveredpackages.count())
 
     @mock.patch("scanpipe.pipes.resolve.resolve_dependencies")
     def test_scanpipe_resolve_dependencies_pipeline_pypi_integration(