Update assemble method for npm packages #2929

    * Properly assign Package resources to correct package
    * Update test results

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

src/packagedcode/npm.py

@@ -36,35 +36,106 @@
 # TODO: add support for "lockfileVersion": 2 for package-lock.json and lockfileVersion: 3
 
 
+def yield_npm_dependencies_from_package_data(package_data, datafile_path, package_uid):
+    """
+    Yield a Dependency for each dependency from ``package_data.dependencies``
+    """
+    dependent_packages = package_data.dependencies
+    if dependent_packages:
+        yield from models.Dependency.from_dependent_packages(
+            dependent_packages=dependent_packages,
+            datafile_path=datafile_path,
+            datasource_id=package_data.datasource_id,
+            package_uid=package_uid,
+        )
+
+
+def yield_npm_dependencies_from_package_resource(resource, package_uid=None):
+    """
+    Yield a Dependency for each dependency from each package from``resource.package_data``
+    """
+    for pkg_data in resource.package_data:
+        pkg_data = models.PackageData.from_dict(pkg_data)
+        yield_npm_dependencies_from_package_data(pkg_data, resource.location, package_uid)
+
+
 class BaseNpmHandler(models.DatafileHandler):
 
     @classmethod
     def assemble(cls, package_data, resource, codebase):
+        """
+        If ``resource``, or one of its siblings, is a package.json file, use it
+        to create and yield the package, the package dependencies, and the
+        package resources.
+
+        When reporting the resources of a package, we alk the codebase, skipping
+        the node_modules directory, assign resources to the package and yield
+        resources.
+
+        For each lock file, assign dependencies to package instances and yield dependencies.
+
+        If there is no package.json, we do not have a package instance. In this
+        case, we yield each of the dependencies in each lock file.
+        """
         datafile_name_patterns = (
-            'package.json',
             'package-lock.json',
             '.package-lock.json',
             'npm-shrinkwrap.json',
             'yarn.lock',
         )
 
-        if resource.has_parent():
-            dir_resource=resource.parent(codebase)
-        else:
-            dir_resource=resource
-
-        for assembled in cls.assemble_from_many_datafiles(
-            datafile_name_patterns=datafile_name_patterns,
-            directory=dir_resource,
-            codebase=codebase,
-        ):
-            if isinstance(assembled, models.Package):
-                cls.assign_package_to_resources(
-                    package=assembled,
-                    resource=resource,
-                    codebase=codebase,
+        package_resource = None
+        if resource.name == 'package.json':
+            package_resource = resource
+        elif resource.name in datafile_name_patterns:
+            if resource.has_parent():
+                siblings = resource.siblings(codebase)
+                package_resource = [r for r in siblings if r.name == 'package.json']
+                if package_resource:
+                    package_resource = package_resource[0]
+
+        if package_resource:
+            # do we have enough to create a package?
+            if package_data.purl:
+                package = models.Package.from_package_data(
+                    package_data=package_data,
+                    datafile_path=package_resource.path,
                 )
-            yield assembled
+                package_uid = package.package_uid
+
+                if not package.license_expression:
+                    package.license_expression = compute_normalized_license(package.declared_license)
+
+                root = resource.parent(codebase)
+                if root:
+                    for npm_res in cls.walk_npm(resource=root, codebase=codebase):
+                        if package_uid not in npm_res.for_packages:
+                            npm_res.for_packages.append(package_uid)
+                            npm_res.save(codebase)
+                        yield npm_res
+
+                yield package
+            else:
+                # we have no package, so deps are not for a specific package uid
+                package_uid = None
+
+            # in all cases yield possible dependencies
+            yield_npm_dependencies_from_package_data(package_data, package_resource.path, package_uid)
+
+            # we yield this as we do not want this further processed
+            yield package_resource
+
+            for sibling in package_resource.siblings(codebase):
+                if sibling.name in datafile_name_patterns:
+                    yield_npm_dependencies_from_package_resource(sibling, package_uid)
+
+                    if package_uid not in sibling.for_packages:
+                        sibling.for_packages.append(package_uid)
+                        sibling.save(codebase)
+                    yield sibling
+        else:
+            # we do not have a package.json
+            yield_npm_dependencies_from_package_resource(resource)
 
     @classmethod
     def walk_npm(cls, resource, codebase, depth=0):
@@ -87,16 +158,6 @@ def walk_npm(cls, resource, codebase, depth=0):
                 for subchild in cls.walk_skip(child, codebase, depth=depth):
                     yield subchild
 
-    # TODO: this MUST BE USED
-    @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
-        """
-        Yield the Resources of an npm Package, ignoring nested mode_modules.
-        """
-        root = resource.parent(codebase)
-        if root:
-            yield from cls.walk_npm(resource=root, codebase=codebase)
-
 
 def get_urls(namespace, name, version):
     return dict(

tests/licensedcode/data/plugin_licenses_reference/scan.expected.json

@@ -13,7 +13,15 @@
       "output_format_version": "2.0.0",
       "message": null,
       "errors": [],
+      "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 2
       }
@@ -314,7 +322,9 @@
       ],
       "percentage_of_license_text": 100.0,
       "package_data": [],
-      "for_packages": [],
+      "for_packages": [
+        "pkg:npm/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758"
+      ],
       "scan_errors": []
     },
     {

tests/packagedcode/data/about/aboutfiles.expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 3
       }

tests/packagedcode/data/bower/scan-expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 1
       }

tests/packagedcode/data/build/bazel/end2end-expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 6
       }

tests/packagedcode/data/build/buck/end2end-expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 7
       }

tests/packagedcode/data/build_gradle/end2end-expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 1
       }

tests/packagedcode/data/cargo/scan.expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 3
       }

tests/packagedcode/data/chef/package.scan.expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 2
       }

tests/packagedcode/data/cocoapods/assemble/many-podspecs-expected.json

@@ -13,6 +13,13 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-107-generic-x86_64-with-Ubuntu-18.04-bionic",
+          "platform_version": "#121~18.04.1-Ubuntu SMP Thu Mar 24 17:21:33 UTC 2022",
+          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
+        },
         "spdx_license_list_version": "3.16",
         "files_count": 9
       }