Do not create for_packages list with None

If the package_uid is None we should not assign this as the package
of a resource when doing package assemble() processing. We should
insyead skip this. This was discovered in npm, but could apply to other
package types.

Reported-by: Thomas Druez <[email protected]>
Reference: aboutcode-org/scancode.io#448
Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/packagedcode/build.py

@@ -177,6 +177,8 @@ def parse(cls, location):
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase, skip_name=None):
         package_uid = package.package_uid
+        if not package_uid:
+            return
         parent = resource.parent(codebase)
         for res in walk_build(resource=parent, codebase=codebase, skip_name=skip_name):
             res.for_packages.append(package_uid)

src/packagedcode/debian.py

@@ -397,20 +397,21 @@ def assemble(cls, package_data, resource, codebase):
         assemblable_paths = (
             f'usr/share/doc/{package_name}/copyright',
         )
-        for res in root_resource.walk(codebase):
-            if not res.path.endswith(assemblable_paths):
-                continue
-
-            for pkgdt in res.package_data:
-                package.update(
-                    package_data=pkgdt,
-                    datafile_path=res.path,
-                )
-
-            res.for_packages.append(package_uid)
-            res.save(codebase)
-
-            yield res
+        if package_uid:
+            for res in root_resource.walk(codebase):
+                if not res.path.endswith(assemblable_paths):
+                    continue
+    
+                for pkgdt in res.package_data:
+                    package.update(
+                        package_data=pkgdt,
+                        datafile_path=res.path,
+                    )
+    
+                res.for_packages.append(package_uid)
+                res.save(codebase)
+    
+                yield res
 
         yield package
 

src/packagedcode/models.py

@@ -970,7 +970,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
         # NOTE: we do not attach files to the Package level. Instead we
         # update `for_packages` of a codebase resource.
         package_uid = package.package_uid
-        if resource:
+        if resource and package_uid:
             resource.for_packages.append(package_uid)
             resource.save(codebase)
             for res in resource.walk(codebase):
@@ -1030,8 +1030,9 @@ def assemble_from_many(cls, pkgdata_resources, codebase,):
                         datafile_path=resource.path,
                     )
                     package_uid = package.package_uid
-                    resource.for_packages.append(package_uid)
-                    resource.save(codebase)
+                    if package_uid:
+                        resource.for_packages.append(package_uid)
+                        resource.save(codebase)
             else:
                 # FIXME: What is the package_data is NOT for the same package as package?
                 # FIXME: What if the update did not do anything? (it does return True or False)
@@ -1040,8 +1041,9 @@ def assemble_from_many(cls, pkgdata_resources, codebase,):
                     package_data=package_data,
                     datafile_path=resource.path,
                 )
-                resource.for_packages.append(package_uid)
-                resource.save(codebase)
+                if package_uid:
+                    resource.for_packages.append(package_uid)
+                    resource.save(codebase)
 
             # in all cases yield possible dependencies
             dependent_packages = package_data.dependencies
@@ -1057,9 +1059,10 @@ def assemble_from_many(cls, pkgdata_resources, codebase,):
             yield resource
 
         # the whole parent subtree of the base_resource is for this package
-        for res in base_resource.walk(codebase):
-            res.for_packages.append(package_uid)
-            res.save(codebase)
+        if package_uid:
+            for res in base_resource.walk(codebase):
+                res.for_packages.append(package_uid)
+                res.save(codebase)
 
         if package:
             if not package.license_expression:
@@ -1336,9 +1339,10 @@ def get_packages_files(self, codebase):
         Yield all the Resource of this package found in codebase.
         """
         package_uid = self.package_uid
-        for resource in codebase.walk():
-            if package_uid in resource.for_packages:
-                yield resource
+        if package_uid:
+            for resource in codebase.walk():
+                if package_uid in resource.for_packages:
+                    yield resource
 
 
 @attr.attributes(slots=True)

src/packagedcode/npm.py

@@ -95,12 +95,12 @@ def assemble(cls, package_data, resource, codebase):
                 root = package_resource.parent(codebase)
                 if root:
                     for npm_res in cls.walk_npm(resource=root, codebase=codebase):
-                        if package_uid not in npm_res.for_packages:
+                        if package_uid and package_uid not in npm_res.for_packages:
                             npm_res.for_packages.append(package_uid)
                             npm_res.save(codebase)
                         yield npm_res
                 elif codebase.has_single_resource:
-                    if package_uid not in package_resource.for_packages:
+                    if package_uid and package_uid not in package_resource.for_packages:
                         package_resource.for_packages.append(package_uid)
                         package_resource.save(codebase)
                 yield package_resource
@@ -119,7 +119,7 @@ def assemble(cls, package_data, resource, codebase):
                 if lock_file.name in lockfile_names:
                     yield from yield_dependencies_from_package_resource(lock_file, package_uid)
 
-                    if package_uid not in lock_file.for_packages:
+                    if package_uid and package_uid not in lock_file.for_packages:
                         lock_file.for_packages.append(package_uid)
                         lock_file.save(codebase)
                     yield lock_file

src/packagedcode/pypi.py

@@ -212,12 +212,12 @@ def assemble(cls, package_data, resource, codebase):
                 for py_res in cls.walk_pypi(resource=root, codebase=codebase):
                     if py_res.is_dir:
                         continue
-                    if package_uid not in py_res.for_packages:
+                    if package_uid and package_uid not in py_res.for_packages:
                         py_res.for_packages.append(package_uid)
                         py_res.save(codebase)
                     yield py_res
             elif codebase.has_single_resource:
-                if package_uid not in package_resource.for_packages:
+                if package_uid and package_uid not in package_resource.for_packages:
                     package_resource.for_packages.append(package_uid)
                     package_resource.save(codebase)
 
@@ -320,9 +320,10 @@ def assign_package_to_resources(cls, package, resource, codebase):
 
         package_uid = package.package_uid
 
-        # save thyself!
-        resource.for_packages.append(package_uid)
-        resource.save(codebase)
+        if package_uid:
+            # save thyself!
+            resource.for_packages.append(package_uid)
+            resource.save(codebase)
 
         # collect actual paths based on the file references
         for file_ref in package_data.file_references:
@@ -342,15 +343,16 @@ def assign_package_to_resources(cls, package, resource, codebase):
                     # TODO:w e should log these kind of things
                     continue
                 else:
-                    ref_resource.for_packages.append(package_uid)
-                    ref_resource.save(codebase)
+                    if package_uid:
+                        ref_resource.for_packages.append(package_uid)
+                        ref_resource.save(codebase)
             else:
                 ref_resource = get_resource_for_path(
                     path=path_ref,
                     root=site_packages,
                     codebase=codebase,
                 )
-                if ref_resource:
+                if ref_resource and package_uid:
                     ref_resource.for_packages.append(package_uid)
                     ref_resource.save(codebase)
 

src/packagedcode/rpm.py

@@ -210,10 +210,11 @@ def assemble(cls, package_data, resource, codebase):
             if not res:
                 missing_file_references.append(ref)
             else:
-                # path is found and processed: remove it, so we can check if we
-                # found all of them
-                res.for_packages.append(package_uid)
-                res.save(codebase)
+                if package_uid:
+                    # path is found and processed: remove it, so we can check if we
+                    # found all of them
+                    res.for_packages.append(package_uid)
+                    res.save(codebase)
 
         # if we have left over file references, add these to extra data
         if missing_file_references:

src/packagedcode/win_reg.py

@@ -380,8 +380,9 @@ def get_root_resource(cls, resource, codebase):
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
         package_uid = package.package_uid
-        resource.for_packages.append(package_uid)
-        resource.save(codebase)
+        if package_uid:
+            resource.for_packages.append(package_uid)
+            resource.save(codebase)
 
         refs = package.file_references
         if not refs:
@@ -406,11 +407,12 @@ def assign_package_to_resources(cls, package, resource, codebase):
             if not ref:
                 continue
 
-            # path is found and processed: remove it, so we can check if we
-            # found all of them
-            del refs_by_path[res.path]
-            res.for_packages.append(package_uid)
-            res.save(codebase)
+            if package_uid:
+                # path is found and processed: remove it, so we can check if we
+                # found all of them
+                del refs_by_path[res.path]
+                res.for_packages.append(package_uid)
+                res.save(codebase)
 
         # if we have left over file references, add these to extra data
         if refs_by_path: