Fix from-file bug for codebase packages

Here `from-file` attributes were being assigned after creation of
top level packages so we had them empty. Adds a temporary fix to
this.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/packagedcode/plugin_package.py

@@ -25,11 +25,11 @@
 from licensedcode.cache import build_spdx_license_expression
 from licensedcode.cache import get_cache
 from licensedcode.detection import DetectionRule
+from licensedcode.detection import populate_matches_with_path
 from packagedcode import get_package_handler
 from packagedcode.licensing import add_referenced_license_matches_for_package
 from packagedcode.licensing import add_referenced_license_detection_from_package
 from packagedcode.licensing import add_license_from_sibling_file
-from packagedcode.licensing import get_license_detection_mappings
 from packagedcode.licensing import get_license_expression_from_detection_mappings
 from packagedcode.models import add_to_package
 from packagedcode.models import Dependency
@@ -328,7 +328,17 @@ def create_package_and_deps(codebase, package_adder=add_to_package, strip_root=F
         strip_root=strip_root,
         **kwargs
     )
-    codebase.attributes.packages.extend(pkg.to_dict() for pkg in packages)
+
+    package_mappings = []
+    for package in packages:
+        for detection in package.license_detections:
+            populate_matches_with_path(
+                matches=detection["matches"],
+                path=package.datafile_paths[0],
+            )
+        package_mappings.append(package.to_dict())
+
+    codebase.attributes.packages.extend(package_mappings)
     codebase.attributes.dependencies.extend(dep.to_dict() for dep in dependencies)
 
 

tests/formattedcode/data/common/manifests-expected.json

@@ -34,7 +34,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "manifests/maven/persistence-api-1.0.pom",
               "matched_length": 8,
               "match_coverage": 100.0,
               "matcher": "2-aho",
@@ -48,7 +48,7 @@
               "score": 100.0,
               "start_line": 2,
               "end_line": 2,
-              "from_file": null,
+              "from_file": "manifests/maven/persistence-api-1.0.pom",
               "matched_length": 7,
               "match_coverage": 100.0,
               "matcher": "2-aho",
@@ -127,7 +127,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "manifests/npm-license-mapping/package.json",
               "matched_length": 3,
               "match_coverage": 100.0,
               "matcher": "1-hash",
@@ -147,7 +147,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "manifests/npm-license-mapping/package.json",
               "matched_length": 3,
               "match_coverage": 100.0,
               "matcher": "1-hash",
@@ -222,7 +222,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "manifests/npm-license-string/package.json",
               "matched_length": 1,
               "match_coverage": 100.0,
               "matcher": "1-spdx-id",

tests/formattedcode/data/common/manifests-expected.jsonlines

@@ -66,7 +66,7 @@
                 "score": 100.0,
                 "start_line": 1,
                 "end_line": 1,
-                "from_file": null,
+                "from_file": "manifests/maven/persistence-api-1.0.pom",
                 "matched_length": 8,
                 "match_coverage": 100.0,
                 "matcher": "2-aho",
@@ -80,7 +80,7 @@
                 "score": 100.0,
                 "start_line": 2,
                 "end_line": 2,
-                "from_file": null,
+                "from_file": "manifests/maven/persistence-api-1.0.pom",
                 "matched_length": 7,
                 "match_coverage": 100.0,
                 "matcher": "2-aho",
@@ -159,7 +159,7 @@
                 "score": 100.0,
                 "start_line": 1,
                 "end_line": 1,
-                "from_file": null,
+                "from_file": "manifests/npm-license-mapping/package.json",
                 "matched_length": 3,
                 "match_coverage": 100.0,
                 "matcher": "1-hash",
@@ -179,7 +179,7 @@
                 "score": 100.0,
                 "start_line": 1,
                 "end_line": 1,
-                "from_file": null,
+                "from_file": "manifests/npm-license-mapping/package.json",
                 "matched_length": 3,
                 "match_coverage": 100.0,
                 "matcher": "1-hash",
@@ -254,7 +254,7 @@
                 "score": 100.0,
                 "start_line": 1,
                 "end_line": 1,
-                "from_file": null,
+                "from_file": "manifests/npm-license-string/package.json",
                 "matched_length": 1,
                 "match_coverage": 100.0,
                 "matcher": "1-spdx-id",

tests/formattedcode/data/common/manifests-expected.yaml

@@ -96,7 +96,7 @@ packages:
           - score: '100.0'
             start_line: 1
             end_line: 1
-            from_file:
+            from_file: manifests/maven/persistence-api-1.0.pom
             matched_length: 8
             match_coverage: '100.0'
             matcher: 2-aho
@@ -108,7 +108,7 @@ packages:
           - score: '100.0'
             start_line: 2
             end_line: 2
-            from_file:
+            from_file: manifests/maven/persistence-api-1.0.pom
             matched_length: 7
             match_coverage: '100.0'
             matcher: 2-aho
@@ -174,7 +174,7 @@ packages:
           - score: '100.0'
             start_line: 1
             end_line: 1
-            from_file:
+            from_file: manifests/npm-license-mapping/package.json
             matched_length: 3
             match_coverage: '100.0'
             matcher: 1-hash
@@ -189,7 +189,7 @@ packages:
           - score: '100.0'
             start_line: 1
             end_line: 1
-            from_file:
+            from_file: manifests/npm-license-mapping/package.json
             matched_length: 3
             match_coverage: '100.0'
             matcher: 1-hash
@@ -254,7 +254,7 @@ packages:
           - score: '100.0'
             start_line: 1
             end_line: 1
-            from_file:
+            from_file: manifests/npm-license-string/package.json
             matched_length: 1
             match_coverage: '100.0'
             matcher: 1-spdx-id

tests/formattedcode/data/yaml/package-and-licenses-expected.yaml

@@ -92,7 +92,7 @@ packages:
           - score: '100.0'
             start_line: 1
             end_line: 1
-            from_file:
+            from_file: package-and-licenses/setup.cfg
             matched_length: 3
             match_coverage: '100.0'
             matcher: 1-hash

tests/licensedcode/data/licenses_reference_reporting/scan-matched-text-with-reference.expected.json

@@ -47,7 +47,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "scan/package.json",
               "matched_length": 5,
               "match_coverage": 100.0,
               "matcher": "1-spdx-id",

tests/licensedcode/data/licenses_reference_reporting/scan-with-reference.expected.json

@@ -47,7 +47,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "scan/package.json",
               "matched_length": 5,
               "match_coverage": 100.0,
               "matcher": "1-spdx-id",

tests/packagedcode/data/about/aboutfiles.expected.json

@@ -42,7 +42,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "aboutfiles/apipkg.ABOUT",
               "matched_length": 1,
               "match_coverage": 100.0,
               "matcher": "1-spdx-id",
@@ -117,7 +117,7 @@
               "score": 100.0,
               "start_line": 1,
               "end_line": 1,
-              "from_file": null,
+              "from_file": "aboutfiles/appdirs.ABOUT",
               "matched_length": 1,
               "match_coverage": 100.0,
               "matcher": "1-spdx-id",

tests/packagedcode/data/alpine/alpine-container-layer.tar.xz-get-installed-expected.json

@@ -43,7 +43,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 4,
             "match_coverage": 100.0,
             "matcher": "1-hash",
@@ -253,7 +253,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 1,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",
@@ -355,7 +355,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 4,
             "match_coverage": 100.0,
             "matcher": "1-hash",
@@ -502,7 +502,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 4,
             "match_coverage": 100.0,
             "matcher": "1-hash",
@@ -712,7 +712,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 1,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",
@@ -1165,7 +1165,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 5,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",
@@ -1267,7 +1267,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 1,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",
@@ -1459,7 +1459,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 1,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",
@@ -1561,7 +1561,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 4,
             "match_coverage": 100.0,
             "matcher": "1-hash",
@@ -1651,7 +1651,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 1,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",
@@ -1753,7 +1753,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 4,
             "match_coverage": 100.0,
             "matcher": "1-hash",
@@ -1852,7 +1852,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 4,
             "match_coverage": 100.0,
             "matcher": "1-hash",
@@ -1942,7 +1942,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 11,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",
@@ -2068,7 +2068,7 @@
             "score": 100.0,
             "start_line": 1,
             "end_line": 1,
-            "from_file": null,
+            "from_file": "alpine-container-layer.tar.xz/lib/apk/db/installed",
             "matched_length": 7,
             "match_coverage": 100.0,
             "matcher": "1-spdx-id",