Fix misc license/package detection bugs

* Add rules for license detection fixes
* Fix detection for pom.properties file
* Fix --todo plugin for package detection
* Fix false positive case for bare word rules
* Fix issue of unknown license detection in package manifest

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/licensedcode/data/rules/bsd-new_1300.RULE

@@ -0,0 +1,11 @@
+---
+license_expression: bsd-new
+is_license_tag: yes
+referenced_filenames:
+    - https://github.com/jimsch/COSE-JAVA/blob/master/LICENSE
+ignorable_urls:
+    - https://github.com/jimsch/COSE-JAVA/blob/master/LICENSE
+---
+
+name: BSD3 
+url: {{https://github.com/jimsch/COSE-JAVA/blob/master/LICENSE}}

src/licensedcode/data/rules/cc0-1.0_203.RULE

@@ -0,0 +1,10 @@
+---
+license_expression: cc0-1.0
+is_license_tag: yes
+relevance: 100
+ignorable_urls:
+    - http://www.creativecommons.org/publicdomain/zero/1.0/
+---
+
+name: CC0 universal
+url: http://www.creativecommons.org/publicdomain/zero/1.0/

src/licensedcode/detection.py

@@ -667,7 +667,14 @@ def to_dict(
 
 def collect_license_detections(codebase, include_license_clues=True):
     """
-    Return a list of LicenseDetectionFromResult from a ``codebase``
+    Return a list of LicenseDetectionFromResult object rehydrated from
+    LicenseDetection mappings, from resources and packages in a ``codebase``.
+
+    As a side effect, this also corrects `declared_license_expression` in packages
+    according to their license detections. This is required because package fields
+    are populated in package plugin, which runs before the license plugin, and thus
+    the license plugin step where unknown references to other files are dereferenced
+    does not show up automatically in package attributes. 
     """
     has_packages = hasattr(codebase.root, 'package_data')
     has_licenses = hasattr(codebase.root, 'license_detections')
@@ -711,13 +718,33 @@ def collect_license_detections(codebase, include_license_clues=True):
             package_data = getattr(resource, 'package_data', []) or []
 
             package_license_detection_mappings = []
+            modified = False
             for package in package_data:
 
-                if package["license_detections"]:
-                    package_license_detection_mappings.extend(package["license_detections"])
-
-                if package["other_license_detections"]:
-                    package_license_detection_mappings.extend(package["other_license_detections"])
+                package_license_detections = package["license_detections"]
+                if package_license_detections:
+                    package_license_detection_mappings.extend(package_license_detections)
+                    detection_is_same, license_expression = verify_package_license_expression(
+                        license_detection_mappings=package_license_detections,
+                        license_expression=package["declared_license_expression"]
+                    )
+                    if not detection_is_same:
+                        package["declared_license_expression"] = license_expression
+                        modified = True
+
+                other_license_detections = package["other_license_detections"]
+                if other_license_detections:
+                    package_license_detection_mappings.extend(other_license_detections)
+                    detection_is_same, license_expression = verify_package_license_expression(
+                        license_detection_mappings=other_license_detections,
+                        license_expression=package["other_license_expression"]
+                    )
+                    if not detection_is_same:
+                        package["other_license_expression"] = license_expression
+                        modified = True
+
+            if modified:
+                codebase.save_resource(resource)
 
             if package_license_detection_mappings:
                 package_license_detection_objects = detections_from_license_detection_mappings(
@@ -729,6 +756,33 @@ def collect_license_detections(codebase, include_license_clues=True):
     return all_license_detections
 
 
+
+def verify_package_license_expression(license_detection_mappings, license_expression):
+    """
+    Returns a tuple of two files: `detection_is_same` and `license_expression` depending
+    on whether the `license_expression` is same as the license_expression computed from
+    `license_detection_mappings`:
+    1. If they are the same, we return True and None for the `license_expression`
+    2. If they are not the same, we return False, and the computed `license_expression`
+    """
+    license_expressions_from_detections = [
+            detection["license_expression"]
+            for detection in license_detection_mappings
+        ]
+
+    license_expression_from_detections = str(combine_expressions(
+        expressions=license_expressions_from_detections,
+        relation='AND',
+        unique=True,
+    ))
+
+    if not license_expression_from_detections == license_expression:
+        return False, license_expression_from_detections
+    else:
+        return True, None
+
+
+
 @attr.s
 class UniqueDetection:
     """
@@ -978,9 +1032,12 @@ def is_false_positive(license_matches, package_license=False):
         match_rule_length == 1
         for match_rule_length in match_rule_length_values
     )
-
-    is_gpl_bare = all(
-        'gpl_bare' in license_match.rule.identifier
+    bare_rules = ['gpl_bare', 'freeware_bare', 'public-domain_bare']
+    is_bare_rule = all(
+        any([
+            bare_rule in license_match.rule.identifier
+            for bare_rule in bare_rules
+        ])
         for license_match in license_matches
     )
 
@@ -995,7 +1052,7 @@ def is_false_positive(license_matches, package_license=False):
 
     is_single_match = len(license_matches) == 1
 
-    if is_single_match and is_gpl_bare:
+    if is_single_match and is_bare_rule:
         return True
 
     if is_gpl and all_match_rule_length_one:

src/packagedcode/maven.py

@@ -313,12 +313,27 @@ def parse(cls, location):
             if TRACE:
                 logger.debug(f'MavenPomPropertiesHandler.parse: properties: {properties!r}')
             if properties:
-                yield models.PackageData(
-                    datasource_id=cls.datasource_id,
-                    type=cls.default_package_type,
-                    primary_language=cls.default_primary_language,
-                    extra_data=dict(pom_properties=properties)
-                )
+                yield from cls.parse_pom_properties(properties=properties) 
+
+    @classmethod
+    def parse_pom_properties(cls, properties):
+        namespace = properties.pop("groupId", None)
+        name = properties.pop("artifactId", None)
+        version = properties.pop("version", None)
+        if properties:
+            extra_data = dict(pom_properties=properties)
+        else:
+            extra_data = {}
+
+        yield models.PackageData(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            primary_language=cls.default_primary_language,
+            name=name,
+            namespace=namespace,
+            version=version,
+            extra_data=extra_data,
+        )
 
 
 def build_url(

src/summarycode/todo.py

@@ -21,6 +21,7 @@
 from licensedcode.detection import get_uuid_on_content
 from licensedcode.detection import UniqueDetection
 from plugincode.post_scan import PostScanPlugin, post_scan_impl
+from packageurl import PackageURL
 
 TRACE = os.environ.get('SCANCODE_DEBUG_REVIEW', False)
 
@@ -171,7 +172,7 @@ def get_ambiguous_package_detections(codebase):
         for package in package_data:
             detection_type = None
             if not package["purl"]:
-                if resource.path not in deps_datafile_paths:
+                if resource.path not in deps_datafile_paths and not resource.for_packages:
                     detection_type=PackageDetectionCategory.CANNOT_CREATE_PURL.value
             else:
                 if package["purl"] not in codebase_packages_purls:
@@ -211,6 +212,11 @@ def get_package_identifier(package_data, file_path):
     return get_uuid_on_content(content=[identifier_elements])
 
 
+def get_unknown_purl(package_type):
+    purl = PackageURL(type=package_type, name="unknown")
+    return purl.to_string()
+
+
 @attr.s
 class AmbiguousDetection:
     """
@@ -252,6 +258,8 @@ class AmbiguousDetection:
     @classmethod
     def from_package(cls, package_data, detection_log, file_path):
         purl = package_data["purl"]
+        if not purl:
+            purl = get_unknown_purl(package_data["type"])
         identifier = get_package_identifier(package_data, file_path)
         detection_id = f"{purl}-{identifier}"
         file_region = FileRegion(

tests/packagedcode/data/license_detection/reference-to-package/samba.expected.json

@@ -795,7 +795,7 @@
           "vcs_url": null,
           "copyright": null,
           "holder": null,
-          "declared_license_expression": "gpl-3.0 AND (gpl-3.0 AND lgpl-3.0 AND gpl-2.0) AND (gpl-2.0-plus AND free-unknown AND gpl-1.0-plus) AND (gpl-1.0-plus AND lgpl-3.0-plus AND gpl-3.0 AND lgpl-3.0) AND (cc-by-sa-3.0 AND cc-by-sa-4.0 AND dco-1.1) AND gpl-2.0 AND gpl-1.0-plus",
+          "declared_license_expression": "gpl-3.0 AND (gpl-3.0 AND lgpl-3.0 AND gpl-2.0) AND (gpl-2.0-plus AND free-unknown AND gpl-1.0-plus) AND (gpl-1.0-plus AND lgpl-3.0-plus AND gpl-3.0 AND lgpl-3.0) AND (cc-by-sa-3.0 AND cc-by-sa-4.0 AND dco-1.1) AND gpl-2.0",
           "declared_license_expression_spdx": "GPL-3.0-only AND (GPL-3.0-only AND LGPL-3.0-only AND GPL-2.0-only) AND (GPL-2.0-or-later AND LicenseRef-scancode-free-unknown AND GPL-1.0-or-later) AND (GPL-1.0-or-later AND LGPL-3.0-or-later AND GPL-3.0-only AND LGPL-3.0-only) AND (CC-BY-SA-3.0 AND CC-BY-SA-4.0 AND LicenseRef-scancode-dco-1.1) AND GPL-2.0-only AND GPL-1.0-or-later",
           "license_detections": [
             {
@@ -1157,7 +1157,7 @@
           "vcs_url": null,
           "copyright": null,
           "holder": null,
-          "declared_license_expression": "gpl-3.0 AND (gpl-3.0 AND lgpl-3.0 AND gpl-2.0) AND (gpl-2.0-plus AND free-unknown AND gpl-1.0-plus) AND (gpl-1.0-plus AND lgpl-3.0-plus AND gpl-3.0 AND lgpl-3.0) AND (cc-by-sa-3.0 AND cc-by-sa-4.0 AND dco-1.1) AND gpl-2.0 AND gpl-1.0-plus",
+          "declared_license_expression": "gpl-3.0 AND (gpl-3.0 AND lgpl-3.0 AND gpl-2.0) AND (gpl-2.0-plus AND free-unknown AND gpl-1.0-plus) AND (gpl-1.0-plus AND lgpl-3.0-plus AND gpl-3.0 AND lgpl-3.0) AND (cc-by-sa-3.0 AND cc-by-sa-4.0 AND dco-1.1) AND gpl-2.0",
           "declared_license_expression_spdx": "GPL-3.0-only AND (GPL-3.0-only AND LGPL-3.0-only AND GPL-2.0-only) AND (GPL-2.0-or-later AND LicenseRef-scancode-free-unknown AND GPL-1.0-or-later) AND (GPL-1.0-or-later AND LGPL-3.0-or-later AND GPL-3.0-only AND LGPL-3.0-only) AND (CC-BY-SA-3.0 AND CC-BY-SA-4.0 AND LicenseRef-scancode-dco-1.1) AND GPL-2.0-only AND GPL-1.0-or-later",
           "license_detections": [
             {