Fix SPDX rule identifer mismatch bug

Fixes the SPDX rule identifier mismatch bug by getting rid of
extra space introduced in SPDX license declaration detection
and text cleaning.

Reference: #3634
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/licensedcode/licenses_reference.py

@@ -69,8 +69,8 @@ def process_codebase(self, codebase, **kwargs):
         Collect the ``license_references`` and ``rule_references``
         list of data mappings and add to the ``codebase``.
         """
-        include_files = 'license' in kwargs
-        include_packages = 'package' in kwargs
+        include_files = hasattr(codebase.attributes, 'license_detections')
+        include_packages = hasattr(codebase.attributes, 'packages')
 
         license_references, rule_references = collect_license_and_rule_references(
             codebase=codebase,
@@ -86,17 +86,24 @@ def collect_license_and_rule_references(codebase, include_packages=True, include
     Return a two-tuple of (``license_references``, ``license_rule_references``)
     sorted lists of unique mappings collected from a ``codebase``.
     """
+    if TRACE:
+        logger_debug(f'include_packages: {include_packages}, include_files: {include_files}')
 
     license_keys = set()
     rules_by_identifier = {}
 
     if include_packages:
         pks, prules = collect_references_from_packages(codebase)
-        license_keys.update(pks)
+        if TRACE:
+            logger_debug(f'collect_references_from_packages: license keys: {pks}')
+            logger_debug(f'collect_references_from_packages: rules by id: {prules}')
         rules_by_identifier.update(prules)
 
     if include_files:
         pks, prules = collect_references_from_files(codebase)
+        if TRACE:
+            logger_debug(f'collect_references_from_files: license keys: {pks}')
+            logger_debug(f'collect_references_from_files: rules by id: {prules}')
         license_keys.update(pks)
         rules_by_identifier.update(prules)
 
@@ -140,10 +147,6 @@ def collect_references_from_packages(codebase):
             if expression:
                 license_keys.update(licensing.license_keys(expression))
 
-        detections = getattr(resource, 'license_detections', []) or []
-        rules_by_id = build_rules_from_detection_data(detections)
-        rules_by_identifier.update(rules_by_id)
-
     for rule in rules_by_identifier.values():
         # TODO: consider using the expresion object directly instead
         expo = rule.license_expression

src/licensedcode/models.py

@@ -30,6 +30,7 @@
 from commoncode.fileutils import file_base_name
 from commoncode.fileutils import file_name
 from commoncode.fileutils import resource_iter
+from commoncode.text import python_safe_name
 from licensedcode import MIN_MATCH_HIGH_LENGTH
 from licensedcode import MIN_MATCH_LENGTH
 from licensedcode import SMALL_RULE
@@ -2125,6 +2126,10 @@ def from_file(cls, rule_file, is_builtin=True):
         rule.load_data(rule_file=rule_file)
         return rule
 
+    @property
+    def pysafe_expression(self):
+        return python_safe_name(self.license_expression)
+
     def load_data(self, rule_file):
         """
         Load data from ``rule_file`` which has both the text and the data (as YAML forntmatter).
@@ -2581,7 +2586,7 @@ class SpdxRule(SynthethicRule):
     """
 
     def __attrs_post_init__(self, *args, **kwargs):
-        self.identifier = f'spdx-license-identifier-{self.license_expression}-{self._unique_id}'
+        self.identifier = f'spdx-license-identifier-{self.pysafe_expression}-{self._unique_id}'
         self.setup()
 
         if not self.license_expression:
@@ -2635,7 +2640,7 @@ class UnDetectedRule(SynthethicRule):
     """
 
     def __attrs_post_init__(self, *args, **kwargs):
-        self.identifier = f'package-manifest-{self.license_expression}-{self._unique_id}'
+        self.identifier = f'package-manifest-{self.pysafe_expression}-{self._unique_id}'
         expression = self.licensing.parse(self.license_expression)
         self.license_expression = expression.render()
         self.license_expression_object = expression

src/licensedcode/query.py

@@ -507,7 +507,7 @@ def tokens_by_line(
             if spdx_start_offset is not None:
                 # keep the line, start/end known pos for SPDX matching
                 spdx_prefix, spdx_expression = split_spdx_lid(line)
-                spdx_text = ' '.join([spdx_prefix or '', spdx_expression])
+                spdx_text = ''.join([spdx_prefix or '', spdx_expression])
                 spdx_start_known_pos = line_first_known_pos + spdx_start_offset
 
                 if spdx_start_known_pos <= line_last_known_pos:

tests/licensedcode/data/match_spdx/scan-expected.json

@@ -1,7 +1,7 @@
 {
   "license_detections": [
     {
-      "identifier": "mit-86af397f-7fb9-6751-2e09-217685ce5b2a",
+      "identifier": "mit-59928f02-ade3-817b-5db4-dfe1c6738ef4",
       "license_expression": "mit",
       "license_expression_spdx": "MIT",
       "detection_count": 1,
@@ -48,14 +48,14 @@
               "matched_length": 5,
               "match_coverage": 100.0,
               "rule_relevance": 100,
-              "rule_identifier": "spdx-license-identifier-mit-6fcd9fa0c61347e06feb569ce3335c1b374640a1",
+              "rule_identifier": "spdx-license-identifier-mit-38fe0b852f1f8545c4a1b7ac0e456b182dbeb3ab",
               "rule_url": null,
               "matched_text": "<a href=\"https://licenses.nuget.org/MIT\">MIT</a>                </div>",
               "matched_text_diagnostics": "licenses.nuget.org/MIT\">MIT</"
             }
           ],
           "detection_log": [],
-          "identifier": "mit-86af397f-7fb9-6751-2e09-217685ce5b2a"
+          "identifier": "mit-59928f02-ade3-817b-5db4-dfe1c6738ef4"
         }
       ],
       "license_clues": [],