Add misc license detection improvements

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/licensedcode/data/rules/lead-in_unknown_43.RULE

@@ -1,6 +1,6 @@
 ---
 license_expression: unknown-license-reference
-is_license_tag: yes
+is_license_clue: yes
 relevance: 60
 notes: Creative commons tag seen in RDF or XML documents
 ---

src/licensedcode/detection.py

@@ -23,6 +23,7 @@
 
 from commoncode.resource import clean_path
 from commoncode.text import python_safe_name
+from commoncode.fileutils import as_posixpath
 from licensedcode.cache import build_spdx_license_expression
 from licensedcode.cache import get_cache
 from licensedcode.cache import get_index
@@ -130,6 +131,7 @@ class DetectionRule(Enum):
     EXTRA_WORDS = 'extra-words'
     LICENSE_CLUES = 'license-clues'
     LOW_QUALITY_MATCH_FRAGMENTS = 'low-quality-matches'
+    IMPERFECT_COVERAGE = 'imperfect-match-coverage'
     FALSE_POSITIVE = 'possible-false-positive'
     NOT_LICENSE_CLUES = 'not-license-clues-as-more-detections-present'
     UNKNOWN_REFERENCE_TO_LOCAL_FILE = 'unknown-reference-to-local-file'
@@ -1139,10 +1141,10 @@ def has_extra_words(license_matches):
 
 def has_low_rule_relevance(license_matches):
     """
-    Return True if any on the matches in ``license_matches`` List of LicenseMatch
+    Return True if all on the matches in ``license_matches`` List of LicenseMatch
     objects has a match with low score because of low rule relevance.
     """
-    return any(
+    return all(
         license_match.rule.relevance < LOW_RELEVANCE_THRESHOLD
         for license_match in license_matches
     )
@@ -1238,11 +1240,16 @@ def has_unknown_matches(license_matches):
 
 def is_unknown_intro(license_match):
     """
-    Return True if the LicenseMatch is an unknown license intro.
+    Return True if the LicenseMatch is unknown and can be considered
+    as a license intro to other license matches.
+    I.e. this is not an unknown when followed by other proper matches.
     """
     return (
         license_match.rule.has_unknown and
-        license_match.rule.is_license_intro
+        (
+            license_match.rule.is_license_intro or license_match.rule.is_license_clue or
+            license_match.rule.license_expression == 'free-unknown'
+        )
     )
 
 
@@ -1338,7 +1345,10 @@ def is_license_intro(license_match):
     from licensedcode.match_aho import MATCH_AHO_EXACT
 
     return (
-        license_match.rule.is_license_intro
+        (
+            license_match.rule.is_license_intro or license_match.rule.is_license_clue or
+            license_match.rule.license_expression == 'free-unknown'
+        )
         and (
             license_match.matcher == MATCH_AHO_EXACT
             or license_match.coverage() == 100
@@ -1554,10 +1564,16 @@ def get_detected_license_expression(
     elif analysis == DetectionCategory.EXTRA_WORDS.value:
         if TRACE_ANALYSIS:
             logger_debug(f'analysis {DetectionCategory.EXTRA_WORDS.value}')
-        # Apply filtering or handling logic if needed
+        # TODO: Fix score if extra words allowed in rules
         matches_for_expression = license_matches
         detection_log.append(DetectionRule.EXTRA_WORDS.value)
 
+    elif analysis == DetectionCategory.IMPERFECT_COVERAGE.value:
+        if TRACE_ANALYSIS:
+            logger_debug(f'analysis {DetectionCategory.IMPERFECT_COVERAGE.value}')
+        matches_for_expression = license_matches
+        detection_log.append(DetectionRule.IMPERFECT_COVERAGE.value)
+
     else:
         if TRACE_ANALYSIS:
             logger_debug(f'analysis not-combined')
@@ -1903,7 +1919,11 @@ def find_referenced_resource_from_package(referenced_filename, resource, codebas
 
         datafile_paths = datafile_paths_by_package_uid.get(package_uid)
         for path in datafile_paths:
-            datafile_path = posixpath.join(root_path, path)
+            # support strip_root and normal cases
+            if not as_posixpath(path).startswith(f"{as_posixpath(root_path)}/"):
+                datafile_path = posixpath.join(root_path, path)
+            else:
+                datafile_path = path
             datafile_resource = codebase.get_resource(path=datafile_path)
             if not datafile_resource or not datafile_resource.parent_path():
                 continue
@@ -1941,8 +1961,6 @@ def find_referenced_resource(referenced_filename, resource, codebase, **kwargs):
         return resource
 
     # Also look at codebase root for referenced file
-    # TODO: look at project root identified by key-files
-    # instead of codebase scan root
     root_path = codebase.root.path
     path = posixpath.join(root_path, referenced_filename)
     resource = codebase.get_resource(path=path)

tests/formattedcode/data/common/manifests-expected.yaml

@@ -29,9 +29,9 @@ headers:
       system_environment:
         operating_system: linux
         cpu_architecture: 64
-        platform: Linux-6.8.0-52-generic-x86_64-with-glibc2.35
-        platform_version: '#53~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Wed Jan 15 19:18:46 UTC 2'
-        python_version: 3.12.9 (main, Feb 20 2025, 19:13:20) [GCC 11.4.0]
+        platform: Linux-5.15.0-141-generic-x86_64-with-glibc2.35
+        platform_version: '#151-Ubuntu SMP Sun May 18 21:35:19 UTC 2025'
+        python_version: 3.10.12 (main, Feb  4 2025, 14:57:36) [GCC 11.4.0]
       spdx_license_list_version: '3.26'
       files_count: 4
 summary:
@@ -1882,13 +1882,14 @@ files:
     is_media: no
     is_source: no
     is_script: no
-    package_data: []
-    for_packages: []
     is_legal: no
     is_manifest: no
     is_readme: no
     is_top_level: yes
     is_key_file: no
+    is_community: no
+    package_data: []
+    for_packages: []
     detected_license_expression:
     detected_license_expression_spdx:
     license_detections: []
@@ -1921,14 +1922,15 @@ files:
     is_media: no
     is_source: no
     is_script: no
-    package_data: []
-    for_packages:
-      - pkg:maven/javax.persistence/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758
     is_legal: no
     is_manifest: no
     is_readme: no
     is_top_level: yes
     is_key_file: no
+    is_community: no
+    package_data: []
+    for_packages:
+      - pkg:maven/javax.persistence/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758
     detected_license_expression:
     detected_license_expression_spdx:
     license_detections: []
@@ -1961,6 +1963,12 @@ files:
     is_media: no
     is_source: no
     is_script: no
+    is_legal: no
+    is_manifest: no
+    is_readme: no
+    is_top_level: no
+    is_key_file: no
+    is_community: no
     package_data:
       - type: maven
         namespace: javax.persistence
@@ -2032,11 +2040,6 @@ files:
         purl: pkg:maven/javax.persistence/[email protected]
     for_packages:
       - pkg:maven/javax.persistence/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758
-    is_legal: no
-    is_manifest: no
-    is_readme: no
-    is_top_level: no
-    is_key_file: no
     detected_license_expression: cddl-1.0
     detected_license_expression_spdx: CDDL-1.0
     license_detections:
@@ -2096,13 +2099,14 @@ files:
     is_media: no
     is_source: no
     is_script: no
-    package_data: []
-    for_packages: []
     is_legal: no
     is_manifest: no
     is_readme: no
     is_top_level: yes
     is_key_file: no
+    is_community: no
+    package_data: []
+    for_packages: []
     detected_license_expression:
     detected_license_expression_spdx:
     license_detections: []
@@ -2135,6 +2139,12 @@ files:
     is_media: no
     is_source: no
     is_script: no
+    is_legal: no
+    is_manifest: no
+    is_readme: no
+    is_top_level: no
+    is_key_file: no
+    is_community: no
     package_data:
       - type: npm
         namespace:
@@ -2315,11 +2325,6 @@ files:
         purl: pkg:npm/[email protected]
     for_packages:
       - pkg:npm/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758
-    is_legal: no
-    is_manifest: no
-    is_readme: no
-    is_top_level: no
-    is_key_file: no
     detected_license_expression: apache-2.0
     detected_license_expression_spdx: Apache-2.0
     license_detections:
@@ -2407,13 +2412,14 @@ files:
     is_media: no
     is_source: no
     is_script: no
-    package_data: []
-    for_packages: []
     is_legal: no
     is_manifest: no
     is_readme: no
     is_top_level: yes
     is_key_file: no
+    is_community: no
+    package_data: []
+    for_packages: []
     detected_license_expression:
     detected_license_expression_spdx:
     license_detections: []
@@ -2446,6 +2452,12 @@ files:
     is_media: no
     is_source: no
     is_script: no
+    is_legal: no
+    is_manifest: no
+    is_readme: no
+    is_top_level: no
+    is_key_file: no
+    is_community: no
     package_data:
       - type: npm
         namespace:
@@ -2559,11 +2571,6 @@ files:
         purl: pkg:npm/[email protected]
     for_packages:
       - pkg:npm/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758
-    is_legal: no
-    is_manifest: no
-    is_readme: no
-    is_top_level: no
-    is_key_file: no
     detected_license_expression: mit
     detected_license_expression_spdx: MIT
     license_detections:
@@ -2628,13 +2635,14 @@ files:
     is_media: no
     is_source: no
     is_script: no
-    package_data: []
-    for_packages: []
     is_legal: no
     is_manifest: no
     is_readme: no
     is_top_level: yes
     is_key_file: no
+    is_community: no
+    package_data: []
+    for_packages: []
     detected_license_expression:
     detected_license_expression_spdx:
     license_detections: []
@@ -2667,6 +2675,12 @@ files:
     is_media: no
     is_source: yes
     is_script: yes
+    is_legal: no
+    is_manifest: no
+    is_readme: no
+    is_top_level: no
+    is_key_file: no
+    is_community: no
     package_data:
       - type: pypi
         namespace:
@@ -2863,11 +2877,6 @@ files:
         purl: pkg:pypi/bluepyopt
     for_packages:
       - pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758
-    is_legal: no
-    is_manifest: no
-    is_readme: no
-    is_top_level: no
-    is_key_file: no
     detected_license_expression: lgpl-3.0
     detected_license_expression_spdx: LGPL-3.0-only
     license_detections: