Merge branch 'improve-required' into 4238-required-phrases-with-stopwords

Author: pombredanne

src/licensedcode/detection.py

@@ -338,20 +338,10 @@ def identifier_with_expression(self):
     def is_unknown(self):
         """
         Return True if there are unknown license keys in the license expression
-        for this detection, return False otherwise.
+        for this detection, return False otherwise. By design these are licenses with "unknown" in
+        their key.
         """
-        unknown_license_keys = [
-            "unknown-license-reference",
-            "unknown-spdx",
-            "unknown",
-            "free-unknown"
-        ]
-
-        for license_key in unknown_license_keys:
-            if license_key in self.license_expression:
-                return True
-
-        return False
+        return 'unknown' in self.license_expression
 
     def get_start_end_line(self):
         """

src/licensedcode/models.py

@@ -2355,7 +2355,7 @@ def _set_continuous(self):
 
     def build_required_phrase_spans(self):
         """
-        Return a list of Spans marking key phrases token positions of that must
+        Return a list of Spans marking required phrases token positions of that must
         be present for this rule to be matched.
         """
         if self.is_from_license:

src/packagedcode/alpine.py

@@ -8,6 +8,7 @@
 #
 import base64
 import codecs
+import dataclasses
 import email
 import posixpath
 import re
@@ -1362,32 +1363,76 @@ def source_handler(value, **kwargs):
 }
 
 
-def detect_declared_license(declared):
+@dataclasses.dataclass
+class ApkLicenseDetection:
+    """
+    Represent the results of an Alpine license detection, including intermediate steps.
     """
-    Return a tuple of (cleaned declared license, detected license expression)
-    strings from a ``declared`` license text. Both can be None.
+    declared_license: str
+    cleaned_license: str
+    mapped_license: str
+    license_detections: list
+    license_expression: str
+
+    def to_dict(self):
+        return dict(
+            declared_license=self.declared_license                        ,
+            cleaned_license=self.cleaned_license                         ,
+            mapped_license=self.mapped_license                          ,
+            license_detections=self.license_detections                            ,
+            license_expression=self.license_expression             ,
+        )
+
+
+def get_alpine_license_detection(declared):
+    """
+    Return an ApkLicenseDetection from a ``declared`` license text
     """
     # cleaning first to fix syntax quirks and try to get something we can parse
-    cleaned = normalize_and_cleanup_declared_license(declared)
-    if not cleaned:
-        return None, None
+    cleaned_license = normalize_and_cleanup_declared_license(declared)
+    if not cleaned_license:
+        return None
 
-    # then we apply mappings for known non-standard symbols
+    # then we apply mappings for known non-standard symbols.
     # the output should be a proper SPDX expression
-    mapped = apply_expressions_mapping(cleaned)
+    mapped_license = apply_expressions_mapping(cleaned_license)
 
     # Finally perform SPDX expressions detection: Alpine uses mostly SPDX, but
     # with some quirks such as some non standard symbols (in addition to the
     # non-standard syntax)
     extra_licenses = {}
     expression_symbols = get_license_symbols(extra_licenses=extra_licenses)
 
-    license_detections, detected_license_expression = get_license_detections_and_expression(
-        extracted_license_statement=mapped,
+    license_detections, license_expression = get_license_detections_and_expression(
+        extracted_license_statement=mapped_license,
         expression_symbols=expression_symbols,
     )
 
-    return cleaned, detected_license_expression, license_detections
+    return ApkLicenseDetection(
+        declared_license=declared,
+        cleaned_license=cleaned_license,
+        mapped_license=mapped_license,
+        license_expression=license_expression,
+        license_detections=license_detections,
+    )
+
+
+def detect_declared_license(declared):
+    """
+    Return a three-tuple of detected license data from a ``declared`` license text, with this shape:
+        (cleaned declared license, detected license expression, license_detections)
+    - cleaned declared license and detected license expression are strings.
+    - license_detections is a list of LicenseDetection.
+    - Any of these can be None.
+    """
+    if alpine_detection := get_alpine_license_detection(declared):
+        return (
+            alpine_detection.cleaned_license,
+            alpine_detection.license_expression,
+            alpine_detection.license_detections,
+        )
+    else:
+        return None, None, None
 
 
 def get_license_symbols(extra_licenses):
@@ -1416,25 +1461,19 @@ def get_license_symbols(extra_licenses):
 def normalize_and_cleanup_declared_license(declared):
     """
     Return a cleaned and normalized declared license.
-
-    The expression should be valida SPDX but are far from this in practice.
-
+    The expressions should be valid SPDX license expressions but they are far from this in practice.
     Several fixes are applied:
-
     - plain text replacemnet aka. syntax fixes are plain text replacements
       to make the expression parsable
-
     - common fixes includes also nadling space-separated and comma-separated
       lists of licenses
     """
     declared = declared or ''
 
-    # normalize spaces
+    # normalize spaces and case
     declared = ' '.join(declared.split())
-
     declared = declared.lower()
 
-    # performa replacements
     declared = apply_syntax_fixes(declared)
 
     # comma-separated as in gpl-2.0+, lgpl-2.1+, zlib
@@ -1516,15 +1555,15 @@ def normalize_and_cleanup_declared_license(declared):
 
 def apply_syntax_fixes(s):
     """
-    Fix the expression string s by aplying replacement for various quirks.
+    Fix the expression string ``s`` by aplying replacement for various quirks to get clean license
+    expression syntax.
     """
     for src, tgt in EXPRESSION_SYNTAX_FIXES.items():
         s = s.replace(src, tgt)
     return s
 
-# These are parsed expression objects replacement that make the expression SPDX compliant
-
 
+# These are parsed expression objects replacement that make the expression SPDX compliant
 # {alpine sub-expression: SPDX subexpression}
 DECLARED_TO_SPDX = {
     'openssl-exception': 'licenseref-scancode-generic-exception',

src/packagedcode/licensing.py

@@ -10,6 +10,7 @@
 import logging
 import os
 
+import saneyaml
 from license_expression import Licensing
 
 from licensedcode.cache import build_spdx_license_expression
@@ -35,9 +36,6 @@
 from summarycode.classify import LEGAL_STARTS_ENDS
 from summarycode.classify import README_STARTS_ENDS
 
-import saneyaml
-
-
 """
 Detect and normalize licenses as found in package manifests data.
 """
@@ -59,7 +57,6 @@ def logger_debug(*args):
     def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
-
 RESOURCE_TO_PACKAGE_LICENSE_FIELDS = {
     'detected_license_expression': 'declared_license_expression',
     'detected_license_expression_spdx': 'declared_license_expression_spdx',
@@ -255,7 +252,7 @@ def add_referenced_license_detection_from_package(resource, codebase):
                 continue
 
             for sibling_detection in sibling_license_detections:
-                
+
                 modified = True
                 detection_modified = True
                 license_match_mappings.extend(sibling_detection["matches"])
@@ -585,7 +582,9 @@ def get_license_expression_from_detection_mappings(
 
 def matches_have_unknown(matches, licensing=Licensing()):
     """
-    Return True if any of the LicenseMatch in `matches` has an unknown license.
+    Return True if any of the LicenseMatch in ``matches`` has an unknown license.
+    Note that by construction and design, an unknown license must have the word "unknown" in its
+    license key, but we only care about two specific license keys, and not all license keys.
     """
     for match in matches:
         exp = match.rule.license_expression_object
@@ -719,8 +718,7 @@ def get_normalized_license_detections(
     expression_symbols=None,
 ):
     """
-    Return a normalized license expression string detected from a list of
-    declared license items.
+    Return a list of LicenseDetection detected in ``extracted license`` data.
     """
     license_detections = []
 
@@ -743,6 +741,7 @@ def get_normalized_license_detections(
                 logger_debug(f'get_normalized_license_detections: str:')
 
         elif isinstance(extracted_license, dict):
+            # FIXME: why ignoring keys?
             for extracted_license_statement in extracted_license.values():
                 detections = get_license_detections_for_extracted_license_statement(
                     extracted_license_statement=extracted_license_statement,
@@ -757,6 +756,7 @@ def get_normalized_license_detections(
                     license_detections.extend(detections)
 
             if not license_detections:
+                # FIXME: we should  Never detect on dict representation, but on a YAML dump instead
                 unknown_dict_object = repr(dict(extracted_license.items()))
                 unknown_detection = get_unknown_license_detection(query_string=unknown_dict_object)
                 license_detections.append(unknown_detection)
@@ -828,7 +828,7 @@ def get_license_detections_and_expression(
     try_as_expression=True,
     approximate=True,
     expression_symbols=None,
-    datasource_id = None,
+    datasource_id=None,
 ):
     """
     Given a text `extracted_license_statement` return a list of LicenseDetection objects.
@@ -893,8 +893,7 @@ def get_license_detections_for_extracted_license_statement(
     expression_symbols=None,
 ):
     """
-    Return a list of LicenseDetection objects after detecting licenses in
-    the given `extracted_license_statement`.
+    Return a list of LicenseDetection detected  the ``extracted_license_statement`` string.
     """
     if not extracted_license_statement:
         return []

src/packagedcode/recognize.py

@@ -118,6 +118,8 @@ def _parse(
             # We should continue when an Exception has occured when trying to
             # recognize a package
             if TRACE:
-                logger_debug(f'_parse: Exception: {str(e)}')
+                import traceback
+                logger_debug(f'_parse: Exception: {str(e)} : {traceback.format_exc()}')
+                raise Exception(f'_parse: error') from e
 
             continue