Update and improve collecting and marking required phrases

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

etc/scripts/licenses/buildrules.py

@@ -16,7 +16,7 @@
 from licensedcode import models
 from licensedcode import match_hash
 from licensedcode import frontmatter
-from licensedcode.models import rule_exists
+from licensedcode.models import get_rule_id_for_text
 from license_expression import Licensing
 
 """
@@ -330,7 +330,7 @@ def cli(licenses_file, dump_to_file_on_errors=False):
 
         text = rule.text
 
-        existing_rule = rule_exists(text)
+        existing_rule = get_rule_id_for_text(text)
         skinny_text = " ".join(text[:80].split()).replace("{", " ").replace("}", " ")
 
         existing_msg = (

src/licensedcode/data/rules/mit_1155.RULE

@@ -2,6 +2,7 @@
 license_expression: mit
 is_license_notice: yes
 relevance: 100
+skip_collecting_required_phrases: yes
 ---
 
 For license terms see {{SLF4J}}

src/licensedcode/data/rules/mit_1302.RULE

@@ -5,4 +5,4 @@ referenced_filenames:
     - LICENSE
 ---
 
-is free software: you can redistribute it and/or modify it under the terms of {{the MIT License}} as published by the Open Source Initiative. See the {{ LICENSE file }} for more details.
+is free software: you can redistribute it and/or modify it under the terms of {{the MIT License}} as published by the Open Source Initiative. See the LICENSE file for more details.

src/licensedcode/data/rules/mit_9.RULE

@@ -6,4 +6,4 @@ ignorable_urls:
     - http://en.wikipedia.org/wiki/MIT_License
 ---
 
-Licensed under the MIT (http://en.wikipedia.org/wiki/MIT_License) license.
+{{Licensed under the MIT}} (http://en.wikipedia.org/wiki/MIT_License) license.

src/licensedcode/models.py

@@ -1844,7 +1844,8 @@ def validate(self, licensing=None, thorough=False):
         is_false_positive = self.is_false_positive
 
         has_license_flags = any(self.license_flag_values)
-        has_many_license_flags = len([l for l in self.license_flag_values if l]) != 1
+        has_no_license_flags = len([l for l in self.license_flag_values if l]) == 0
+        has_many_license_flags = len([l for l in self.license_flag_values if l]) > 1
 
         license_expression = self.license_expression
 
@@ -1887,6 +1888,9 @@ def validate(self, licensing=None, thorough=False):
 
             if not (0 <= self.relevance <= 100):
                 yield 'Invalid rule relevance. Should be between 0 and 100.'
+            
+            if has_no_license_flags:
+                yield 'Invalid rule no is_license_* flags present.'
 
             if has_many_license_flags:
                 yield 'Invalid rule is_license_* flags. Only one allowed.'
@@ -2243,10 +2247,7 @@ def build_required_phrase_spans(self):
         if self.is_from_license:
             return []
         try:
-            return [
-                required_phrase.span
-                for required_phrase in get_required_phrase_spans(self.text)
-            ]
+            return get_required_phrase_spans(self.text)
         except Exception as e:
             raise InvalidRule(f'Invalid rule: {self}') from e
 
@@ -2327,6 +2328,9 @@ def load(self, rule_file, with_checks=True):
             raise e
 
         known_attributes = set(attr.fields_dict(self.__class__))
+        # This is an attirbute used to debug marking required phrases, and is not needed
+        if "sources" in data:
+            data.pop("sources")
         data_file_attributes = set(data)
         if with_checks:
             unknown_attributes = data_file_attributes.difference(known_attributes)
@@ -2870,8 +2874,16 @@ def rule_exists(text):
     if len(matches) > 1:
         return False
     match = matches[0]
-    if match.matcher == MATCH_HASH and match.score() == 100:
-        return match.rule.identifier
+    if match.matcher == MATCH_HASH and match.coverage() == 100:
+        return match.rule
+
+
+def get_rule_id_for_text(text):
+    rule = rule_exists(text=text)
+    if rule:
+        return rule.identifier
+    else:
+        return False
 
 
 def find_rule_base_location(name_prefix, rules_directory=rules_data_dir):