add more tests for is_extra_words_at_valid_positions and improve detection

Signed-off-by: Alok Kumar <[email protected]>

Author: alok1304

src/licensedcode/detection.py

@@ -31,6 +31,7 @@
 from licensedcode.match import LicenseMatch
 from licensedcode.match import set_matched_lines
 from licensedcode.match import is_extra_words_position_valid
+from licensedcode.match import is_extra_words_at_valid_positions
 from licensedcode.models import compute_relevance
 from licensedcode.models import Rule
 from licensedcode.models import UnDetectedRule
@@ -1167,21 +1168,6 @@ def has_low_rule_relevance(license_matches):
     )
 
 
-def is_extra_words_at_valid_positions(license_matches):
-    """
-    Return True if all the matches in `license_matches List of LicenseMatch
-    has extra words are in the correct place.
-    """
-    for match in license_matches:
-        # check when we have `extra-words` detection
-        # if `query_coverage_coefficient` is positive number then 'extra-words` exit
-        if calculate_query_coverage_coefficient(match) > 0:
-            if not is_extra_words_position_valid(match):
-                return False
-            
-    # at the end return True if all matches have no extra-wors or this extra-words are in the right place
-    return True
-
 def is_false_positive(license_matches, package_license=False):
     """
     Return True if all of the matches in ``license_matches`` List of LicenseMatch

src/licensedcode/match.py

@@ -1133,18 +1133,27 @@ def is_extra_words_position_valid(match):
 
     rule_index+=1    
 
-    # check if any `extra-words` is present and return False because this `extra-words` are not at marked place
+    # check if any `extra-words` is present after checking all `extra-phrase-spans` in rules
     while (matched_index < len(matched_tokens) and
         matched_tokens[matched_index] == rule_tokens[rule_index]):        
         matched_index+=1
         rule_index+=1
-
+    
+    # some `extra-words` are found
     if matched_index != len(matched_tokens):
         return False    
 
     return True
 
 
+def is_extra_words_at_valid_positions(license_matches):
+    """
+    Return True if any of the matches in `license_matches` that have `extra-words`
+    are in the right place.
+    """
+    return any(is_extra_words_position_valid(match) for match in license_matches)
+
+
 def filter_contained_matches(
     matches,
     trace=TRACE_FILTER_CONTAINED,

tests/licensedcode/test_match.py

@@ -21,6 +21,7 @@
 from licensedcode.match import get_full_matched_text
 from licensedcode.match import get_matching_regions
 from licensedcode.match import is_extra_words_position_valid
+from licensedcode.match import is_extra_words_at_valid_positions
 from licensedcode.match import LicenseMatch
 from licensedcode.match import merge_matches
 from licensedcode.match import reportable_tokens
@@ -1400,9 +1401,11 @@ def test_extra_words_at_wrong_position(self):
         idx = index.LicenseIndex([rule])
 
         query = """
-        Redistribution and amazing use in great source and binary forms are permitted.
+        Redistribution and amazing use in source and binary forms are permitted.
         """
+        # here 'amazing' word are at wrong place
         match = idx.match(query_string=query, _skip_hash_match=True)[0]
+
         assert is_extra_words_position_valid(match) is False
 
     def test_exact_match_without_extra_markers(self):
@@ -1437,6 +1440,140 @@ def test_extra_words_one_at_right_place_and_one_at_not_right_place(self):
         match = idx.match(query_string=query, _skip_hash_match=True)[0]
         assert is_extra_words_position_valid(match) is False
 
+    def test_extra_words_if_one_match_have_extra_words_at_right_place_and_another_match_have_no_extra_words(self):
+        r1_text = "Redistribution and use [[3]] in source and binary forms are permitted."
+        r1 = create_rule_from_text_and_expression(
+            license_expression='extra-words',
+            text=r1_text
+        ) 
+
+        r2_text = "under the MIT license"
+        r2 = create_rule_from_text_and_expression(
+            license_expression='mit',
+            text=r2_text
+        ) 
+
+        idx = index.LicenseIndex([r1,r2])
+
+        query = """
+              Redistribution and use of this software in source and binary forms are permitted.
+              under the MIT license
+              """
+        
+        matches = idx.match(query_string=query, _skip_hash_match=True)
+
+        assert len(matches) == 2
+        assert is_extra_words_at_valid_positions(matches) is True
+
+    def test_extra_words_if_one_match_have_extra_words_at_right_place_but_exceed_limit_and_another_match_have_no_extra_words(self):
+        r1_text = "Redistribution and use [[3]] in source and binary forms are permitted."
+        r1 = create_rule_from_text_and_expression(
+            license_expression='extra-words',
+            text=r1_text
+        ) 
+
+        r2_text = "under the MIT license"
+        r2 = create_rule_from_text_and_expression(
+            license_expression='mit',
+            text=r2_text
+        ) 
+
+        idx = index.LicenseIndex([r1,r2])
+
+        query = """
+              Redistribution and use of this software AAA in source and binary forms are permitted.
+              under the MIT license
+              """
+        
+        matches = idx.match(query_string=query, _skip_hash_match=True)
+
+        assert len(matches) == 2
+
+        # one match have `extra-words` but it exceed the limit here there are
+        # four `extra-words` i.e 'of','this','software','AAA'
+        assert is_extra_words_at_valid_positions(matches) is False
+
+    def test_extra_words_if_all_match_have_no_extra_words(self):
+        r1_text = "Redistribution and use in source and binary forms are permitted."
+        r1 = create_rule_from_text_and_expression(
+            license_expression='extra-words',
+            text=r1_text
+        ) 
+
+        r2_text = "under the MIT license"
+        r2 = create_rule_from_text_and_expression(
+            license_expression='mit',
+            text=r2_text
+        ) 
+
+        idx = index.LicenseIndex([r1,r2])
+
+        query = """
+              Redistribution and use in source and binary forms are permitted.
+              under the MIT license
+              """
+        
+        matches = idx.match(query_string=query, _skip_hash_match=True)
+
+        assert len(matches) == 2
+
+        assert is_extra_words_at_valid_positions(matches) is False
+
+    def test_extra_words_if_one_match_have_extra_words_at_right_place_and_another_match_at_wrong_place(self):
+        r1_text = "Redistribution and use [[3]] in source and binary forms are permitted."
+        r1 = create_rule_from_text_and_expression(
+            license_expression='extra-words',
+            text=r1_text
+        ) 
+
+        r2_text = "Neither the name of [[3]] nor the names of its"
+        r2 = create_rule_from_text_and_expression(
+            license_expression='extra-words2',
+            text=r2_text
+        ) 
+
+        idx = index.LicenseIndex([r1,r2])
+
+        query = """
+              Redistribution and use of this software in source and binary forms are permitted.
+              Neither the name of William Henry James nor the names of Harris its
+              """
+        
+        matches = idx.match(query_string=query, _skip_hash_match=True)
+
+        assert len(matches) == 2
+
+        # one match have `extra-words` at correct place but another match
+        # have `extra-words` at correct place but one words 'Harris' at wrong place
+        # this `is_extra_words_at_valid_positions` return True because one match
+        # have `extra-words` at correct place
+        assert is_extra_words_at_valid_positions(matches) is True
+
+    def test_extra_words_all_match_have_extra_words_at_right_place(self):
+        r1_text = "Redistribution and use [[3]] in source and binary forms are permitted."
+        r1 = create_rule_from_text_and_expression(
+            license_expression='extra-words',
+            text=r1_text
+        ) 
+
+        r2_text = "Neither the name of [[3]] nor the names of its"
+        r2 = create_rule_from_text_and_expression(
+            license_expression='extra-words2',
+            text=r2_text
+        ) 
+
+        idx = index.LicenseIndex([r1,r2])
+
+        query = """
+              Redistribution and use of this software in source and binary forms are permitted.
+              Neither the name of William Henry James nor the names of its
+              """
+        
+        matches = idx.match(query_string=query, _skip_hash_match=True)
+
+        assert len(matches) == 2
+        assert is_extra_words_at_valid_positions(matches) is True
+
 
 class TestLicenseMatchScore(FileBasedTesting):
     test_data_dir = TEST_DATA_DIR