improve and add more test for is_extra_words_position_valid

Signed-off-by: Alok Kumar <[email protected]>

Author: alok1304

src/licensedcode/detection.py

@@ -1156,17 +1156,6 @@ def has_extra_words(license_matches):
     )
 
 
-def has_extra_words_spans(license_matches):
-    """
-    Return True if all of the matches rules in ``license_matches`` (a list of LicenseMatch)
-    has `extra_phrase` marker and also have matcher `3-seq`.
-    """
-    return all(
-        match.matcher == '3-seq' and match.rule.extra_phrase_spans
-        for match in license_matches
-    )
-
-
 def has_low_rule_relevance(license_matches):
     """
     Return True if all on the matches in ``license_matches`` List of LicenseMatch

src/licensedcode/match.py

@@ -1106,6 +1106,9 @@ def is_extra_words_position_valid(match):
     # Count of extra phrase markers   
     extra_phrase_count = 0
 
+    rule_index = 0
+    matched_index = 0
+
     for span, allowed_extra_word in extra_phrase_spans:
         rule_index = span.start
 
@@ -1128,6 +1131,17 @@ def is_extra_words_position_valid(match):
 
         extra_phrase_count += 1
 
+    rule_index+=1    
+
+    # check if any `extra-words` is present and return False because this `extra-words` are not at marked place
+    while (matched_index < len(matched_tokens) and
+        matched_tokens[matched_index] == rule_tokens[rule_index]):        
+        matched_index+=1
+        rule_index+=1
+
+    if matched_index != len(matched_tokens):
+        return False    
+    
     return True
 
 

tests/licensedcode/test_detection_validate.py

@@ -18,7 +18,6 @@
 from licensedcode import cache
 from licensedcode import models
 from licensedcode.detection import is_correct_detection
-from licensedcode.detection import has_extra_words_spans
 from licensedcode.models import licenses_data_dir
 from licensedcode.models import rules_data_dir
 from licensedcode.models import License
@@ -100,7 +99,7 @@ def check_rule_or_license_can_be_detected_exactly(licensish):
         assert results == expected
 
     icm = is_correct_detection(matches)
-    if not icm and not has_extra_words_spans(matches):
+    if not icm:
         expected.append(f'file://{licensish.rule_file()}')
         assert results == expected
 

tests/licensedcode/test_match.py

@@ -1421,6 +1421,22 @@ def test_exact_match_without_extra_markers(self):
         match = idx.match(query_string=query, _skip_hash_match=True)[0]
         assert is_extra_words_position_valid(match) is False
 
+    def test_extra_words_one_at_right_place_and_one_at_not_right_place(self):
+        rule_text = """
+        Redistribution and use [[3]] in source and binary forms are permitted.
+        """
+        rule = create_rule_from_text_and_expression(
+            license_expression='extra-words',
+            text=rule_text
+        )
+        idx = index.LicenseIndex([rule])
+
+        query = """
+        Redistribution and use of this software in source and binary extra-words forms are permitted.
+        """
+        match = idx.match(query_string=query, _skip_hash_match=True)[0]
+        assert is_extra_words_position_valid(match) is False
+
 
 class TestLicenseMatchScore(FileBasedTesting):
     test_data_dir = TEST_DATA_DIR