Adds fuzzy matching based on edit distance

Author: ESultanik

src/vendetect/_cli.py

@@ -17,7 +17,7 @@
 from rich.text import Text
 
 from .detector import Detection, Status, VenDetector, get_lexer_for_filename
-from .diffing import CollapsedDiffLine, Differ, DiffLineStatus, Document
+from .diffing import CollapsedDiffLine, Differ, DiffLineStatus, Document, normalized_edit_distance
 from .errors import VendetectError
 from .repo import File, Repository
 
@@ -233,7 +233,13 @@ def read_file_content(file: File) -> str:
                         if diff_line.status == DiffLineStatus.COPIED:
                             status_col = Text("←", style="red reverse bold")
                         else:
-                            status_col = Text("✓", style="green reverse")
+                            # calculate the normalized edit distance
+                            if diff_line.left is not None and diff_line.right is not None and \
+                                    normalized_edit_distance(diff_line.left, diff_line.right) < 0.75:
+                                # the lines are at least 25% similar!
+                                status_col = Text("↜", style="yellow reverse")
+                            else:
+                                status_col = Text("✓", style="green reverse")
                         if diff_line.left is None:
                             left: ConsoleRenderable = Text("")
                         else:

src/vendetect/detector.py

@@ -63,7 +63,7 @@ def __repr__(self) -> str:
 
     def byte_offset_slice_to_lines_slice(self, to_convert: Slice) -> Slice:
         from_line = self.file.get_line(to_convert.from_index, rounding=Rounding.DOWN)
-        to_line = self.file.get_line(to_convert.to_index, rounding=Rounding.UP, min_line=from_line+1)
+        to_line = self.file.get_line(to_convert.to_index, rounding=Rounding.UP, min_line=from_line + 1)
         return Slice(from_line, to_line)
 
     def to_str(self, max_slices: int = -1) -> str:

src/vendetect/diffing.py

@@ -200,3 +200,20 @@ def diff_from_offsets(
                 collapse_identical_lines_threshold=collapse_identical_lines_threshold,
             )
         )
+
+
+def edit_distance(s1: str, s2: str) -> int:
+    """Calculates the minimum number of edits to convert s1 into s2"""
+    # HACK: insert a newline after every character, then use the Myers Diff algorithm since it's built into Python;
+    #       the result should be the same as Levenshtein distance!
+    newline = "\n"
+    s1 = f"{newline.join(s1)}\n"
+    s2 = f"{newline.join(s2)}\n"
+    return sum(1 for diff_line in ndiff(s1, s2) if diff_line[:2] in ("- ", "+ "))
+
+
+def normalized_edit_distance(s1: str, s2: str) -> float:
+    if s1 == s2:
+        return 0
+    ed = edit_distance(s1, s2)
+    return ed / max(len(s1), len(s2))