DOP-3068: Properly handle rst source files with invalid UTF-8

Author: i80and

snooty/diagnostics.py

@@ -144,16 +144,15 @@ class GitMergeConflictArtifactFound(Diagnostic):
 
     def __init__(
         self,
-        path: Path,
+        path: Optional[Path],
         start: Union[int, Tuple[int, int]],
         end: Union[None, int, Tuple[int, int]] = None,
     ) -> None:
         super().__init__(
-            f"Git merge conflict artifact found in {str(path)} on line {str(start)}",
+            f"Git merge conflict artifact found{' in ' + str(path) if path else ''} on line {str(start)}",
             start,
             end,
         )
-        self.path = path
 
 
 class DocUtilsParseError(Diagnostic):
@@ -347,12 +346,14 @@ class CannotOpenFile(Diagnostic):
 
     def __init__(
         self,
-        path: PurePath,
+        path: Optional[PurePath],
         reason: str,
         start: Union[int, Tuple[int, int]],
         end: Union[None, int, Tuple[int, int]] = None,
     ) -> None:
-        super().__init__(f"Error opening {str(path)}: {reason}", start, end)
+        super().__init__(
+            f"Error opening{' ' + str(path) if path else ''}: {reason}", start, end
+        )
         self.path = path
         self.reason = reason
 

snooty/test_parser.py

@@ -21,9 +21,15 @@
     UnknownTabID,
     UnknownTabset,
 )
+from .n import FileId
 from .parser import InlineJSONVisitor, JSONVisitor
 from .types import ProjectConfig
-from .util_test import ast_to_testing_string, check_ast_testing_string, parse_rst
+from .util_test import (
+    ast_to_testing_string,
+    check_ast_testing_string,
+    make_test,
+    parse_rst,
+)
 
 ROOT_PATH = Path("test_data")
 
@@ -3337,3 +3343,18 @@ def test_field_list_in_list() -> None:
 
     page.finish(diagnostics)
     assert len(diagnostics) == 2
+
+
+def test_invalid_utf8() -> None:
+    with make_test(
+        {
+            Path(
+                "source/index.txt"
+            ): b"""test line 1
+test line 2
+here is an invalid character sequence\x80 oh noooo
+"""
+        }
+    ) as result:
+        diagnostics = result.diagnostics[FileId("index.txt")]
+        assert [(type(d), d.start[0]) for d in diagnostics] == [(CannotOpenFile, 2)]

snooty/types.py

@@ -11,6 +11,7 @@
 
 from . import n, specparser
 from .diagnostics import (
+    CannotOpenFile,
     ConstantNotDeclared,
     Diagnostic,
     GitMergeConflictArtifactFound,
@@ -203,15 +204,19 @@ def read(
         self, path: Path, text: Optional[str] = None
     ) -> Tuple[str, List[Diagnostic]]:
         if text is None:
-            text = path.read_text(encoding="utf-8")
+            try:
+                text = path.read_text(encoding="utf-8")
+            except UnicodeDecodeError as err:
+                error_line = path.read_bytes()[: err.start].count(b"\n")
+                return ("", [CannotOpenFile(None, str(err), error_line)])
 
         text, diagnostics = self.substitute(text)
         match_found = PAT_GIT_MARKER.finditer(text)
 
         if match_found:
             for match in match_found:
                 lineno = text.count("\n", 0, match.start())
-                diagnostics.append(GitMergeConflictArtifactFound(path, lineno))
+                diagnostics.append(GitMergeConflictArtifactFound(None, lineno))
 
         return (text, diagnostics)
 

snooty/util_test.py

@@ -7,7 +7,7 @@
 import xml.etree.ElementTree as ET
 from collections import defaultdict
 from pathlib import Path, PurePath
-from typing import Any, Dict, Iterator, List, Optional, Tuple
+from typing import Any, AnyStr, Dict, Iterator, List, Optional, Tuple
 from xml.sax.saxutils import escape
 
 from . import n, rstparser
@@ -204,7 +204,7 @@ def flush(self) -> None:
 
 @contextlib.contextmanager
 def make_test(
-    files: Dict[PurePath, str], name: str = ""
+    files: Dict[PurePath, AnyStr], name: str = ""
 ) -> Iterator[BackendTestResults]:
     """Create a temporary test project with the given files."""
     need_to_make_snooty_toml = Path("snooty.toml") not in files
@@ -222,7 +222,10 @@ def make_test(
             file_path = root.joinpath(filename)
             if file_path.parent != root:
                 os.makedirs(file_path.parent, exist_ok=True)
-            file_path.write_text(file_text)
+            if isinstance(file_text, str):
+                file_path.write_text(file_text)
+            else:
+                file_path.write_bytes(file_text)
 
         backend = BackendTestResults()