fix(encoding): Encode document in UTF-8

Author: Walz

pygitguardian/models.py

@@ -56,7 +56,7 @@ def validate_document(self, document: str) -> None:
         """
         validate that document is smaller than scan limit
         """
-        encoded = document.encode("utf-8")
+        encoded = document.encode("utf-8", errors="replace")
         if len(encoded) > DOCUMENT_SIZE_THRESHOLD_BYTES:
             raise ValidationError(
                 "file exceeds the maximum allowed size of {}B".format(
@@ -71,6 +71,15 @@ def replace_0_bytes(self, in_data: Dict[str, Any], **kwargs: Any) -> Dict[str, A
         in_data["document"] = doc.replace("\0", "\uFFFD")
         return in_data
 
+    @post_load
+    def force_utf_8_encoding(
+        self, in_data: Dict[str, Any], **kwargs: Any
+    ) -> Dict[str, Any]:
+        doc = in_data["document"]
+        # Force UTF-8 and substitute ? for encoding errors
+        in_data["document"] = doc.encode("utf-8", errors="replace").decode("utf-8")
+        return in_data
+
 
 class Document(Base):
     """

tests/test_models.py

@@ -40,6 +40,12 @@ def test_document_handle_0_bytes(self):
         )
         assert document["document"] == "hello\uFFFDworld"
 
+    def test_document_handle_surrogates(self):
+        document = Document.SCHEMA.load(
+            {"filename": "name", "document": "hello\udbdeworld"}
+        )
+        assert document["document"] == "hello?world", document
+
     @pytest.mark.parametrize(
         "schema_klass, expected_klass, instance_data",
         [