adding support for serialization of bytes taht are not utf8 compatible

Author: seperman

deepdiff/serialization.py

@@ -12,6 +12,7 @@
 import orderly_set  # NOQA
 import collections  # NOQA
 import ipaddress
+import base64
 from copy import deepcopy, copy
 from functools import partial
 from collections.abc import Mapping, KeysView
@@ -607,13 +608,25 @@ def _serialize_tuple(value):
     return value
 
 
+def _serialize_bytes(value):
+    """
+    Serialize bytes to JSON-compatible format.
+    First tries UTF-8 decoding for backward compatibility.
+    Falls back to base64 encoding for binary data.
+    """
+    try:
+        return value.decode('utf-8')
+    except UnicodeDecodeError:
+        return base64.b64encode(value).decode('ascii')
+
+
 JSON_CONVERTOR = {
     decimal.Decimal: _serialize_decimal,
     SetOrdered: list,
     orderly_set.StableSetEq: list,
     set: list,
     type: lambda x: x.__name__,
-    bytes: lambda x: x.decode('utf-8'),
+    bytes: _serialize_bytes,
     datetime.datetime: lambda x: x.isoformat(),
     uuid.UUID: lambda x: str(x),
     np_float32: float,

tests/test_serialization.py

@@ -6,6 +6,8 @@
 import pytest
 import datetime
 import numpy as np
+import hashlib
+import base64
 from typing import NamedTuple, Optional
 from pickle import UnpicklingError
 from decimal import Decimal
@@ -455,3 +457,113 @@ def test_dict_keys(self):
         dic = {"foo": "bar", "apple": "too sweet"}
         serialized = json_dumps(dic.keys())
         assert '["foo","apple"]' == serialized
+
+    def test_non_utf8_bytes_serialization(self):
+        """Test that non-UTF-8 bytes are properly base64 encoded"""
+        # Create binary data that cannot be decoded as UTF-8
+        binary_data = b'\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f'
+        
+        # Verify it's not UTF-8 decodable
+        with pytest.raises(UnicodeDecodeError):
+            binary_data.decode('utf-8')
+        
+        # Test serialization
+        test_data = {"binary": binary_data}
+        serialized = json_dumps(test_data)
+        
+        # Should contain base64 encoded data
+        expected_b64 = base64.b64encode(binary_data).decode('ascii')
+        assert expected_b64 in serialized
+        
+        # Should be deserializable
+        deserialized = json_loads(serialized)
+        assert deserialized == {"binary": expected_b64}
+
+    def test_hash_bytes_serialization(self):
+        """Test serialization of hash-like binary data (blake3, sha256, etc.)"""
+        # Generate various hash-like byte sequences
+        test_cases = [
+            hashlib.md5(b"test").digest(),
+            hashlib.sha1(b"test").digest(),
+            hashlib.sha256(b"test").digest(),
+            hashlib.sha512(b"test").digest()[:16],  # Truncated
+            b'\xff\xfe\xfd\xfc' * 8,  # Artificial binary pattern
+        ]
+        
+        for i, hash_bytes in enumerate(test_cases):
+            test_data = {"hash": hash_bytes}
+            
+            # Should not raise UnicodeDecodeError
+            serialized = json_dumps(test_data)
+            assert serialized  # Should produce valid JSON
+            
+            # Should contain base64 if not UTF-8 decodable, or string if UTF-8 decodable
+            try:
+                utf8_decoded = hash_bytes.decode('utf-8')
+                # If UTF-8 decodable, should be in JSON as string
+                assert utf8_decoded in serialized
+            except UnicodeDecodeError:
+                # If not UTF-8 decodable, should be base64 encoded
+                expected_b64 = base64.b64encode(hash_bytes).decode('ascii')
+                assert expected_b64 in serialized
+
+    def test_mixed_utf8_and_binary_bytes(self):
+        """Test data structure with both UTF-8 decodable and binary bytes"""
+        test_data = {
+            "utf8_text": b"hello world",  # UTF-8 decodable
+            "binary_hash": hashlib.sha256(b"secret").digest(),  # Binary
+            "empty_bytes": b"",  # Edge case
+            "utf8_unicode": "café".encode('utf-8'),  # UTF-8 with unicode
+            "non_utf8_byte": b"\xff\xfe\xfd",  # Non-UTF-8 bytes
+        }
+        
+        # Should serialize without errors
+        serialized = json_dumps(test_data)
+        deserialized = json_loads(serialized)
+        
+        # UTF-8 decodable bytes should remain as strings
+        assert "hello world" in serialized
+        assert deserialized["utf8_text"] == "hello world"
+        
+        # Unicode UTF-8 should work
+        assert deserialized["utf8_unicode"] == "café"
+        
+        # Binary data should be base64 encoded
+        expected_hash_b64 = base64.b64encode(test_data["binary_hash"]).decode('ascii')
+        assert expected_hash_b64 in serialized
+        assert deserialized["binary_hash"] == expected_hash_b64
+        
+        # Empty bytes should be empty string
+        assert deserialized["empty_bytes"] == ""
+        
+        # Non-UTF-8 bytes should be base64 encoded
+        expected_non_utf8_b64 = base64.b64encode(test_data["non_utf8_byte"]).decode('ascii')
+        assert expected_non_utf8_b64 in serialized
+        assert deserialized["non_utf8_byte"] == expected_non_utf8_b64
+
+    def test_bytes_in_deepdiff_serialization(self):
+        """Test that bytes work correctly in DeepDiff JSON serialization"""
+        t1 = {
+            "text": b"hello",
+            "hash": hashlib.sha256(b"data1").digest(),
+        }
+        t2 = {
+            "text": b"world", 
+            "hash": hashlib.sha256(b"data2").digest(),
+        }
+        
+        diff = DeepDiff(t1, t2)
+        
+        # Should serialize without errors
+        json_output = diff.to_json()
+        assert json_output
+        
+        # Should contain both UTF-8 decoded strings and base64 encoded hashes
+        assert "hello" in json_output
+        assert "world" in json_output
+        
+        # Hash values should be base64 encoded
+        expected_hash1 = base64.b64encode(t1["hash"]).decode('ascii') 
+        expected_hash2 = base64.b64encode(t2["hash"]).decode('ascii')
+        assert expected_hash1 in json_output
+        assert expected_hash2 in json_output