perf: use blake3 instead of sha256

Signed-off-by: thiswillbeyourgithub <[email protected]>

Author: thiswillbeyourgithub

setup.py

@@ -165,6 +165,7 @@ def run(self):
         "uuid6 >= 2025.0.1",  # for time sortable timestamp
         "PersistDict >= 0.2.14",  # by me, like a dict but an LMDB database, to fix langchain's caches
         "nltk>=3.9.2",  # needed for punkt_tab download in post-install
+        "blake3>=1.0.8",  # faster than sha256
         "pandas >= 2.3.3",
         # some loaders are included by default:
         "playwright >= 1.45.0",  # for online_media and urls

tests/test_parsing.py

@@ -1,6 +1,6 @@
 import os
 import tempfile
-import hashlib
+from blake3 import blake3
 from pathlib import Path
 
 import pytest
@@ -134,9 +134,9 @@ def test_parse_docx():
         f.write(response.content)
 
     # Verify SHA512 checksum
-    expected_hash = "64b73b409688cc5b5675c07d9df4b83d353fa85026a9d686d6725e50f388930e1d57c56cc6cfebd5f2cecc06d7ef89ae7495bd5411ca0eac4b0df63a7d6c82dc"
+    expected_hash = "e69f84a2b494c767255d5d6934e3d12648fccb7dbc06e1cd47cad1b1c6efc902"
     with open(tmp_path, "rb") as f:
-        file_hash = hashlib.sha512(f.read()).hexdigest()
+        file_hash = blake3(f.read()).hexdigest()
     assert file_hash == expected_hash, (
         f"File hash {file_hash} does not match expected {expected_hash}"
     )

wdoc/utils/misc.py

@@ -2,7 +2,7 @@
 Miscellanous functions etc.
 """
 
-import hashlib
+from blake3 import blake3
 import re
 import inspect
 import json
@@ -359,7 +359,7 @@ def wrapper(*args, **kwargs):
 def hasher(text: str) -> str:
     """used to hash the text contant of each doc to cache the splitting and
     embeddings"""
-    return hashlib.sha256(text.encode()).hexdigest()[:20]
+    return blake3(text.encode()).hexdigest()[:20]
 
 
 def file_hasher(doc: dict) -> str:
@@ -396,7 +396,7 @@ def file_hasher(doc: dict) -> str:
 @hashdoc_cache.cache
 def _file_hasher(abs_path: str, stats: List[Union[int, float]]) -> str:
     with open(abs_path, "rb") as f:
-        return hashlib.sha256(f.read()).hexdigest()[:20]
+        return blake3(f.read()).hexdigest()[:20]
 
 
 def html_to_text(html: str, remove_image: bool = False) -> str:
@@ -614,7 +614,7 @@ def __post_init__(self):
                     with open(
                         Path(self.model).resolve().absolute().__str__(), "rb"
                     ) as f:
-                        h = hashlib.sha256(f.read() + str(self.model)).hexdigest()[:15]
+                        h = blake3(f.read() + str(self.model)).hexdigest()[:15]
                     self.sanitized = Path(self.model).name + "_" + h
             except Exception:
                 pass