Vendor Hasher class from datasets library in order to remove larger dependency (#8621)

* Vendor Hasher class from datasets library in order to remove larger dependency

* Run ruff - add change notice for Apache

Author: BenMcH

dspy/clients/utils_finetune.py

@@ -66,7 +66,7 @@ def write_lines(file_path, data):
 def save_data(
     data: list[dict[str, Any]],
 ) -> str:
-    from datasets.fingerprint import Hasher
+    from dspy.utils.hasher import Hasher
 
     # Assign a unique name to the file based on the data hash
     hash = Hasher.hash(data)

dspy/teleprompt/bootstrap.py

@@ -241,7 +241,7 @@ def _bootstrap_one_example(self, example, round_idx=0):
                 # If there are multiple traces for the same predictor in the sample example,
                 # sample 50/50 from the first N-1 traces or the last trace.
                 if len(demos) > 1:
-                    from datasets.fingerprint import Hasher
+                    from dspy.utils.hasher import Hasher
 
                     rng = random.Random(Hasher.hash(tuple(demos)))
                     demos = [rng.choice(demos[:-1]) if rng.random() < 0.5 else demos[-1]]

dspy/utils/hasher.py

@@ -0,0 +1,43 @@
+from pickle import dumps
+from typing import Any
+
+import xxhash
+
+"""
+The following class was pulled from the `datasets` package from Hugging Face.
+The reason for vendoring this code is to avoid a hard dependency on `datasets`,
+which is a large package that is not needed for the majority of use cases.
+
+License: Apache License 2.0
+Author: Hugging Face Inc.
+URL: https://github.com/huggingface/datasets/blob/fa73ab472eecf9136a3daf7a0fbff16a3dffa7a6/src/datasets/fingerprint.py#L170
+Changes: 2025-08-10 - Ran ruff to format the code to DSPy styles.
+"""
+class Hasher:
+    """Hasher that accepts python objects as inputs."""
+
+    dispatch: dict = {}
+
+    def __init__(self):
+        self.m = xxhash.xxh64()
+
+    @classmethod
+    def hash_bytes(cls, value: bytes | list[bytes]) -> str:
+        value = [value] if isinstance(value, bytes) else value
+        m = xxhash.xxh64()
+        for x in value:
+            m.update(x)
+        return m.hexdigest()
+
+    @classmethod
+    def hash(cls, value: Any) -> str:
+        return cls.hash_bytes(dumps(value))
+
+    def update(self, value: Any) -> None:
+        header_for_update = f"=={type(value)}=="
+        value_for_update = self.hash(value)
+        self.m.update(header_for_update.encode("utf8"))
+        self.m.update(value_for_update.encode("utf-8"))
+
+    def hexdigest(self) -> str:
+        return self.m.hexdigest()

pyproject.toml

@@ -24,9 +24,7 @@ dependencies = [
     "backoff>=2.2",
     "joblib~=1.3",
     "openai>=0.28.1",
-    "datasets>=2.14.6", # needed for Bootstrap's Hasher
     "regex>=2023.10.3",
-    "datasets>=2.14.6", # needed for Bootstrap's Hasher
     "ujson>=5.8.0",
     "tqdm>=4.66.1",
     "requests>=2.31.0",
@@ -43,6 +41,7 @@ dependencies = [
     "cloudpickle>=3.0.0",
     "rich>=13.7.1",
     "numpy>=1.26.0",
+    "xxhash>=3.5.0",
 ]
 
 [project.optional-dependencies]