improvements

Author: coyotte508

packages/README.md

@@ -0,0 +1,25 @@
+JS and WASM implementations of https://github.com/BLAKE3-team/BLAKE3
+
+Using [AssemblyScript](https://www.assemblyscript.org/) to generate a lean WASM.
+
+## Usage
+
+```javascript
+import { blake3, blake3Hex, createHasher, update, finalize } from '@huggingface/gearhash-wasm';
+
+// Create a Uint8Array of data to search through
+const data = new Uint8Array(1_000_000); // Example: 1MB of data
+// ... fill data with your content ...
+
+const hashUint8 = blake3(data);
+const hashHex = blake3Hex(data);
+
+// Or streaming fashion
+const hasher = createHasher();
+
+for (const chunk of dataSource) {
+  hasher.update(chunk);
+}
+
+const hash = hasher.finalize();
+```

packages/blake3-wasm/README.md

@@ -371,3 +371,17 @@ export function blake3Hex(input: Uint8Array): string {
 	}
 	return hex.join("");
 }
+
+export function createHasher(): Blake3Hasher {
+	return new Blake3Hasher();
+}
+
+export function update(hasher: Blake3Hasher, input: Uint8Array): void {
+	hasher.update(input);
+}
+
+export function finalize(hasher: Blake3Hasher): Uint8Array {
+	const output = new Uint8Array(32);
+	hasher.finalize(output);
+	return output;
+}

packages/blake3-wasm/assembly/blake3.ts

@@ -1,4 +1,4 @@
 {
-	"extends": "../node_modules/.pnpm/[email protected].37/node_modules/assemblyscript/std/assembly.json",
+	"extends": "../node_modules/.pnpm/[email protected].36/node_modules/assemblyscript/std/assembly.json",
 	"include": ["./**/*.ts"]
 }

packages/blake3-wasm/assembly/tsconfig.json

@@ -28,6 +28,6 @@
 		}
 	},
 	"devDependencies": {
-		"assemblyscript": "^0.27.36"
+		"assemblyscript": "0.27.36"
 	}
 }

packages/blake3-wasm/package.json

@@ -11,15 +11,14 @@ import { nextMatch } from '@huggingface/gearhash-wasm';
 const data = new Uint8Array(1000000); // Example: 1MB of data
 // ... fill data with your content ...
 
-// Search for a pattern with a specific mask
-const mask = 0x0000d90003530000n; // Example mask as a BigInt
+const mask = 0x0000d90003530000n; // Example mask as a BigInt, more 0s => bigger chunks
 const match = nextMatch(data, mask);
 const allMatches = nextMatches(data, mask).matches;
 ```
 
 The `nextMatch` function takes two parameters:
 - `data`: A Uint8Array containing the data to search through
-- `mask`: A BigInt representing the pattern mask to search for
+- `mask`: A BigInt, the bigger it is the bigger the chunks are
 
 The function returns an object with the `position` (i32) and `hash` (u64) properties
 

packages/blake3-wasm/pnpm-lock.yaml

@@ -28,6 +28,6 @@
 		}
 	},
 	"devDependencies": {
-		"assemblyscript": "^0.27.36"
+		"assemblyscript": "0.27.36"
 	}
 }

packages/gearhash-wasm/README.md

@@ -0,0 +1,27 @@
+JS and WASM implementations of https://github.com/huggingface/xet-core/blob/main/deduplication/src/chunking.rs
+
+Using [AssemblyScript](https://www.assemblyscript.org/) to generate a lean WASM.
+
+## Usage
+
+```javascript
+import { createChunker, getChunks, nextBlock, finalize } from '@huggingface/xetchunk-wasm';
+
+const TARGET_CHUNK_SIZE = Math.pow(2, 12);
+
+// Create a Uint8Array of data to search through
+const data = new Uint8Array(1000000); // Example: 1MB of data
+// ... fill data with your content ...
+
+const chunks = getChunks(data, TARGET_CHUNK_SIZE);
+
+// Alternative, in case your data is streaming
+const chunker = createChunker(TARGET_CHUNK_SIZE);
+
+for await (const data of source) {
+  const chunks = nextBlock(chunker, data);
+  console.log(chunks);
+}
+
+console.log("last chunk", finalize(chunker));
+```