Add third-party library

jackzhxng · jackzhxng · commit 898ece798e95 · 2025-02-10T15:30:34.000-08:00
diff --git a/.gitmodules b/.gitmodules
@@ -70,3 +70,6 @@
 [submodule "third-party/pocketfft"]
 	path = third-party/pocketfft
 	url = https://github.com/mreineck/pocketfft
+[submodule "extension/llm/third-party/tokenizers"]
+	path = extension/llm/third-party/tokenizers
+	url = https://github.com/pytorch-labs/tokenizers.git
diff --git a/extension/llm/third-party/tokenizers b/extension/llm/third-party/tokenizers
@@ -0,0 +1 @@
+Subproject commit f2fc3d6dfe8c5264710a7dcdd59d9bfe7e37f4b7
