From 898ece798e95776b791243480670f78d5f945b3e Mon Sep 17 00:00:00 2001
From: Jack Zhang <32371937+jackzhxng@users.noreply.github.com>
Date: Mon, 10 Feb 2025 15:30:34 -0800
Subject: [PATCH] Add third-party library

---
 .gitmodules                          | 3 +++
 extension/llm/third-party/tokenizers | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 extension/llm/third-party/tokenizers

diff --git a/.gitmodules b/.gitmodules
index f7da7e771fb..f4f363a66c8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -70,3 +70,6 @@
 [submodule "third-party/pocketfft"]
 	path = third-party/pocketfft
 	url = https://github.com/mreineck/pocketfft
+[submodule "extension/llm/third-party/tokenizers"]
+	path = extension/llm/third-party/tokenizers
+	url = https://github.com/pytorch-labs/tokenizers.git
diff --git a/extension/llm/third-party/tokenizers b/extension/llm/third-party/tokenizers
new file mode 160000
index 00000000000..f2fc3d6dfe8
--- /dev/null
+++ b/extension/llm/third-party/tokenizers
@@ -0,0 +1 @@
+Subproject commit f2fc3d6dfe8c5264710a7dcdd59d9bfe7e37f4b7