Add tokenizers package (#1675)

`tokenizers` is used for tokenization of strings. While we do currently import titoken as well, we want to use `tokenizers` to
count tokens the same way Seer does in Sentry for stacktrace token counts. We could consider converting usages of titoken
to `tokenizers` to maintain a dependency on only one tokenization package, but for now we will have them both.

Author: yuvmen

packages.ini

@@ -573,6 +573,7 @@ validate_incorrect_missing_deps = psycopg2-binary
 [filelock==3.15.3]
 [filelock==3.16.1]
 [filelock==3.18.0]
+[filelock==3.19.1]
 
 [flake8==4.0.1]
 [flake8==5.0.2]
@@ -614,6 +615,8 @@ validate_incorrect_missing_deps = psycopg2-binary
 [frozenlist==1.4.1]
 [frozenlist==1.5.0]
 
+[fsspec==2025.9.0]
+
 [ghp-import==2.1.0]
 
 [gitdb==4.0.5]
@@ -825,6 +828,8 @@ python_versions = <3.13
 
 [hera==5.25.1]
 
+[hf-xet==1.1.10]
+
 [hiredis==0.3.1]
 python_versions = <3.12
 [hiredis==2.0.0]
@@ -856,6 +861,8 @@ python_versions = <3.12
 [httpx==0.27.2]
 [httpx==0.28.1]
 
+[huggingface-hub==0.35.3]
+
 [hyperframe==6.0.1]
 [hyperframe==6.1.0]
 
@@ -1675,6 +1682,9 @@ python_versions = <3.13
 [pyyaml==6.0.2]
 apt_requires = libyaml-dev
 brew_requires = libyaml
+[pyyaml==6.0.3]
+apt_requires = libyaml-dev
+brew_requires = libyaml
 
 [pyyaml-env-tag==1.1]
 
@@ -2970,6 +2980,8 @@ python_versions = <3.13
 [tokenize-rt==5.2.0]
 [tokenize-rt==6.1.0]
 
+[tokenizers==0.22.1]
+
 [toml==0.10.2]
 
 [tomli==2.0.1]