From 346d64e2fdf965ed80a2487372fadbfdc0fc0147 Mon Sep 17 00:00:00 2001
From: Yuval Mandelboum <yuval.mandelboum@sentry.io>
Date: Mon, 29 Sep 2025 12:53:51 -0700
Subject: [PATCH] Add `transformers` package

Transfomers is used for tokenization of strings. While we do currently import `titoken` as well, we want
to use `transformers` to count tokens the same way Seer does in Sentry for stacktrace token counts.
 We could consider converting usages of `titoken` to `transformers` to maintain a dependency on only one
 tokenization package, but for now we will have them both.
---
 packages.ini | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/packages.ini b/packages.ini
index 17398f0c..ef833640 100644
--- a/packages.ini
+++ b/packages.ini
@@ -573,6 +573,7 @@ validate_incorrect_missing_deps = psycopg2-binary
 [filelock==3.15.3]
 [filelock==3.16.1]
 [filelock==3.18.0]
+[filelock==3.19.1]
 
 [flake8==4.0.1]
 [flake8==5.0.2]
@@ -614,6 +615,8 @@ validate_incorrect_missing_deps = psycopg2-binary
 [frozenlist==1.4.1]
 [frozenlist==1.5.0]
 
+[fsspec==2025.9.0]
+
 [ghp-import==2.1.0]
 
 [gitdb==4.0.5]
@@ -825,6 +828,8 @@ python_versions = <3.13
 
 [hera==5.25.1]
 
+[hf-xet==1.1.10]
+
 [hiredis==0.3.1]
 python_versions = <3.12
 [hiredis==2.0.0]
@@ -856,6 +861,8 @@ python_versions = <3.12
 [httpx==0.27.2]
 [httpx==0.28.1]
 
+[huggingface-hub==0.35.3]
+
 [hyperframe==6.0.1]
 [hyperframe==6.1.0]
 
@@ -1148,6 +1155,8 @@ python_versions = <3.13
 [nodeenv==1.8.0]
 [nodeenv==1.9.1]
 
+[numpy==2.3.3]
+
 [oauthlib==3.1.0]
 [oauthlib==3.2.0]
 [oauthlib==3.2.1]
@@ -1675,6 +1684,9 @@ python_versions = <3.13
 [pyyaml==6.0.2]
 apt_requires = libyaml-dev
 brew_requires = libyaml
+[pyyaml==6.0.3]
+apt_requires = libyaml-dev
+brew_requires = libyaml
 
 [pyyaml-env-tag==1.1]
 
@@ -1704,6 +1716,7 @@ python_versions = <3.12
 [regex==2022.8.17]
 [regex==2022.9.13]
 [regex==2023.12.25]
+[regex==2025.9.18]
 
 [reportlab==3.6.13]
 python_versions = <3.12
@@ -1775,6 +1788,8 @@ python_versions = <3.13
 [s3transfer==0.10.0]
 [s3transfer==0.10.1]
 
+[safetensors==0.6.2]
+
 [selenium==4.3.0]
 [selenium==4.4.3]
 [selenium==4.11.2]
@@ -2970,6 +2985,8 @@ python_versions = <3.13
 [tokenize-rt==5.2.0]
 [tokenize-rt==6.1.0]
 
+[tokenizers==0.22.1]
+
 [toml==0.10.2]
 
 [tomli==2.0.1]
@@ -2990,6 +3007,8 @@ python_versions = <3.13
 [tqdm==4.66.4]
 [tqdm==4.67.1]
 
+[transformers==4.56.2]
+
 [trio==0.21.0]
 [trio==0.22.2]
 [trio==0.23.1]