diff --git a/ModelForge/app.py b/ModelForge/app.py index 8336972..5d9ae26 100644 --- a/ModelForge/app.py +++ b/ModelForge/app.py @@ -23,9 +23,10 @@ ## Static files frontend_dir = os.path.join(os.path.dirname(__file__), "./Frontend/build") app_name = "ModelForge" -origins = [ - "http://localhost:8000", -] + +# CORS origins - configurable via environment variable +cors_origins_env = os.getenv("CORS_ORIGINS", "http://localhost:8000") +origins = [origin.strip() for origin in cors_origins_env.split(",")] app.add_middleware( CORSMiddleware, diff --git a/ModelForge/globals/globals.py b/ModelForge/globals/globals.py index 10cb855..e4e9b3f 100644 --- a/ModelForge/globals/globals.py +++ b/ModelForge/globals/globals.py @@ -7,6 +7,7 @@ class GlobalSettings: _instance = None + _initialized = False file_manager = None hardware_detector = None settings_builder = None @@ -23,19 +24,27 @@ def __new__(cls): return cls._instance def __init__(self): - self.file_manager = FileManager() - self.hardware_detector = HardwareDetector() - self.settings_builder = SettingsBuilder(None, None, None) - self.settings_cache = {} - self.finetuning_status = {"status": "idle", "progress": 0, "message": ""} - self.datasets_dir = self.file_manager.return_default_dirs()["datasets"] - self.model_path = self.file_manager.return_default_dirs()["models"] - self.db_manager = DatabaseManager(db_path=os.path.join(self.file_manager.return_default_dirs()["database"], "modelforge.sqlite")) - self.app_name = "ModelForge" + # Only initialize once to maintain singleton behavior + if not GlobalSettings._initialized: + self.file_manager = FileManager() + self.hardware_detector = HardwareDetector() + self.settings_builder = SettingsBuilder(None, None, None) + self.settings_cache = {} + # NOTE: finetuning_status is accessed from multiple places (callback, background task) + # without locking. Python's GIL provides basic protection, but be cautious with + # complex operations. Consider using threading.Lock if race conditions occur. + self.finetuning_status = {"status": "idle", "progress": 0, "message": ""} + self.datasets_dir = self.file_manager.return_default_dirs()["datasets"] + self.model_path = self.file_manager.return_default_dirs()["models"] + self.db_manager = DatabaseManager(db_path=os.path.join(self.file_manager.return_default_dirs()["database"], "modelforge.sqlite")) + self.app_name = "ModelForge" + GlobalSettings._initialized = True @classmethod def get_instance(cls): - return cls.__new__(cls) + if cls._instance is None: + cls._instance = cls() + return cls._instance def clear_settings_cache(self): self.settings_cache.clear() diff --git a/ModelForge/routers/finetuning_router.py b/ModelForge/routers/finetuning_router.py index e02c7c4..4161b2f 100644 --- a/ModelForge/routers/finetuning_router.py +++ b/ModelForge/routers/finetuning_router.py @@ -1,13 +1,14 @@ from datetime import datetime import json import os +import shutil import traceback import uuid from fastapi import APIRouter, HTTPException, UploadFile, File, Form, BackgroundTasks from fastapi import Request from starlette.responses import JSONResponse -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, model_validator from ..utilities.finetuning.CausalLLMTuner import CausalLLMFinetuner from ..utilities.finetuning.QuestionAnsweringTuner import QuestionAnsweringTuner @@ -20,13 +21,17 @@ prefix="/finetune", ) +# Valid task types +VALID_TASKS = ["text-generation", "summarization", "extractive-question-answering"] +VALID_TASKS_STR = "'text-generation', 'summarization', or 'extractive-question-answering'" + ## Pydantic Data Validator Classes class TaskFormData(BaseModel): task: str @field_validator("task") def validate_task(cls, task): - if task not in ["text-generation", "summarization", "extractive-question-answering"]: - raise ValueError("Invalid task. Must be one of 'text-generation', 'summarization', or 'extractive-question-answering'.") + if task not in VALID_TASKS: + raise ValueError(f"Invalid task. Must be one of {VALID_TASKS_STR}.") return task class SelectedModelFormData(BaseModel): @@ -83,8 +88,8 @@ def validate_dataset_prescence(cls, dataset): return dataset @field_validator("task") def validate_task(cls, task): - if task not in ["text-generation", "summarization", "question-answering"]: - raise ValueError("Invalid task. Must be one of 'text-generation', 'summarization', or 'question-answering'.") + if task not in VALID_TASKS: + raise ValueError(f"Invalid task. Must be one of {VALID_TASKS_STR}.") return task @field_validator("model_name") def validate_model_name(cls, model_name): @@ -95,7 +100,7 @@ def validate_model_name(cls, model_name): def validate_num_train_epochs(cls, num_train_epochs): if num_train_epochs <= 0: raise ValueError("Number of training epochs must be greater than 0.") - elif num_train_epochs > 30: + if num_train_epochs >= 50: raise ValueError("Number of training epochs must be less than 50.") return num_train_epochs @field_validator("compute_specs") @@ -154,13 +159,9 @@ def validate_bf16(cls, bf16): raise ValueError("bf16 must be true or false.") return bf16 @field_validator("per_device_train_batch_size") - def validate_per_device_train_batch_size(cls, per_device_train_batch_size, compute_specs): + def validate_per_device_train_batch_size(cls, per_device_train_batch_size): if per_device_train_batch_size <= 0: raise ValueError("Batch size must be greater than 0.") - elif per_device_train_batch_size > 3 and compute_specs != "high_end": - raise ValueError("Batch size must be less than 4. Your device cannot support a higher batch size.") - elif per_device_train_batch_size > 8 and compute_specs == "high_end": - raise ValueError("Batch size must be less than 9. Higher batch sizes cause out of memory error.") return per_device_train_batch_size @field_validator("per_device_eval_batch_size") def validate_per_device_eval_batch_size(cls, per_device_eval_batch_size): @@ -236,11 +237,20 @@ def validate_dataset(cls, dataset): if not dataset: raise ValueError("Dataset cannot be empty.") return dataset + + @model_validator(mode='after') + def validate_batch_size_with_compute_specs(self): + """Validate batch size based on compute specs""" + if self.per_device_train_batch_size > 3 and self.compute_specs != "high_end": + raise ValueError("Batch size must be 3 or less. Your device cannot support a higher batch size.") + elif self.per_device_train_batch_size > 8 and self.compute_specs == "high_end": + raise ValueError("Batch size must be 8 or less. Higher batch sizes cause out of memory error.") + return self @router.get("/detect") async def detect_hardware_page(request: Request) -> JSONResponse: - global_manager.clear_global_manager.settings_cache() # Clear the cache to ensure fresh detection + global_manager.clear_settings_cache() # Clear the cache to ensure fresh detection return JSONResponse({ "app_name": global_manager.app_name, "message": "Ready to detect hardware" @@ -290,7 +300,7 @@ async def detect_hardware(request: Request) -> JSONResponse: print(e) raise HTTPException( status_code=400, - detail="Invalid task. Must be one of 'text-generation', 'summarization', or 'question-answering'." + detail=f"Invalid task. Must be one of {VALID_TASKS_STR}." ) except Exception as e: print("General exception triggered") @@ -317,6 +327,14 @@ async def set_model(request: Request) -> None: @router.post("/validate_custom_model", response_class=JSONResponse) async def validate_custom_model(request: Request) -> JSONResponse: + """ + Validate a custom model from HuggingFace Hub. + + Note: Currently validates repository existence but not task compatibility. + Consider adding architecture-task compatibility checks (e.g., ensure + summarization models aren't used for text generation tasks) for better + user experience and to prevent fine-tuning failures. + """ try: form = await request.json() validation_data = CustomModelValidationData(repo_name=form["repo_name"]) @@ -424,15 +442,14 @@ async def load_settings(json_file: UploadFile = File(...), settings: str = Form( def finetuning_task(llm_tuner) -> None: + output_dir = None try: llm_tuner.load_dataset(global_manager.settings_builder.dataset) + output_dir = llm_tuner.output_dir # Store for cleanup on failure path = llm_tuner.finetune() - # Handle both absolute and relative paths - if os.path.isabs(path): - model_path = path - else: - model_path = os.path.join(os.path.dirname(__file__), path.replace("./", "")) + # Use the path returned from finetune (should be absolute) + model_path = os.path.abspath(path) if not os.path.isabs(path) else path model_data = { "model_name": global_manager.settings_builder.fine_tuned_name.split('/')[-1] if global_manager.settings_builder.fine_tuned_name else os.path.basename(model_path), @@ -445,6 +462,17 @@ def finetuning_task(llm_tuner) -> None: "is_custom_base_model": global_manager.settings_builder.is_custom_model } global_manager.db_manager.add_model(model_data) + + except Exception as e: + print(f"Fine-tuning failed: {e}") + # Cleanup failed fine-tuning artifacts + if output_dir and os.path.exists(output_dir): + try: + shutil.rmtree(output_dir) + print(f"Cleaned up failed fine-tuning artifacts at: {output_dir}") + except Exception as cleanup_error: + print(f"Warning: Could not cleanup output directory: {cleanup_error}") + raise finally: global_manager.settings_cache.clear() @@ -485,6 +513,19 @@ async def start_finetuning_page(request: Request, background_task: BackgroundTas status_code=400, detail="A finetuning is already in progress. Please wait until it completes." ) + + # Validate available disk space (require at least 10GB free) + try: + stat = shutil.disk_usage(global_manager.model_path) + available_gb = stat.free / (1024 ** 3) # Convert to GB + if available_gb < 10: + raise HTTPException( + status_code=400, + detail=f"Insufficient disk space. Available: {available_gb:.2f}GB. Required: at least 10GB." + ) + except Exception as e: + print(f"Warning: Could not check disk space: {e}") + global_manager.finetuning_status["status"] = "initializing" global_manager.finetuning_status["message"] = "Starting finetuning process..." if global_manager.settings_builder.task == "text-generation": @@ -508,7 +549,7 @@ async def start_finetuning_page(request: Request, background_task: BackgroundTas else: raise HTTPException( status_code=400, - detail="Invalid task. Must be one of 'text-generation', 'summarization', or 'question-answering'." + detail=f"Invalid task. Must be one of {VALID_TASKS_STR}." ) llm_tuner.set_settings(**global_manager.settings_builder.get_settings()) diff --git a/ModelForge/routers/hub_management_router.py b/ModelForge/routers/hub_management_router.py index be44175..3ba4bbc 100644 --- a/ModelForge/routers/hub_management_router.py +++ b/ModelForge/routers/hub_management_router.py @@ -72,8 +72,8 @@ async def push_model_to_hub(request: Request) -> JSONResponse: f"Please ensure your huggingface token grants you write access. " f"If you are pushing to an organization, contact the administrator for write access."}, status_code=403) except HfHubHTTPError as e: - return JSONResponse({f"error": "Failed to push model to HuggingFace Hub. " - f"Please check your network connection and authentication token." + return JSONResponse({"error": f"Failed to push model to HuggingFace Hub. " + f"Please check your network connection and authentication token. " f"Error received is: {e}"}, status_code=500) except Exception as e: - return JSONResponse({"Unknown error": str(e)}, status_code=500) + return JSONResponse({"error": str(e)}, status_code=500) diff --git a/ModelForge/routers/playground_router.py b/ModelForge/routers/playground_router.py index 103b29e..68ca492 100644 --- a/ModelForge/routers/playground_router.py +++ b/ModelForge/routers/playground_router.py @@ -7,10 +7,21 @@ from ..globals.globals_instance import global_manager +from pydantic import BaseModel, field_validator + router = APIRouter( prefix="/playground", ) +class PlaygroundRequest(BaseModel): + model_path: str + + @field_validator("model_path") + def validate_model_path(cls, model_path): + if not model_path or not model_path.strip(): + raise ValueError("Model path cannot be empty.") + return model_path.strip() + @router.get("/model_path") async def get_model_path(request: Request) -> JSONResponse: return JSONResponse({ @@ -23,15 +34,19 @@ async def get_model_path(request: Request) -> JSONResponse: async def new_playground(request: Request) -> None: form = await request.json() print(form) - model_path = form["model_path"] + playground_request = PlaygroundRequest(model_path=form["model_path"]) + model_path = playground_request.model_path base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "utilities")) chat_script = os.path.join(base_path, "chat_playground.py") if os.name == "nt": # Windows + # Note: shell=True is required for 'start' command (cmd.exe built-in) + # Input is validated via PlaygroundRequest Pydantic model command = ["cmd.exe", "/c", "start", "python", chat_script, "--model_path", model_path] subprocess.Popen(command, shell=True) else: # Unix/Linux/Mac - command = ["x-terminal-emulator", "-e", f"python {chat_script} --model_path {model_path}"] + # Use list format without string interpolation for security + command = ["x-terminal-emulator", "-e", "python", chat_script, "--model_path", model_path] try: subprocess.Popen(command) except FileNotFoundError: @@ -39,4 +54,4 @@ async def new_playground(request: Request) -> None: try: subprocess.Popen(["gnome-terminal", "--", "python3", chat_script, "--model_path", model_path]) except FileNotFoundError: - subprocess.Popen(["xterm", "-e", f"python3 {chat_script} --model_path {model_path}"]) + subprocess.Popen(["xterm", "-e", "python3", chat_script, "--model_path", model_path]) diff --git a/ModelForge/utilities/finetuning/Seq2SeqLMTuner.py b/ModelForge/utilities/finetuning/Seq2SeqLMTuner.py index 5039605..7abcf63 100644 --- a/ModelForge/utilities/finetuning/Seq2SeqLMTuner.py +++ b/ModelForge/utilities/finetuning/Seq2SeqLMTuner.py @@ -20,30 +20,14 @@ def format_example(example: dict, specs: str, keys=None) -> Dict | None: if keys is None: keys = ["article", "summary"] - if specs == "low_end": - return { - "text": f''' - ["role": "system", "content": "You are a text summarization assistant."], - [role": "user", "content": {example[keys[0]]}], - ["role": "assistant", "content": {example[keys[1]]}] - ''' - } - elif specs == "mid_range": - return { - "text": f''' - ["role": "system", "content": "You are a text summarization assistant."], - [role": "user", "content": {example[keys[0]]}], - ["role": "assistant", "content": {example[keys[1]]}] - ''' - } - elif specs == "high_end": - return { - "text": f''' - ["role": "system", "content": "You are a text summarization assistant."], - [role": "user", "content": {example[keys[0]]}], - ["role": "assistant", "content": {example[keys[1]]}] - ''' - } + # Format is the same regardless of specs, so we can simplify + return { + "text": f''' + ["role": "system", "content": "You are a text summarization assistant."], + ["role": "user", "content": {example[keys[0]]}], + ["role": "assistant", "content": {example[keys[1]]}] + ''' + } def load_dataset(self, dataset_path: str) -> None: dataset = load_dataset("json", data_files=dataset_path, split="train") diff --git a/ModelForge/utilities/settings_managers/DBManager.py b/ModelForge/utilities/settings_managers/DBManager.py index 3b127ba..d58c043 100644 --- a/ModelForge/utilities/settings_managers/DBManager.py +++ b/ModelForge/utilities/settings_managers/DBManager.py @@ -6,6 +6,13 @@ from typing import Any class DatabaseManager: + """ + Manages SQLite database operations for ModelForge. + + Note: Currently opens/closes connections for each operation. For better performance + in high-traffic scenarios, consider implementing connection pooling using libraries + like SQLAlchemy or maintaining a connection pool manually. + """ _instance = None def __new__(cls, *args, **kwargs): @@ -153,4 +160,5 @@ def delete_model(self, model_id) -> bool: def kill_connection(self) -> None: if self.conn: self.conn.close() + self.conn = None del self.cursor \ No newline at end of file diff --git a/ModelForge/utilities/settings_managers/FileManager.py b/ModelForge/utilities/settings_managers/FileManager.py index 4b8175a..013ea03 100644 --- a/ModelForge/utilities/settings_managers/FileManager.py +++ b/ModelForge/utilities/settings_managers/FileManager.py @@ -50,7 +50,7 @@ def validate_or_create_file(cls, check_path: str) -> str: return check_path @classmethod - def save_file(cls, content:bytes, file_path: str) -> str | None: + def save_file(cls, file_path: str, content: bytes) -> str | None: try: file_dir = os.path.dirname(file_path) cls.validate_or_create_dirs(os.path.abspath(file_dir)) diff --git a/pyproject.toml b/pyproject.toml index 9900bba..2a47a47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,9 +18,9 @@ dependencies = [ "setuptools==78.1.0", "tensorboard==2.19.0", "tensorboard-data-server==0.7.2", - "tokenizers==0.21.1", + "tokenizers==0.21.0", "tqdm==4.67.1", - "transformers>=4.45.1", + "transformers==4.48.3", "trl==0.16.0", "uvicorn", "platformdirs", diff --git a/uv.lock b/uv.lock index 045c5b8..afc59ce 100644 --- a/uv.lock +++ b/uv.lock @@ -447,9 +447,9 @@ requires-dist = [ { name = "setuptools", specifier = "==78.1.0" }, { name = "tensorboard", specifier = "==2.19.0" }, { name = "tensorboard-data-server", specifier = "==0.7.2" }, - { name = "tokenizers", specifier = "==0.21.1" }, + { name = "tokenizers", specifier = "==0.21.0" }, { name = "tqdm", specifier = "==4.67.1" }, - { name = "transformers", specifier = ">=4.45.1" }, + { name = "transformers", specifier = "==4.48.3" }, { name = "trl", specifier = "==0.16.0" }, { name = "uvicorn" }, ] @@ -1088,27 +1088,27 @@ wheels = [ [[package]] name = "tokenizers" -version = "0.21.1" +version = "0.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/92/76/5ac0c97f1117b91b7eb7323dcd61af80d72f790b4df71249a7850c195f30/tokenizers-0.21.1.tar.gz", hash = "sha256:a1bb04dc5b448985f86ecd4b05407f5a8d97cb2c0532199b2a302a604a0165ab", size = 343256, upload-time = "2025-03-13T10:51:18.189Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/41/c2be10975ca37f6ec40d7abd7e98a5213bb04f284b869c1a24e6504fd94d/tokenizers-0.21.0.tar.gz", hash = "sha256:ee0894bf311b75b0c03079f33859ae4b2334d675d4e93f5a4132e1eae2834fe4", size = 343021, upload-time = "2024-11-27T13:11:23.89Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/1f/328aee25f9115bf04262e8b4e5a2050b7b7cf44b59c74e982db7270c7f30/tokenizers-0.21.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e78e413e9e668ad790a29456e677d9d3aa50a9ad311a40905d6861ba7692cf41", size = 2780767, upload-time = "2025-03-13T10:51:09.459Z" }, - { url = "https://files.pythonhosted.org/packages/ae/1a/4526797f3719b0287853f12c5ad563a9be09d446c44ac784cdd7c50f76ab/tokenizers-0.21.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:cd51cd0a91ecc801633829fcd1fda9cf8682ed3477c6243b9a095539de4aecf3", size = 2650555, upload-time = "2025-03-13T10:51:07.692Z" }, - { url = "https://files.pythonhosted.org/packages/4d/7a/a209b29f971a9fdc1da86f917fe4524564924db50d13f0724feed37b2a4d/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28da6b72d4fb14ee200a1bd386ff74ade8992d7f725f2bde2c495a9a98cf4d9f", size = 2937541, upload-time = "2025-03-13T10:50:56.679Z" }, - { url = "https://files.pythonhosted.org/packages/3c/1e/b788b50ffc6191e0b1fc2b0d49df8cff16fe415302e5ceb89f619d12c5bc/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:34d8cfde551c9916cb92014e040806122295a6800914bab5865deb85623931cf", size = 2819058, upload-time = "2025-03-13T10:50:59.525Z" }, - { url = "https://files.pythonhosted.org/packages/36/aa/3626dfa09a0ecc5b57a8c58eeaeb7dd7ca9a37ad9dd681edab5acd55764c/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aaa852d23e125b73d283c98f007e06d4595732104b65402f46e8ef24b588d9f8", size = 3133278, upload-time = "2025-03-13T10:51:04.678Z" }, - { url = "https://files.pythonhosted.org/packages/a4/4d/8fbc203838b3d26269f944a89459d94c858f5b3f9a9b6ee9728cdcf69161/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a21a15d5c8e603331b8a59548bbe113564136dc0f5ad8306dd5033459a226da0", size = 3144253, upload-time = "2025-03-13T10:51:01.261Z" }, - { url = "https://files.pythonhosted.org/packages/d8/1b/2bd062adeb7c7511b847b32e356024980c0ffcf35f28947792c2d8ad2288/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2fdbd4c067c60a0ac7eca14b6bd18a5bebace54eb757c706b47ea93204f7a37c", size = 3398225, upload-time = "2025-03-13T10:51:03.243Z" }, - { url = "https://files.pythonhosted.org/packages/8a/63/38be071b0c8e06840bc6046991636bcb30c27f6bb1e670f4f4bc87cf49cc/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dd9a0061e403546f7377df940e866c3e678d7d4e9643d0461ea442b4f89e61a", size = 3038874, upload-time = "2025-03-13T10:51:06.235Z" }, - { url = "https://files.pythonhosted.org/packages/ec/83/afa94193c09246417c23a3c75a8a0a96bf44ab5630a3015538d0c316dd4b/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:db9484aeb2e200c43b915a1a0150ea885e35f357a5a8fabf7373af333dcc8dbf", size = 9014448, upload-time = "2025-03-13T10:51:10.927Z" }, - { url = "https://files.pythonhosted.org/packages/ae/b3/0e1a37d4f84c0f014d43701c11eb8072704f6efe8d8fc2dcdb79c47d76de/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:ed248ab5279e601a30a4d67bdb897ecbe955a50f1e7bb62bd99f07dd11c2f5b6", size = 8937877, upload-time = "2025-03-13T10:51:12.688Z" }, - { url = "https://files.pythonhosted.org/packages/ac/33/ff08f50e6d615eb180a4a328c65907feb6ded0b8f990ec923969759dc379/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:9ac78b12e541d4ce67b4dfd970e44c060a2147b9b2a21f509566d556a509c67d", size = 9186645, upload-time = "2025-03-13T10:51:14.723Z" }, - { url = "https://files.pythonhosted.org/packages/5f/aa/8ae85f69a9f6012c6f8011c6f4aa1c96154c816e9eea2e1b758601157833/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e5a69c1a4496b81a5ee5d2c1f3f7fbdf95e90a0196101b0ee89ed9956b8a168f", size = 9384380, upload-time = "2025-03-13T10:51:16.526Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5b/a5d98c89f747455e8b7a9504910c865d5e51da55e825a7ae641fb5ff0a58/tokenizers-0.21.1-cp39-abi3-win32.whl", hash = "sha256:1039a3a5734944e09de1d48761ade94e00d0fa760c0e0551151d4dd851ba63e3", size = 2239506, upload-time = "2025-03-13T10:51:20.643Z" }, - { url = "https://files.pythonhosted.org/packages/e6/b6/072a8e053ae600dcc2ac0da81a23548e3b523301a442a6ca900e92ac35be/tokenizers-0.21.1-cp39-abi3-win_amd64.whl", hash = "sha256:0f0dcbcc9f6e13e675a66d7a5f2f225a736745ce484c1a4e07476a89ccdad382", size = 2435481, upload-time = "2025-03-13T10:51:19.243Z" }, + { url = "https://files.pythonhosted.org/packages/b0/5c/8b09607b37e996dc47e70d6a7b6f4bdd4e4d5ab22fe49d7374565c7fefaf/tokenizers-0.21.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3c4c93eae637e7d2aaae3d376f06085164e1660f89304c0ab2b1d08a406636b2", size = 2647461, upload-time = "2024-11-27T13:11:07.911Z" }, + { url = "https://files.pythonhosted.org/packages/22/7a/88e58bb297c22633ed1c9d16029316e5b5ac5ee44012164c2edede599a5e/tokenizers-0.21.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:f53ea537c925422a2e0e92a24cce96f6bc5046bbef24a1652a5edc8ba975f62e", size = 2563639, upload-time = "2024-11-27T13:11:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/f7/14/83429177c19364df27d22bc096d4c2e431e0ba43e56c525434f1f9b0fd00/tokenizers-0.21.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b177fb54c4702ef611de0c069d9169f0004233890e0c4c5bd5508ae05abf193", size = 2903304, upload-time = "2024-11-27T13:10:51.315Z" }, + { url = "https://files.pythonhosted.org/packages/7e/db/3433eab42347e0dc5452d8fcc8da03f638c9accffefe5a7c78146666964a/tokenizers-0.21.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6b43779a269f4629bebb114e19c3fca0223296ae9fea8bb9a7a6c6fb0657ff8e", size = 2804378, upload-time = "2024-11-27T13:10:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/57/8b/7da5e6f89736c2ade02816b4733983fca1c226b0c42980b1ae9dc8fcf5cc/tokenizers-0.21.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9aeb255802be90acfd363626753fda0064a8df06031012fe7d52fd9a905eb00e", size = 3095488, upload-time = "2024-11-27T13:11:00.662Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f6/5ed6711093dc2c04a4e03f6461798b12669bc5a17c8be7cce1240e0b5ce8/tokenizers-0.21.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8b09dbeb7a8d73ee204a70f94fc06ea0f17dcf0844f16102b9f414f0b7463ba", size = 3121410, upload-time = "2024-11-27T13:10:55.674Z" }, + { url = "https://files.pythonhosted.org/packages/81/42/07600892d48950c5e80505b81411044a2d969368cdc0d929b1c847bf6697/tokenizers-0.21.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:400832c0904f77ce87c40f1a8a27493071282f785724ae62144324f171377273", size = 3388821, upload-time = "2024-11-27T13:10:58.401Z" }, + { url = "https://files.pythonhosted.org/packages/22/06/69d7ce374747edaf1695a4f61b83570d91cc8bbfc51ccfecf76f56ab4aac/tokenizers-0.21.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84ca973b3a96894d1707e189c14a774b701596d579ffc7e69debfc036a61a04", size = 3008868, upload-time = "2024-11-27T13:11:03.734Z" }, + { url = "https://files.pythonhosted.org/packages/c8/69/54a0aee4d576045b49a0eb8bffdc495634309c823bf886042e6f46b80058/tokenizers-0.21.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:eb7202d231b273c34ec67767378cd04c767e967fda12d4a9e36208a34e2f137e", size = 8975831, upload-time = "2024-11-27T13:11:10.32Z" }, + { url = "https://files.pythonhosted.org/packages/f7/f3/b776061e4f3ebf2905ba1a25d90380aafd10c02d406437a8ba22d1724d76/tokenizers-0.21.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:089d56db6782a73a27fd8abf3ba21779f5b85d4a9f35e3b493c7bbcbbf0d539b", size = 8920746, upload-time = "2024-11-27T13:11:13.238Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ee/ce83d5ec8b6844ad4c3ecfe3333d58ecc1adc61f0878b323a15355bcab24/tokenizers-0.21.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:c87ca3dc48b9b1222d984b6b7490355a6fdb411a2d810f6f05977258400ddb74", size = 9161814, upload-time = "2024-11-27T13:11:16.675Z" }, + { url = "https://files.pythonhosted.org/packages/18/07/3e88e65c0ed28fa93aa0c4d264988428eef3df2764c3126dc83e243cb36f/tokenizers-0.21.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4145505a973116f91bc3ac45988a92e618a6f83eb458f49ea0790df94ee243ff", size = 9357138, upload-time = "2024-11-27T13:11:20.09Z" }, + { url = "https://files.pythonhosted.org/packages/15/b0/dc4572ca61555fc482ebc933f26cb407c6aceb3dc19c301c68184f8cad03/tokenizers-0.21.0-cp39-abi3-win32.whl", hash = "sha256:eb1702c2f27d25d9dd5b389cc1f2f51813e99f8ca30d9e25348db6585a97e24a", size = 2202266, upload-time = "2024-11-27T13:11:28.784Z" }, + { url = "https://files.pythonhosted.org/packages/44/69/d21eb253fa91622da25585d362a874fa4710be600f0ea9446d8d0217cec1/tokenizers-0.21.0-cp39-abi3-win_amd64.whl", hash = "sha256:87841da5a25a3a5f70c102de371db120f41873b854ba65e52bccd57df5a3780c", size = 2389192, upload-time = "2024-11-27T13:11:25.724Z" }, ] [[package]] @@ -1159,7 +1159,7 @@ wheels = [ [[package]] name = "transformers" -version = "4.53.3" +version = "4.48.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, @@ -1173,9 +1173,9 @@ dependencies = [ { name = "tokenizers" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/5c/49182918b58eaa0b4c954fd0e37c79fc299e5643e69d70089d0b0eb0cd9b/transformers-4.53.3.tar.gz", hash = "sha256:b2eda1a261de79b78b97f7888fe2005fc0c3fabf5dad33d52cc02983f9f675d8", size = 9197478, upload-time = "2025-07-22T07:30:51.51Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/82/cebeb7af5e64440f1638f18c4ed0f89156d0eeaa6290d98da8ca93ac3872/transformers-4.48.3.tar.gz", hash = "sha256:a5e8f1e9a6430aa78215836be70cecd3f872d99eeda300f41ad6cc841724afdb", size = 8373458, upload-time = "2025-02-07T10:10:47.402Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/b1/d7520cc5cb69c825599042eb3a7c986fa9baa8a8d2dea9acd78e152c81e2/transformers-4.53.3-py3-none-any.whl", hash = "sha256:5aba81c92095806b6baf12df35d756cf23b66c356975fb2a7fa9e536138d7c75", size = 10826382, upload-time = "2025-07-22T07:30:48.458Z" }, + { url = "https://files.pythonhosted.org/packages/b6/1a/efeecb8d83705f2f4beac98d46f2148c95ecd7babfb31b5c0f1e7017e83d/transformers-4.48.3-py3-none-any.whl", hash = "sha256:78697f990f5ef350c23b46bf86d5081ce96b49479ab180b2de7687267de8fd36", size = 9669412, upload-time = "2025-02-07T10:10:43.395Z" }, ] [[package]]