diff --git a/fastembed/common/model_description.py b/fastembed/common/model_description.py index 432f8186..f1d7ac45 100644 --- a/fastembed/common/model_description.py +++ b/fastembed/common/model_description.py @@ -7,6 +7,11 @@ class ModelSource: hf: Optional[str] = None url: Optional[str] = None + _deprecated_tar_struct: bool = False + + @property + def deprecated_tar_struct(self) -> bool: + return self._deprecated_tar_struct def __post_init__(self) -> None: if self.hf is None and self.url is None: diff --git a/fastembed/common/model_management.py b/fastembed/common/model_management.py index 33e0994f..ce004cb6 100644 --- a/fastembed/common/model_management.py +++ b/fastembed/common/model_management.py @@ -330,9 +330,10 @@ def retrieve_model_gcs( model_name: str, source_url: str, cache_dir: str, + deprecated_tar_struct: bool = False, local_files_only: bool = False, ) -> Path: - fast_model_name = f"fast-{model_name.split('/')[-1]}" + fast_model_name = f"{'fast-' if deprecated_tar_struct else ''}{model_name.split('/')[-1]}" cache_tmp_dir = Path(cache_dir) / "tmp" model_tmp_dir = cache_tmp_dir / fast_model_name model_dir = Path(cache_dir) / fast_model_name @@ -438,6 +439,7 @@ def download_model(cls, model: T, cache_dir: str, retries: int = 3, **kwargs: An model.model, str(url_source), str(cache_dir), + deprecated_tar_struct=model.sources.deprecated_tar_struct, local_files_only=local_files_only, ) except Exception: diff --git a/fastembed/text/onnx_embedding.py b/fastembed/text/onnx_embedding.py index d272c37a..e683e2eb 100644 --- a/fastembed/text/onnx_embedding.py +++ b/fastembed/text/onnx_embedding.py @@ -21,6 +21,7 @@ sources=ModelSource( hf="Qdrant/fast-bge-base-en", url="https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en.tar.gz", + _deprecated_tar_struct=True, ), model_file="model_optimized.onnx", ), @@ -36,6 +37,7 @@ sources=ModelSource( hf="qdrant/bge-base-en-v1.5-onnx-q", url="https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en-v1.5.tar.gz", + _deprecated_tar_struct=True, ), model_file="model_optimized.onnx", ), @@ -63,6 +65,7 @@ sources=ModelSource( hf="Qdrant/bge-small-en", url="https://storage.googleapis.com/qdrant-fastembed/BAAI-bge-small-en.tar.gz", + _deprecated_tar_struct=True, ), model_file="model_optimized.onnx", ), @@ -90,6 +93,7 @@ sources=ModelSource( hf="Qdrant/bge-small-zh-v1.5", url="https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-zh-v1.5.tar.gz", + _deprecated_tar_struct=True, ), model_file="model_optimized.onnx", ), diff --git a/fastembed/text/pooled_embedding.py b/fastembed/text/pooled_embedding.py index 1dc8c9f5..4e67e3be 100644 --- a/fastembed/text/pooled_embedding.py +++ b/fastembed/text/pooled_embedding.py @@ -82,6 +82,7 @@ sources=ModelSource( hf="qdrant/multilingual-e5-large-onnx", url="https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz", + _deprecated_tar_struct=True, ), model_file="model.onnx", additional_files=["model.onnx_data"], diff --git a/fastembed/text/pooled_normalized_embedding.py b/fastembed/text/pooled_normalized_embedding.py index ed825eca..ab15bae4 100644 --- a/fastembed/text/pooled_normalized_embedding.py +++ b/fastembed/text/pooled_normalized_embedding.py @@ -22,6 +22,7 @@ sources=ModelSource( url="https://storage.googleapis.com/qdrant-fastembed/sentence-transformers-all-MiniLM-L6-v2.tar.gz", hf="qdrant/all-MiniLM-L6-v2-onnx", + _deprecated_tar_struct=True, ), model_file="model.onnx", ), diff --git a/pyproject.toml b/pyproject.toml index f4af5889..4effb948 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fastembed" -version = "0.5.1" +version = "0.6.0" description = "Fast, light, accurate library built for retrieval embedding generation" authors = ["Qdrant Team ", "NirantK "] license = "Apache License"