Add object ttl

Author: dirkkul

integration/conftest.py

@@ -21,6 +21,7 @@
 import weaviate
 from weaviate.collections import Collection, CollectionAsync
 from weaviate.collections.classes.config import (
+    _ObjectTTLCreate,
     Configure,
     DataType,
     Property,
@@ -66,6 +67,7 @@ def __call__(
         vector_config: Optional[
             Optional[Union[_VectorConfigCreate, List[_VectorConfigCreate]]]
         ] = None,
+        object_ttl: Optional[_ObjectTTLCreate] = None,
     ) -> Collection[Any, Any]:
         """Typing for fixture."""
         ...
@@ -140,6 +142,7 @@ def _factory(
         vector_config: Optional[
             Optional[Union[_VectorConfigCreate, List[_VectorConfigCreate]]]
         ] = None,
+        object_ttl: Optional[_ObjectTTLCreate] = None,
     ) -> Collection[Any, Any]:
         try:
             nonlocal client_fixture, name_fixtures, call_counter  # noqa: F824
@@ -172,6 +175,7 @@ def _factory(
                 vector_index_config=vector_index_config,
                 reranker_config=reranker_config,
                 vector_config=vector_config,
+                object_ttl_config=object_ttl,
             )
             return collection
         except Exception as e:

integration/test_collection_config.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Generator, List, Optional, Union
 
 import pytest as pytest
@@ -1831,3 +1832,63 @@ def test_uncompressed_quantitizer(collection_factory: CollectionFactory) -> None
     assert config.vector_index_config is not None
     assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW)
     assert config.vector_index_config.quantizer is None
+
+
+def test_object_ttl_creation(collection_factory: CollectionFactory) -> None:
+    dummy = collection_factory("dummy")
+    if dummy._connection._weaviate_version.is_lower_than(1, 35, 0):
+        pytest.skip("object ttl is not supported in Weaviate versions lower than 1.35.0")
+
+    collection = collection_factory(
+        object_ttl=Configure.ObjectTTL.delete_by_creation_time(
+            time_to_live=datetime.timedelta(days=30),
+            post_search_filter=True,
+        ),
+        inverted_index_config=Configure.inverted_index(index_timestamps=True),
+    )
+
+    config = collection.config.get()
+    assert config.object_ttl_config is not None
+    assert config.object_ttl_config.delete_on == "creationTime"
+    assert config.object_ttl_config.time_to_live == datetime.timedelta(days=30)
+
+
+def test_object_ttl_update(collection_factory: CollectionFactory) -> None:
+    dummy = collection_factory("dummy")
+    if dummy._connection._weaviate_version.is_lower_than(1, 35, 0):
+        pytest.skip("object ttl is not supported in Weaviate versions lower than 1.35.0")
+
+    collection = collection_factory(
+        object_ttl=Configure.ObjectTTL.delete_by_update_time(
+            time_to_live=datetime.timedelta(days=30),
+            post_search_filter=True,
+        ),
+        inverted_index_config=Configure.inverted_index(index_timestamps=True),
+    )
+
+    config = collection.config.get()
+    assert config.object_ttl_config is not None
+    assert config.object_ttl_config.delete_on == "updateTime"
+    assert config.object_ttl_config.post_search_filter
+    assert config.object_ttl_config.time_to_live == datetime.timedelta(days=30)
+
+
+def test_object_ttl_custom(collection_factory: CollectionFactory) -> None:
+    dummy = collection_factory("dummy")
+    if dummy._connection._weaviate_version.is_lower_than(1, 35, 0):
+        pytest.skip("object ttl is not supported in Weaviate versions lower than 1.35.0")
+
+    collection = collection_factory(
+        properties=[wvc.config.Property(name="customDate", data_type=DataType.DATE)],
+        object_ttl=Configure.ObjectTTL.delete_by_date_property(
+            date_property="customDate",
+            post_search_filter=False,
+        ),
+        inverted_index_config=Configure.inverted_index(index_timestamps=True),
+    )
+
+    config = collection.config.get()
+    assert config.object_ttl_config is not None
+    assert config.object_ttl_config.delete_on == "customDate"
+    # assert config.object_ttl_config.time_to_live is None
+    assert not config.object_ttl_config.post_search_filter

weaviate/collections/classes/config.py

@@ -1,3 +1,4 @@
+import datetime
 from dataclasses import dataclass
 from typing import (
     Any,
@@ -31,6 +32,7 @@
     _NamedVectors,
     _NamedVectorsUpdate,
 )
+from weaviate.collections.classes.config_object_ttl import _ObjectTTL, _ObjectTTLCreate
 from weaviate.collections.classes.config_vector_index import (
     PQEncoderDistribution,
     PQEncoderType,
@@ -1136,64 +1138,6 @@ def contextualai(
         return _RerankerContextualAIConfig(model=model, instruction=instruction, topN=top_n)
 
 
-class _CollectionConfigCreateBase(_ConfigCreateModel):
-    description: Optional[str] = Field(default=None)
-    invertedIndexConfig: Optional[_InvertedIndexConfigCreate] = Field(
-        default=None, alias="inverted_index_config"
-    )
-    multiTenancyConfig: Optional[_MultiTenancyConfigCreate] = Field(
-        default=None, alias="multi_tenancy_config"
-    )
-    replicationConfig: Optional[_ReplicationConfigCreate] = Field(
-        default=None, alias="replication_config"
-    )
-    shardingConfig: Optional[_ShardingConfigCreate] = Field(default=None, alias="sharding_config")
-    vectorIndexConfig: Optional[_VectorIndexConfigCreate] = Field(
-        default=None, alias="vector_index_config"
-    )
-    moduleConfig: _VectorizerConfigCreate = Field(
-        default=_Vectorizer.none(), alias="vectorizer_config"
-    )
-    generativeSearch: Optional[_GenerativeProvider] = Field(default=None, alias="generative_config")
-    rerankerConfig: Optional[_RerankerProvider] = Field(default=None, alias="reranker_config")
-
-    def _to_dict(self) -> Dict[str, Any]:
-        ret_dict: Dict[str, Any] = {}
-
-        for cls_field in type(self).model_fields:
-            val = getattr(self, cls_field)
-            if cls_field in ["name", "model", "properties", "references"] or val is None:
-                continue
-            elif isinstance(val, (bool, float, str, int)):
-                ret_dict[cls_field] = str(val)
-            elif isinstance(val, _GenerativeProvider):
-                self.__add_to_module_config(ret_dict, val.generative.value, val._to_dict())
-            elif isinstance(val, _RerankerProvider):
-                self.__add_to_module_config(ret_dict, val.reranker.value, val._to_dict())
-            elif isinstance(val, _VectorizerConfigCreate):
-                ret_dict["vectorizer"] = val.vectorizer.value
-                if val.vectorizer != Vectorizers.NONE:
-                    self.__add_to_module_config(ret_dict, val.vectorizer.value, val._to_dict())
-            elif isinstance(val, _VectorIndexConfigCreate):
-                ret_dict["vectorIndexType"] = val.vector_index_type()
-                ret_dict[cls_field] = val._to_dict()
-            else:
-                assert isinstance(val, _ConfigCreateModel)
-                ret_dict[cls_field] = val._to_dict()
-        if self.vectorIndexConfig is None:
-            ret_dict["vectorIndexType"] = VectorIndexType.HNSW.value
-        return ret_dict
-
-    @staticmethod
-    def __add_to_module_config(
-        return_dict: Dict[str, Any], addition_key: str, addition_val: Dict[str, Any]
-    ) -> None:
-        if "moduleConfig" not in return_dict:
-            return_dict["moduleConfig"] = {addition_key: addition_val}
-        else:
-            return_dict["moduleConfig"][addition_key] = addition_val
-
-
 class _CollectionConfigUpdate(_ConfigUpdateModel):
     description: Optional[str] = Field(default=None)
     property_descriptions: Optional[Dict[str, str]] = Field(default=None)
@@ -1764,13 +1708,25 @@ def to_dict(self) -> Dict:
 NamedVectorConfig = _NamedVectorConfig
 
 
+@dataclass
+class _ObjectTTLConfig(_ConfigBase):
+    enabled: bool
+    time_to_live: Optional[datetime.timedelta]
+    post_search_filter: bool
+    delete_on: Union[str, Literal["updateTime"], Literal["creationTime"]]
+
+
+ObjectTTLConfig = _ObjectTTLConfig
+
+
 @dataclass
 class _CollectionConfig(_ConfigBase):
     name: str
     description: Optional[str]
     generative_config: Optional[GenerativeConfig]
     inverted_index_config: InvertedIndexConfig
     multi_tenancy_config: MultiTenancyConfig
+    object_ttl_config: Optional[ObjectTTLConfig]
     properties: List[PropertyConfig]
     references: List[ReferencePropertyConfig]
     replication_config: ReplicationConfig
@@ -1841,6 +1797,7 @@ class _CollectionConfigSimple(_ConfigBase):
     vectorizer_config: Optional[VectorizerConfig]
     vectorizer: Optional[Union[Vectorizers, str]]
     vector_config: Optional[Dict[str, _NamedVectorConfig]]
+    object_ttl_config: Optional[ObjectTTLConfig]
 
 
 CollectionConfigSimple = _CollectionConfigSimple
@@ -1992,6 +1949,7 @@ class _CollectionConfigCreate(_ConfigCreateModel):
     multiTenancyConfig: Optional[_MultiTenancyConfigCreate] = Field(
         default=None, alias="multi_tenancy_config"
     )
+    objectTtlConfig: Optional[_ObjectTTLCreate] = Field(default=None, alias="object_ttl_config")
     replicationConfig: Optional[_ReplicationConfigCreate] = Field(
         default=None, alias="replication_config"
     )
@@ -2155,6 +2113,7 @@ class Configure:
     NamedVectors = _NamedVectors
     Vectors = _Vectors
     MultiVectors = _MultiVectors
+    ObjectTTL = _ObjectTTL
 
     @staticmethod
     def inverted_index(

weaviate/collections/classes/config_methods.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Any, Dict, List, Optional, Union, cast
 
 from weaviate.collections.classes.config import (
@@ -25,6 +26,7 @@
     _NamedVectorConfig,
     _NamedVectorizerConfig,
     _NestedProperty,
+    _ObjectTTLConfig,
     _PQConfig,
     _PQEncoderConfig,
     _Property,
@@ -293,6 +295,7 @@ def _collection_config_simple_from_json(schema: Dict[str, Any]) -> _CollectionCo
         name=schema["class"],
         description=schema.get("description"),
         generative_config=__get_generative_config(schema),
+        object_ttl_config=_get_object_ttl_config(schema),
         properties=(
             _properties_from_config(schema) if schema.get("properties") is not None else []
         ),
@@ -340,6 +343,7 @@ def _collection_config_from_json(schema: Dict[str, Any]) -> _CollectionConfig:
                 "autoTenantActivation", False
             ),
         ),
+        object_ttl_config=_get_object_ttl_config(schema),
         properties=(
             _properties_from_config(schema) if schema.get("properties") is not None else []
         ),
@@ -378,6 +382,27 @@ def _collection_config_from_json(schema: Dict[str, Any]) -> _CollectionConfig:
     )
 
 
+def _get_object_ttl_config(schema: Dict[str, Any]) -> Optional[_ObjectTTLConfig]:
+    if "objectTtlConfig" in schema and schema["objectTtlConfig"].get("enabled", False):
+        time_to_live = schema["objectTtlConfig"].get("defaultTtl")
+        if time_to_live is not None and isinstance(time_to_live, int):
+            time_to_live = datetime.timedelta(seconds=time_to_live)
+        delete_on = schema["objectTtlConfig"]["deleteOn"]
+        if delete_on == "_lastUpdateTimeUnix":
+            delete_on = "updateTime"
+        elif delete_on == "_creationTimeUnix":
+            delete_on = "creationTime"
+
+        return _ObjectTTLConfig(
+            enabled=True,
+            delete_on=delete_on,
+            post_search_filter=schema["objectTtlConfig"]["postSearchFilter"],
+            time_to_live=time_to_live,
+        )
+    else:
+        return None
+
+
 def _collection_configs_from_json(schema: Dict[str, Any]) -> Dict[str, _CollectionConfig]:
     configs = {
         schema["class"]: _collection_config_from_json(schema) for schema in schema["classes"]

weaviate/collections/classes/config_object_ttl.py

@@ -0,0 +1,70 @@
+import datetime
+from typing import Optional
+
+from weaviate.collections.classes.config_base import _ConfigCreateModel
+
+
+class _ObjectTTLCreate(_ConfigCreateModel):
+    enabled: bool = True
+    postSearchFilter: Optional[bool]
+    deleteOn: Optional[str]
+    defaultTtl: Optional[int]
+
+
+class _ObjectTTL:
+    """Configuration class for Weaviate's object time-to-live (TTL) feature."""
+
+    @staticmethod
+    def delete_by_update_time(
+        time_to_live: int | datetime.timedelta,
+        post_search_filter: Optional[bool] = None,
+    ) -> _ObjectTTLCreate:
+        """Create an `ObjectTimeToLiveConfig` object to be used when defining the object time-to-live configuration of Weaviate.
+
+        Args:
+            time_to_live: The time-to-live for objects in seconds.
+            post_search_filter: If enabled search results will be filtered to remove expired objects that have not yet been deleted.
+        """
+        if isinstance(time_to_live, datetime.timedelta):
+            time_to_live = int(time_to_live.total_seconds())
+        return _ObjectTTLCreate(
+            deleteOn="_lastUpdateTimeUnix",
+            postSearchFilter=post_search_filter,
+            defaultTtl=time_to_live,
+        )
+
+    @staticmethod
+    def delete_by_creation_time(
+        time_to_live: int | datetime.timedelta,
+        post_search_filter: Optional[bool] = None,
+    ) -> _ObjectTTLCreate:
+        """Create an `ObjectTimeToLiveConfig` object to be used when defining the object time-to-live configuration of Weaviate.
+
+        Args:
+            time_to_live: The time-to-live for objects in seconds.
+            post_search_filter: If enabled search results will be filtered to remove expired objects that have not yet been deleted.
+        """
+        if isinstance(time_to_live, datetime.timedelta):
+            time_to_live = int(time_to_live.total_seconds())
+        return _ObjectTTLCreate(
+            deleteOn="_creationTimeUnix",
+            postSearchFilter=post_search_filter,
+            defaultTtl=time_to_live,
+        )
+
+    @staticmethod
+    def delete_by_date_property(
+        date_property: str,
+        post_search_filter: Optional[bool] = None,
+    ) -> _ObjectTTLCreate:
+        """Create an Object ttl config for a custom date property.
+
+        Args:
+            date_property: The name of the date property to use for object expiration.
+            post_search_filter: If enabled search results will be filtered to remove expired objects that have not yet been deleted.
+        """
+        return _ObjectTTLCreate(
+            deleteOn=date_property,
+            postSearchFilter=post_search_filter,
+            defaultTtl=None,
+        )

weaviate/collections/collections/async_.pyi

@@ -18,6 +18,7 @@ from weaviate.collections.classes.config import (
     _VectorIndexConfigCreate,
     _VectorizerConfigCreate,
 )
+from weaviate.collections.classes.config_object_ttl import _ObjectTTLCreate
 from weaviate.collections.classes.internal import References
 from weaviate.collections.classes.types import (
     Properties,
@@ -36,6 +37,7 @@ class _CollectionsAsync(_CollectionsBase[ConnectionAsync]):
         generative_config: Optional[_GenerativeProvider] = None,
         inverted_index_config: Optional[_InvertedIndexConfigCreate] = None,
         multi_tenancy_config: Optional[_MultiTenancyConfigCreate] = None,
+        object_ttl_config: Optional[_ObjectTTLCreate] = None,
         properties: Optional[Sequence[Property]] = None,
         references: Optional[List[_ReferencePropertyBase]] = None,
         replication_config: Optional[_ReplicationConfigCreate] = None,
@@ -62,6 +64,7 @@ class _CollectionsAsync(_CollectionsBase[ConnectionAsync]):
         generative_config: Optional[_GenerativeProvider] = None,
         inverted_index_config: Optional[_InvertedIndexConfigCreate] = None,
         multi_tenancy_config: Optional[_MultiTenancyConfigCreate] = None,
+        object_ttl_config: Optional[_ObjectTTLCreate] = None,
         properties: Optional[Sequence[Property]] = None,
         references: Optional[List[_ReferencePropertyBase]] = None,
         replication_config: Optional[_ReplicationConfigCreate] = None,
@@ -88,6 +91,7 @@ class _CollectionsAsync(_CollectionsBase[ConnectionAsync]):
         generative_config: Optional[_GenerativeProvider] = None,
         inverted_index_config: Optional[_InvertedIndexConfigCreate] = None,
         multi_tenancy_config: Optional[_MultiTenancyConfigCreate] = None,
+        object_ttl_config: Optional[_ObjectTTLCreate] = None,
         properties: Optional[Sequence[Property]] = None,
         references: Optional[List[_ReferencePropertyBase]] = None,
         replication_config: Optional[_ReplicationConfigCreate] = None,