Merge pull request #1899 from weaviate/object_ttl

Add support for Object TTL

Author: dirkkul

.github/workflows/main.yaml

@@ -26,6 +26,7 @@ env:
   WEAVIATE_132: 1.32.16
   WEAVIATE_133: 1.33.4
   WEAVIATE_134: 1.34.1
+  WEAVIATE_135: 1.35.0-dev-0b0bd3d
 
 jobs:
   lint-and-format:
@@ -305,6 +306,7 @@ jobs:
           $WEAVIATE_132,
           $WEAVIATE_133,
           $WEAVIATE_134
+          $WEAVIATE_135
         ]
     steps:
       - name: Checkout

integration/conftest.py

@@ -1,4 +1,5 @@
 import os
+import time
 from typing import (
     Any,
     AsyncGenerator,
@@ -11,16 +12,16 @@
     Type,
     Union,
 )
+from typing import Callable, TypeVar
 
 import pytest
 import pytest_asyncio
 from _pytest.fixtures import SubRequest
-import time
-from typing import Callable, TypeVar
 
 import weaviate
 from weaviate.collections import Collection, CollectionAsync
 from weaviate.collections.classes.config import (
+    _ObjectTTLConfigCreate,
     Configure,
     DataType,
     Property,
@@ -37,7 +38,6 @@
 from weaviate.collections.classes.config_named_vectors import _NamedVectorConfigCreate
 from weaviate.collections.classes.types import Properties
 from weaviate.config import AdditionalConfig
-
 from weaviate.exceptions import UnexpectedStatusCodeError
 
 
@@ -66,6 +66,7 @@ def __call__(
         vector_config: Optional[
             Optional[Union[_VectorConfigCreate, List[_VectorConfigCreate]]]
         ] = None,
+        object_ttl: Optional[_ObjectTTLConfigCreate] = None,
     ) -> Collection[Any, Any]:
         """Typing for fixture."""
         ...
@@ -140,6 +141,7 @@ def _factory(
         vector_config: Optional[
             Optional[Union[_VectorConfigCreate, List[_VectorConfigCreate]]]
         ] = None,
+        object_ttl: Optional[_ObjectTTLConfigCreate] = None,
     ) -> Collection[Any, Any]:
         try:
             nonlocal client_fixture, name_fixtures, call_counter  # noqa: F824
@@ -172,6 +174,7 @@ def _factory(
                 vector_index_config=vector_index_config,
                 reranker_config=reranker_config,
                 vector_config=vector_config,
+                object_ttl_config=object_ttl,
             )
             return collection
         except Exception as e:

integration/test_collection_config.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Generator, List, Optional, Union
 
 import pytest as pytest
@@ -1831,3 +1832,119 @@ def test_uncompressed_quantitizer(collection_factory: CollectionFactory) -> None
     assert config.vector_index_config is not None
     assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW)
     assert config.vector_index_config.quantizer is None
+
+
+def test_object_ttl_creation(collection_factory: CollectionFactory) -> None:
+    dummy = collection_factory("dummy")
+    if dummy._connection._weaviate_version.is_lower_than(1, 35, 0):
+        pytest.skip("object ttl is not supported in Weaviate versions lower than 1.35.0")
+
+    collection = collection_factory(
+        object_ttl=Configure.ObjectTTL.delete_by_creation_time(
+            time_to_live=datetime.timedelta(days=30),
+            filter_expired_objects=True,
+        ),
+        inverted_index_config=Configure.inverted_index(index_timestamps=True),
+    )
+
+    config = collection.config.get()
+    assert config.object_ttl_config is not None
+    assert config.object_ttl_config.delete_on == "creationTime"
+    assert config.object_ttl_config.time_to_live == datetime.timedelta(days=30)
+
+
+def test_object_ttl_update_time(collection_factory: CollectionFactory) -> None:
+    dummy = collection_factory("dummy")
+    if dummy._connection._weaviate_version.is_lower_than(1, 35, 0):
+        pytest.skip("object ttl is not supported in Weaviate versions lower than 1.35.0")
+
+    collection = collection_factory(
+        object_ttl=Configure.ObjectTTL.delete_by_update_time(
+            time_to_live=datetime.timedelta(days=30),
+            filter_expired_objects=True,
+        ),
+        inverted_index_config=Configure.inverted_index(index_timestamps=True),
+    )
+
+    config = collection.config.get()
+    assert config.object_ttl_config is not None
+    assert config.object_ttl_config.delete_on == "updateTime"
+    assert config.object_ttl_config.filter_expired_objects
+    assert config.object_ttl_config.time_to_live == datetime.timedelta(days=30)
+
+
+def test_object_ttl_custom(collection_factory: CollectionFactory) -> None:
+    dummy = collection_factory("dummy")
+    if dummy._connection._weaviate_version.is_lower_than(1, 35, 0):
+        pytest.skip("object ttl is not supported in Weaviate versions lower than 1.35.0")
+
+    collection = collection_factory(
+        properties=[wvc.config.Property(name="customDate", data_type=DataType.DATE)],
+        object_ttl=Configure.ObjectTTL.delete_by_date_property(
+            property_name="customDate", filter_expired_objects=False, ttl_offset=-1
+        ),
+        inverted_index_config=Configure.inverted_index(index_timestamps=True),
+    )
+
+    config = collection.config.get()
+    assert config.object_ttl_config is not None
+    assert config.object_ttl_config.delete_on == "customDate"
+    assert config.object_ttl_config.time_to_live == datetime.timedelta(seconds=-1)
+    assert not config.object_ttl_config.filter_expired_objects
+
+
+def test_object_ttl_update(collection_factory: CollectionFactory) -> None:
+    dummy = collection_factory("dummy")
+    if dummy._connection._weaviate_version.is_lower_than(1, 35, 0):
+        pytest.skip("object ttl is not supported in Weaviate versions lower than 1.35.0")
+
+    collection = collection_factory(
+        properties=[
+            wvc.config.Property(name="customDate", data_type=DataType.DATE),
+            wvc.config.Property(name="customDate2", data_type=DataType.DATE),
+        ],
+        inverted_index_config=Configure.inverted_index(index_timestamps=True),
+    )
+
+    conf = collection.config.get()
+    assert conf.object_ttl_config is None
+
+    collection.config.update(
+        object_ttl_config=Reconfigure.ObjectTTL.delete_by_date_property(
+            property_name="customDate", filter_expired_objects=True, ttl_offset=3600
+        ),
+    )
+
+    conf = collection.config.get()
+    assert conf.object_ttl_config is not None
+    assert conf.object_ttl_config.delete_on == "customDate"
+    assert conf.object_ttl_config.time_to_live == datetime.timedelta(seconds=3600)
+    assert conf.object_ttl_config.filter_expired_objects
+
+    collection.config.update(
+        object_ttl_config=Reconfigure.ObjectTTL.delete_by_update_time(filter_expired_objects=False),
+    )
+
+    conf = collection.config.get()
+    assert conf.object_ttl_config is not None
+    assert conf.object_ttl_config.delete_on == "updateTime"
+    assert conf.object_ttl_config.time_to_live == datetime.timedelta(seconds=3600)
+    assert not conf.object_ttl_config.filter_expired_objects
+
+    collection.config.update(
+        object_ttl_config=Reconfigure.ObjectTTL.delete_by_creation_time(
+            time_to_live=datetime.timedelta(seconds=600),
+        ),
+    )
+
+    conf = collection.config.get()
+    assert conf.object_ttl_config is not None
+    assert conf.object_ttl_config.delete_on == "creationTime"
+    assert conf.object_ttl_config.time_to_live == datetime.timedelta(seconds=600)
+    assert not conf.object_ttl_config.filter_expired_objects
+
+    collection.config.update(
+        object_ttl_config=Reconfigure.ObjectTTL.disable(),
+    )
+    conf = collection.config.get()
+    assert conf.object_ttl_config is None

mock_tests/test_collection.py

@@ -106,6 +106,7 @@ def test_missing_multi_tenancy_config(
         reranker_config=None,
         vectorizer_config=None,
         vector_config=None,
+        object_ttl_config=None,
         inverted_index_config=InvertedIndexConfig(
             bm25=BM25Config(b=0, k1=0),
             cleanup_interval_seconds=0,