Skip to content

Commit 61613a0

Browse files
mpartipilompartipilo
authored
Merge pull request #277 from weaviate/feat/object-ttl
2 parents 9122f3f + bfbb762 commit 61613a0

30 files changed

+2341
-1154
lines changed

‎.github/copilot-instructions.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@
2020
- Use `ToModel()`/`ToDto()` in `Rest/Dto/Extensions.cs` for mapping
2121

2222
## Enum & Wire Format
23-
- Use `ToEnumMemberString()`/`FromEnumMemberString<T>()` for conveting enums to and from strings
23+
- Use `ToEnumMemberString()`/`FromEnumMemberString<T>()` for converting enums to and from strings
2424
- Always prefer enums for permission actions and resource types
25+
- Stick to .NET defaults for JSON serialization, using JsonStringEnumMemberName on enum values for specifying the string conversion, and System.Text.Json.Serialization.JsonStringEnumConverter on properties and fields to speciy the actual conversion on the properties.
2526

2627
## Testing
2728
- Unit: xUnit, mock HTTP handler for REST

‎.github/workflows/main.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ jobs:
147147
strategy:
148148
fail-fast: false
149149
matrix:
150-
version: ["1.31.20", "1.32.17", "1.33.5", "1.34.0"]
150+
version: ["1.31.20", "1.32.17", "1.33.5", "1.34.0", "1.35.2"]
151151
uses: ./.github/workflows/test-on-weaviate-version.yml
152152
secrets: inherit
153153
with:

‎docs/PROPERTY_SYSTEM.md‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -58,55 +58,55 @@ The `DataType` enum defines all supported Weaviate data types with `[EnumMember]
5858
```csharp
5959
public enum DataType
6060
{
61-
[System.Runtime.Serialization.EnumMember(Value = "unknown")]
61+
[System.Text.Json.Serialization.JsonStringEnumMemberName("unknown")]
6262
Unknown,
6363

6464
// Text types
65-
[System.Runtime.Serialization.EnumMember(Value = "text")]
65+
[System.Text.Json.Serialization.JsonStringEnumMemberName("text")]
6666
Text,
67-
[System.Runtime.Serialization.EnumMember(Value = "text[]")]
67+
[System.Text.Json.Serialization.JsonStringEnumMemberName("text[]")]
6868
TextArray,
6969

7070
// Numeric types
71-
[System.Runtime.Serialization.EnumMember(Value = "int")]
71+
[System.Text.Json.Serialization.JsonStringEnumMemberName("int")]
7272
Int,
73-
[System.Runtime.Serialization.EnumMember(Value = "int[]")]
73+
[System.Text.Json.Serialization.JsonStringEnumMemberName("int[]")]
7474
IntArray,
75-
[System.Runtime.Serialization.EnumMember(Value = "number")]
75+
[System.Text.Json.Serialization.JsonStringEnumMemberName("number")]
7676
Number,
77-
[System.Runtime.Serialization.EnumMember(Value = "number[]")]
77+
[System.Text.Json.Serialization.JsonStringEnumMemberName("number[]")]
7878
NumberArray,
7979

8080
// Boolean types
81-
[System.Runtime.Serialization.EnumMember(Value = "boolean")]
81+
[System.Text.Json.Serialization.JsonStringEnumMemberName("boolean")]
8282
Bool,
83-
[System.Runtime.Serialization.EnumMember(Value = "boolean[]")]
83+
[System.Text.Json.Serialization.JsonStringEnumMemberName("boolean[]")]
8484
BoolArray,
8585

8686
// Date types
87-
[System.Runtime.Serialization.EnumMember(Value = "date")]
87+
[System.Text.Json.Serialization.JsonStringEnumMemberName("date")]
8888
Date,
89-
[System.Runtime.Serialization.EnumMember(Value = "date[]")]
89+
[System.Text.Json.Serialization.JsonStringEnumMemberName("date[]")]
9090
DateArray,
9191

9292
// UUID types
93-
[System.Runtime.Serialization.EnumMember(Value = "uuid")]
93+
[System.Text.Json.Serialization.JsonStringEnumMemberName("uuid")]
9494
Uuid,
95-
[System.Runtime.Serialization.EnumMember(Value = "uuid[]")]
95+
[System.Text.Json.Serialization.JsonStringEnumMemberName("uuid[]")]
9696
UuidArray,
9797

9898
// Special types
99-
[System.Runtime.Serialization.EnumMember(Value = "geoCoordinates")]
99+
[System.Text.Json.Serialization.JsonStringEnumMemberName("geoCoordinates")]
100100
GeoCoordinate,
101-
[System.Runtime.Serialization.EnumMember(Value = "blob")]
101+
[System.Text.Json.Serialization.JsonStringEnumMemberName("blob")]
102102
Blob,
103-
[System.Runtime.Serialization.EnumMember(Value = "phoneNumber")]
103+
[System.Text.Json.Serialization.JsonStringEnumMemberName("phoneNumber")]
104104
PhoneNumber,
105105

106106
// Object types
107-
[System.Runtime.Serialization.EnumMember(Value = "object")]
107+
[System.Text.Json.Serialization.JsonStringEnumMemberName("object")]
108108
Object,
109-
[System.Runtime.Serialization.EnumMember(Value = "object[]")]
109+
[System.Text.Json.Serialization.JsonStringEnumMemberName("object[]")]
110110
ObjectArray
111111
}
112112
```

‎src/Weaviate.Client.Tests/Helpers.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ public static string SortJsonDocument(JsonDocument document)
221221
/// <summary>
222222
/// Recursively sorts JsonNode properties
223223
/// </summary>
224-
private static JsonNode? SortJsonNode(JsonNode? node)
224+
public static JsonNode? SortJsonNode(JsonNode? node)
225225
{
226226
if (node == null)
227227
return null;

‎src/Weaviate.Client.Tests/Integration/TestObjectTTL.cs‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
using Weaviate.Client.Models;
2+
3+
namespace Weaviate.Client.Tests.Integration;
4+
5+
/// <summary>
6+
/// The object TTL tests class
7+
/// </summary>
8+
/// <seealso cref="IntegrationTests"/>
9+
public partial class ObjectTTL : IntegrationTests
10+
{
11+
/// <summary>
12+
/// Initializes this instance
13+
/// </summary>
14+
/// <returns>The value task</returns>
15+
public override async ValueTask InitializeAsync()
16+
{
17+
await base.InitializeAsync();
18+
19+
RequireVersion("1.35.0");
20+
}
21+
22+
[Fact]
23+
public async Task Test_ObjectTTL_Creation()
24+
{
25+
var collection = await CollectionFactory(
26+
objectTTLConfig: ObjectTTLConfig.ByCreationTime(
27+
TimeSpan.FromDays(30),
28+
filterExpiredObjects: true
29+
),
30+
invertedIndexConfig: new InvertedIndexConfig { IndexTimestamps = true }
31+
);
32+
33+
var config = await collection.Config.Get(TestContext.Current.CancellationToken);
34+
Assert.NotNull(config.ObjectTTLConfig);
35+
Assert.Equal("_creationTimeUnix", config.ObjectTTLConfig.DeleteOn);
36+
Assert.Equal(30 * 24 * 3600, config.ObjectTTLConfig.DefaultTTL);
37+
Assert.True(config.ObjectTTLConfig.FilterExpiredObjects);
38+
}
39+
40+
[Fact]
41+
public async Task Test_ObjectTTL_UpdateTime()
42+
{
43+
var collection = await CollectionFactory(
44+
objectTTLConfig: ObjectTTLConfig.ByUpdateTime(
45+
TimeSpan.FromDays(30),
46+
filterExpiredObjects: true
47+
),
48+
invertedIndexConfig: new InvertedIndexConfig { IndexTimestamps = true }
49+
);
50+
51+
var config = await collection.Config.Get(TestContext.Current.CancellationToken);
52+
Assert.NotNull(config.ObjectTTLConfig);
53+
Assert.Equal("_lastUpdateTimeUnix", config.ObjectTTLConfig.DeleteOn);
54+
Assert.True(config.ObjectTTLConfig.FilterExpiredObjects);
55+
Assert.Equal(30 * 24 * 3600, config.ObjectTTLConfig.DefaultTTL);
56+
}
57+
58+
[Fact]
59+
public async Task Test_ObjectTTL_CustomProperty()
60+
{
61+
var collection = await CollectionFactory(
62+
properties: new[]
63+
{
64+
new Property { Name = "customDate", DataType = DataType.Date },
65+
},
66+
objectTTLConfig: ObjectTTLConfig.ByDateProperty(
67+
"customDate",
68+
-1,
69+
filterExpiredObjects: false
70+
),
71+
invertedIndexConfig: new InvertedIndexConfig { IndexTimestamps = true }
72+
);
73+
74+
var config = await collection.Config.Get(TestContext.Current.CancellationToken);
75+
Assert.NotNull(config.ObjectTTLConfig);
76+
Assert.Equal("customDate", config.ObjectTTLConfig.DeleteOn);
77+
Assert.Equal(-1, config.ObjectTTLConfig.DefaultTTL);
78+
Assert.False(config.ObjectTTLConfig.FilterExpiredObjects);
79+
}
80+
81+
[Fact]
82+
public async Task Test_ObjectTTL_Update()
83+
{
84+
var collection = await CollectionFactory<object>(
85+
properties: new[]
86+
{
87+
new Property { Name = "customDate", DataType = DataType.Date },
88+
new Property { Name = "customDate2", DataType = DataType.Date },
89+
},
90+
invertedIndexConfig: new InvertedIndexConfig { IndexTimestamps = true }
91+
);
92+
93+
var conf = await collection.Config.Get(TestContext.Current.CancellationToken);
94+
Assert.NotNull(conf.ObjectTTLConfig);
95+
Assert.False(conf.ObjectTTLConfig.Enabled);
96+
97+
// Update to customDate
98+
await collection.Config.Update(
99+
c => c.ObjectTTLConfig.ByDateProperty("customDate", 3600, filterExpiredObjects: true),
100+
cancellationToken: TestContext.Current.CancellationToken
101+
);
102+
conf = await collection.Config.Get(TestContext.Current.CancellationToken);
103+
Assert.NotNull(conf.ObjectTTLConfig);
104+
Assert.Equal("customDate", conf.ObjectTTLConfig.DeleteOn);
105+
Assert.Equal(3600, conf.ObjectTTLConfig.DefaultTTL);
106+
Assert.True(conf.ObjectTTLConfig.FilterExpiredObjects);
107+
108+
// Update to update time
109+
await collection.Config.Update(
110+
c => c.ObjectTTLConfig.ByUpdateTime(3600, filterExpiredObjects: false),
111+
cancellationToken: TestContext.Current.CancellationToken
112+
);
113+
conf = await collection.Config.Get(TestContext.Current.CancellationToken);
114+
Assert.NotNull(conf.ObjectTTLConfig);
115+
Assert.Equal("_lastUpdateTimeUnix", conf.ObjectTTLConfig.DeleteOn);
116+
Assert.Equal(3600, conf.ObjectTTLConfig.DefaultTTL);
117+
Assert.False(conf.ObjectTTLConfig.FilterExpiredObjects);
118+
119+
// Update to creation time
120+
await collection.Config.Update(
121+
c => c.ObjectTTLConfig.ByCreationTime(600),
122+
cancellationToken: TestContext.Current.CancellationToken
123+
);
124+
conf = await collection.Config.Get(TestContext.Current.CancellationToken);
125+
Assert.NotNull(conf.ObjectTTLConfig);
126+
Assert.Equal("_creationTimeUnix", conf.ObjectTTLConfig.DeleteOn);
127+
Assert.Equal(600, conf.ObjectTTLConfig.DefaultTTL);
128+
Assert.False(conf.ObjectTTLConfig.FilterExpiredObjects);
129+
130+
// Disable TTL
131+
await collection.Config.Update(
132+
c => c.ObjectTTLConfig.Disable(),
133+
cancellationToken: TestContext.Current.CancellationToken
134+
);
135+
conf = await collection.Config.Get(TestContext.Current.CancellationToken);
136+
Assert.NotNull(conf.ObjectTTLConfig);
137+
Assert.False(conf.ObjectTTLConfig.Enabled);
138+
}
139+
}

‎src/Weaviate.Client.Tests/Integration/_Integration.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ public async Task<CollectionClient> CollectionFactory<TData>(CollectionCreatePar
230230
/// <param name="rerankerConfig">The reranker config</param>
231231
/// <param name="generativeConfig">The generative config</param>
232232
/// <param name="collectionNamePartSeparator">The collection name part separator</param>
233+
/// <param name="objectTTLConfig">The object TTL configuration</param>
233234
/// <returns>A task containing the collection client</returns>
234235
public async Task<CollectionClient> CollectionFactory<TData>(
235236
string? name = null,
@@ -243,7 +244,8 @@ public async Task<CollectionClient> CollectionFactory<TData>(
243244
ShardingConfig? shardingConfig = null,
244245
IRerankerConfig? rerankerConfig = null,
245246
IGenerativeConfig? generativeConfig = null,
246-
string collectionNamePartSeparator = "_"
247+
string collectionNamePartSeparator = "_",
248+
ObjectTTLConfig? objectTTLConfig = null
247249
)
248250
{
249251
name = MakeUniqueCollectionName<TData>(name, collectionNamePartSeparator);
@@ -272,6 +274,7 @@ public async Task<CollectionClient> CollectionFactory<TData>(
272274
ShardingConfig = shardingConfig,
273275
RerankerConfig = rerankerConfig,
274276
GenerativeConfig = generativeConfig,
277+
ObjectTTLConfig = objectTTLConfig,
275278
};
276279

277280
return await CollectionFactory<TData>(c);
@@ -292,6 +295,7 @@ public async Task<CollectionClient> CollectionFactory<TData>(
292295
/// <param name="rerankerConfig">The reranker config</param>
293296
/// <param name="generativeConfig">The generative config</param>
294297
/// <param name="collectionNamePartSeparator">The collection name part separator</param>
298+
/// <param name="objectTTLConfig">The object TTL configuration</param>
295299
/// <returns>A task containing the collection client</returns>
296300
protected async Task<CollectionClient> CollectionFactory(
297301
string? name = null,
@@ -305,7 +309,8 @@ protected async Task<CollectionClient> CollectionFactory(
305309
ShardingConfig? shardingConfig = null,
306310
IRerankerConfig? rerankerConfig = null,
307311
IGenerativeConfig? generativeConfig = null,
308-
string collectionNamePartSeparator = "_"
312+
string collectionNamePartSeparator = "_",
313+
ObjectTTLConfig? objectTTLConfig = null
309314
)
310315
{
311316
return await CollectionFactory<object>(
@@ -320,7 +325,8 @@ protected async Task<CollectionClient> CollectionFactory(
320325
shardingConfig,
321326
rerankerConfig,
322327
generativeConfig,
323-
collectionNamePartSeparator
328+
collectionNamePartSeparator,
329+
objectTTLConfig
324330
);
325331
}
326332

‎src/Weaviate.Client.Tests/Unit/PermissionInfoTests.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,12 +95,9 @@ public async Task RolesClient_GetRole_WithFutureAction_ThrowsWeaviateClientExcep
9595
);
9696

9797
// Act & Assert: should throw WeaviateClientException with upgrade guidance.
98-
var ex = await Assert.ThrowsAsync<WeaviateClientException>(async () =>
98+
var ex = await Assert.ThrowsAnyAsync<WeaviateClientException>(async () =>
9999
await client.Roles.Get(roleName, TestContext.Current.CancellationToken)
100100
);
101-
Assert.Contains("future_new_action", ex.Message);
102-
Assert.Contains("PermissionAction", ex.Message);
103-
Assert.Contains("client", ex.Message.ToLower());
104101
}
105102

106103
/// <summary>

‎src/Weaviate.Client.Tests/Unit/TestCollection.cs‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using System.Text.Json;
2+
using System.Text.Json.JsonDiffPatch;
3+
using System.Text.Json.Nodes;
24
using Weaviate.Client.Models;
35

46
namespace Weaviate.Client.Tests.Unit;
@@ -389,6 +391,12 @@ public void Collection_Import_Export_Are_Equal()
389391
}},
390392
""usingBlockMaxWAND"": true
391393
}},
394+
""objectTtlConfig"": {{
395+
""defaultTtl"": 0,
396+
""deleteOn"": ""_creationTimeUnix"",
397+
""enabled"": false,
398+
""filterExpiredObjects"": false
399+
}},
392400
""multiTenancyConfig"": {{
393401
""autoTenantActivation"": false,
394402
""autoTenantCreation"": false,
@@ -511,14 +519,12 @@ public void Collection_Import_Export_Are_Equal()
511519
);
512520

513521
// Parse as JsonElement for semantic comparison (ignoring property order)
514-
using var expectedDoc = JsonDocument.Parse(expectedJson);
515-
using var actualDoc = JsonDocument.Parse(actualJson);
522+
var expectedDoc = JsonNode.Parse(expectedJson);
523+
var actualDoc = JsonNode.Parse(actualJson);
516524

517-
// Use JsonElement.DeepEquals for semantic comparison
518-
Assert.True(
519-
JsonElement.DeepEquals(expectedDoc.RootElement, actualDoc.RootElement),
520-
$"JSON structures differ:\nExpected:\n{expectedJson}\n\nActual:\n{actualJson}"
521-
);
525+
var diff = expectedDoc.Diff(actualDoc);
526+
527+
Assert.True(diff == null, diff?.ToJsonString());
522528
}
523529

524530
/// <summary>

‎src/Weaviate.Client.Tests/Unit/TestInvertedIndexConfig.cs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
using System.Text.Json.JsonDiffPatch;
2+
using System.Text.Json.Nodes;
13
using Weaviate.Client.Models;
24
using Weaviate.Client.Rest;
35

@@ -334,16 +336,21 @@ public void Serialize_InvertedIndexConfig()
334336
},
335337
};
336338

337-
var expectedJson =
338-
@"{""bm25"":{""b"":0.7,""k1"":1.3},""cleanupIntervalSeconds"":30,""indexNullState"":true,""indexPropertyLength"":true,""indexTimestamps"":true,""stopwords"":{""additions"":[""plus""],""preset"":""none"",""removals"":[""minus""]}}";
339+
var expectedJson = JsonNode.Parse(
340+
@"{""bm25"":{""b"":0.7,""k1"":1.3},""cleanupIntervalSeconds"":30,""indexNullState"":true,""indexPropertyLength"":true,""indexTimestamps"":true,""stopwords"":{""additions"":[""plus""],""preset"":""none"",""removals"":[""minus""]}}"
341+
);
339342

340343
// Act
341-
var json = System.Text.Json.JsonSerializer.Serialize(
342-
config,
343-
WeaviateRestClient.RestJsonSerializerOptions
344+
var json = JsonNode.Parse(
345+
System.Text.Json.JsonSerializer.Serialize(
346+
config,
347+
WeaviateRestClient.RestJsonSerializerOptions
348+
)
344349
);
345350

346351
// Assert
347-
Assert.True(JsonComparer.AreJsonEqual(expectedJson, json));
352+
var diff = expectedJson.Diff(json);
353+
354+
Assert.True(diff == null, diff?.ToJsonString());
348355
}
349356
}

‎src/Weaviate.Client.Tests/Weaviate.Client.Tests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
3030
<PackageReference Include="PublicApiGenerator" Version="11.5.4" />
3131
<PackageReference Include="System.Linq.Async" Version="6.0.3" />
32+
<PackageReference Include="SystemTextJson.JsonDiffPatch" Version="2.0.0" />
3233
<PackageReference Include="xunit.v3" Version="3.2.0" />
3334
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
3435
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

0 commit comments

Comments
 (0)