Skip to content

Commit 427a3a6

Browse files
dirkkuldirkkul
authored
Merge pull request #129 from weaviate/chore/improve-batch-inserts
Refactor batch insertion methods for improved clarity and consistency
2 parents a7dc299 + 3318f57 commit 427a3a6

File tree

14 files changed

+263
-190
lines changed

14 files changed

+263
-190
lines changed

‎src/Example/Program.cs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,9 @@ static async Task Main()
9696
if (_useBatchInsert)
9797
{
9898
// Batch Insertion Demo
99-
var batchInsertions = await collection.Data.InsertMany(add =>
100-
{
101-
cats.ForEach(c => add(c.Data, vectors: new() { { "default", c.Vector } }));
102-
});
99+
var requests = cats.Select(c => (c.Data, new Vectors { { "default", c.Vector } }));
100+
101+
var batchInsertions = await collection.Data.InsertMany(requests);
103102
}
104103
else
105104
{

‎src/Weaviate.Client.Tests/Integration/Datasets/Datasets.cs‎

Lines changed: 87 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ public static Dictionary<
265265
int expectedErrors,
266266
int expectedReferences,
267267
int expectedReferencedObjects,
268-
Action<DataClient<dynamic>.InsertDelegate>[] batcher
268+
IEnumerable<BatchInsertRequest<object>[]> data
269269
)
270270
> Cases =>
271271
new()
@@ -276,14 +276,16 @@ public static Dictionary<
276276
0,
277277
0,
278278
[
279-
add =>
280-
{
281-
add(
279+
[
280+
BatchInsertRequest.Create<object>(
282281
new { Name = "some name" },
283-
vectors: new() { { "default", 1, 2, 3 } }
284-
);
285-
add(new { Name = "some other name" }, id: _reusableUuids[0]);
286-
},
282+
vectors: Vector.Create(1, 2, 3)
283+
),
284+
BatchInsertRequest.Create<object>(
285+
new { Name = "some other name" },
286+
id: _reusableUuids[0]
287+
),
288+
],
287289
]
288290
),
289291
["all data types"] = (
@@ -292,9 +294,8 @@ public static Dictionary<
292294
0,
293295
0,
294296
[
295-
add =>
296-
{
297-
add(
297+
[
298+
BatchInsertRequest.Create<object>(
298299
new
299300
{
300301
Name = "some name",
@@ -303,8 +304,8 @@ public static Dictionary<
303304
IsAvailable = true,
304305
AvailableSince = new DateTime(2023, 1, 1),
305306
}
306-
);
307-
},
307+
),
308+
],
308309
]
309310
),
310311
["wrong type for property"] = (
@@ -313,10 +314,7 @@ public static Dictionary<
313314
0,
314315
0,
315316
[
316-
add =>
317-
{
318-
add(new { Name = 1 });
319-
},
317+
[BatchInsertRequest.Create<object>(new { Name = 1 })],
320318
]
321319
),
322320
["batch with self-reference"] = (
@@ -325,20 +323,30 @@ public static Dictionary<
325323
1,
326324
1,
327325
[
328-
add =>
329-
{
330-
add(new { Name = "Name 1" }, id: _reusableUuids[0]);
331-
add(new { Name = "Name 2" }, id: _reusableUuids[1]);
332-
add(new { Name = "Name 3" }, id: _reusableUuids[2]);
333-
add(new { Name = "Name 4" }, id: _reusableUuids[3]);
334-
},
335-
add =>
336-
{
337-
add(
326+
[
327+
BatchInsertRequest.Create<object>(
328+
new { Name = "Name 1" },
329+
id: _reusableUuids[0]
330+
),
331+
BatchInsertRequest.Create<object>(
332+
new { Name = "Name 2" },
333+
id: _reusableUuids[1]
334+
),
335+
BatchInsertRequest.Create<object>(
336+
new { Name = "Name 3" },
337+
id: _reusableUuids[2]
338+
),
339+
BatchInsertRequest.Create<object>(
340+
new { Name = "Name 4" },
341+
id: _reusableUuids[3]
342+
),
343+
],
344+
[
345+
BatchInsertRequest.Create<object>(
338346
new { Name = "Name 5" },
339-
references: [new ObjectReference("ref", _reusableUuids[1])]
340-
);
341-
},
347+
references: [new("ref", _reusableUuids[1])]
348+
),
349+
],
342350
]
343351
),
344352
["batch with multiple self-references"] = (
@@ -347,27 +355,30 @@ public static Dictionary<
347355
1,
348356
2,
349357
[
350-
add =>
351-
{
352-
add(new { Name = "Name 1" }, id: _reusableUuids[0]);
353-
add(new { Name = "Name 2" }, id: _reusableUuids[1]);
354-
add(new { Name = "Name 3" }, id: _reusableUuids[2]);
355-
add(new { Name = "Name 4" }, id: _reusableUuids[3]);
356-
},
357-
add =>
358-
{
359-
add(
358+
[
359+
BatchInsertRequest.Create<object>(
360+
new { Name = "Name 1" },
361+
id: _reusableUuids[0]
362+
),
363+
BatchInsertRequest.Create<object>(
364+
new { Name = "Name 2" },
365+
id: _reusableUuids[1]
366+
),
367+
BatchInsertRequest.Create<object>(
368+
new { Name = "Name 3" },
369+
id: _reusableUuids[2]
370+
),
371+
BatchInsertRequest.Create<object>(
372+
new { Name = "Name 4" },
373+
id: _reusableUuids[3]
374+
),
375+
],
376+
[
377+
BatchInsertRequest.Create<object>(
360378
new { Name = "Name 5" },
361-
references:
362-
[
363-
new ObjectReference(
364-
"ref",
365-
_reusableUuids[1],
366-
_reusableUuids[2]
367-
),
368-
]
369-
);
370-
},
379+
references: [new("ref", _reusableUuids[1], _reusableUuids[2])]
380+
),
381+
],
371382
]
372383
),
373384
["batch with multiple self-reference properties"] = (
@@ -376,32 +387,42 @@ public static Dictionary<
376387
3,
377388
4,
378389
[
379-
add =>
380-
{
381-
add(new { Name = "Name 1" }, id: _reusableUuids[0]);
382-
add(new { Name = "Name 2" }, id: _reusableUuids[1]);
383-
add(new { Name = "Name 3" }, id: _reusableUuids[2]);
384-
add(new { Name = "Name 4" }, id: _reusableUuids[3]);
385-
},
386-
add =>
387-
{
388-
add(
390+
[
391+
BatchInsertRequest.Create<object>(
392+
new { Name = "Name 1" },
393+
id: _reusableUuids[0]
394+
),
395+
BatchInsertRequest.Create<object>(
396+
new { Name = "Name 2" },
397+
id: _reusableUuids[1]
398+
),
399+
BatchInsertRequest.Create<object>(
400+
new { Name = "Name 3" },
401+
id: _reusableUuids[2]
402+
),
403+
BatchInsertRequest.Create<object>(
404+
new { Name = "Name 4" },
405+
id: _reusableUuids[3]
406+
),
407+
],
408+
[
409+
BatchInsertRequest.Create<object>(
389410
new { Name = "Name 5" },
390411
references: [new("ref", _reusableUuids[1])]
391-
);
392-
add(
412+
),
413+
BatchInsertRequest.Create<object>(
393414
new { Name = "Name 6" },
394415
references: [new("ref2", _reusableUuids[2])]
395-
);
396-
add(
416+
),
417+
BatchInsertRequest.Create<object>(
397418
new { Name = "Name 7" },
398419
references:
399420
[
400421
new("ref", _reusableUuids[1]),
401422
new("ref2", _reusableUuids[2]),
402423
]
403-
);
404-
},
424+
),
425+
],
405426
]
406427
),
407428
};

‎src/Weaviate.Client.Tests/Integration/TestBatch.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public async Task InsertMany(string key)
1414
int expectedErrors,
1515
int expectedReferences,
1616
int expectedReferencedObjects,
17-
Action<DataClient<dynamic>.InsertDelegate>[] batcher
17+
IEnumerable<BatchInsertRequest<object>[]> requests
1818
) = DatasetBatchInsertMany.Cases[key];
1919

2020
var client = await CollectionFactory(
@@ -32,7 +32,7 @@ public async Task InsertMany(string key)
3232
await client.Config.AddReference(new Reference("ref", client.Name));
3333
await client.Config.AddReference(new Reference("ref2", client.Name));
3434

35-
var result = await client.Data.InsertMany(batcher);
35+
var result = await client.Data.InsertMany(requests);
3636

3737
var data = await client.Query.FetchObjects(returnReferences: [new("ref"), new("ref2")]);
3838

@@ -57,8 +57,8 @@ public async Task Test_Batch_ReferenceAddMany()
5757
int numObjects = 10;
5858

5959
// Insert objects into the referenced collection and get their UUIDs
60-
var refInsertResult = await refCollection.Data.InsertMany(add =>
61-
Enumerable.Range(0, numObjects).ToList().ForEach(i => add(new { Number = i }))
60+
var refInsertResult = await refCollection.Data.InsertMany(
61+
Enumerable.Range(0, numObjects).Select(i => new { Number = i })
6262
);
6363

6464
Guid[] uuidsTo = [.. refInsertResult.Select(r => r.ID!.Value)];
@@ -72,8 +72,8 @@ public async Task Test_Batch_ReferenceAddMany()
7272
);
7373

7474
// Insert objects into the main collection and get their UUIDs
75-
var fromInsertResult = await collection.Data.InsertMany(add =>
76-
Enumerable.Range(0, numObjects).ToList().ForEach(i => add(new { Num = i }))
75+
var fromInsertResult = await collection.Data.InsertMany(
76+
Enumerable.Range(0, numObjects).Select(i => new { Num = i })
7777
);
7878

7979
Guid[] uuidsFrom = [.. fromInsertResult.Select(r => r.ID!.Value)];

‎src/Weaviate.Client.Tests/Integration/TestBatchDelete.cs‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,7 @@ public async Task Test_Delete_Many_Return()
1313
vectorConfig: new VectorConfig("default", new Vectorizer.SelfProvided())
1414
);
1515

16-
await collection.Data.InsertMany(batcher =>
17-
{
18-
batcher(new { name = "delet me" }, Guid.NewGuid());
19-
});
16+
await collection.Data.InsertMany((new { name = "delet me" }, Guid.NewGuid()));
2017

2118
var result = await collection.Data.DeleteMany(
2219
where: Filter.Property("name").Equal("delet me")
@@ -36,12 +33,11 @@ public async Task Test_Delete_Many_Or()
3633
vectorConfig: new VectorConfig("default", new Vectorizer.SelfProvided())
3734
);
3835

39-
await collection.Data.InsertMany(batcher =>
40-
{
41-
batcher(new { age = 10, name = "Timmy" }, Guid.NewGuid());
42-
batcher(new { age = 20, name = "Tim" }, Guid.NewGuid());
43-
batcher(new { age = 30, name = "Timothy" }, Guid.NewGuid());
44-
});
36+
await collection.Data.InsertMany(
37+
(new { age = 10, name = "Timmy" }, Guid.NewGuid()),
38+
(new { age = 20, name = "Tim" }, Guid.NewGuid()),
39+
(new { age = 30, name = "Timothy" }, Guid.NewGuid())
40+
);
4541

4642
var objects = await collection.Query.FetchObjects();
4743
Assert.Equal(3, objects.Objects.Count());
@@ -64,11 +60,10 @@ public async Task Test_Delete_Many_And()
6460
vectorConfig: new VectorConfig("default", new Vectorizer.SelfProvided())
6561
);
6662

67-
await collection.Data.InsertMany(batcher =>
68-
{
69-
batcher(new { age = 10, name = "Timmy" }, Guid.NewGuid());
70-
batcher(new { age = 10, name = "Tommy" }, Guid.NewGuid());
71-
});
63+
await collection.Data.InsertMany(
64+
(new { age = 10, name = "Timmy" }, Guid.NewGuid()),
65+
(new { age = 10, name = "Tommy" }, Guid.NewGuid())
66+
);
7267

7368
var objects = await collection.Query.FetchObjects();
7469
Assert.Equal(2, objects.Objects.Count());

‎src/Weaviate.Client.Tests/Integration/TestCollectionAggregate.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ public async Task Test_Collection_Length(int howMany)
1717
vectorConfig: Configure.Vectors.SelfProvided()
1818
);
1919

20-
await collectionClient.Data.InsertMany([.. Enumerable.Repeat(new { }, howMany)]);
20+
await collectionClient.Data.InsertMany(
21+
Enumerable.Repeat(BatchInsertRequest.Create<object>(new { }), howMany)
22+
);
2123

2224
var count = await collectionClient.Count();
2325

@@ -593,7 +595,7 @@ public async Task Test_Simple_Aggregation_AllTypes(string propertyType, object[]
593595
_ => throw new ArgumentException("Unknown property type"),
594596
};
595597

596-
await collectionClient.Data.InsertMany(insertObj);
598+
await collectionClient.Data.InsertMany(BatchInsertRequest.Create(insertObj));
597599

598600
Aggregate.Metric metric = propertyType switch
599601
{

‎src/Weaviate.Client.Tests/Integration/TestCollections.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public async Task CollectionClient_Creates_And_Retrieves_Collection()
1919
);
2020

2121
// Assert
22-
var collection = await _weaviate.Collections.Use<dynamic>(collectionClient.Name).Get();
22+
var collection = await _weaviate.Collections.Use(collectionClient.Name).Get();
2323
Assert.NotNull(collection);
2424
Assert.Equal(
2525
$"CollectionClient_Creates_And_Retrieves_Collection_{TestContext.Current.Test?.UniqueID}_Object_RandomCollectionName",
@@ -935,7 +935,9 @@ public async Task Test_Return_Blob_Property()
935935
var uuid = await collection.Data.Insert(new { blob = blobData });
936936

937937
// Insert many
938-
await collection.Data.InsertMany(new[] { new { blob = blobData } });
938+
await collection.Data.InsertMany(
939+
BatchInsertRequest.Create<object>(new { blob = blobData })
940+
);
939941

940942
// Fetch by id
941943
var obj = await collection.Query.FetchObjectByID(uuid, returnProperties: new[] { "blob" });

0 commit comments

Comments
 (0)