Skip to content

Commit f958238

Browse files
jparismorganjparismorgan
authored
Update FeatureVectorArray to be able to read from URIs at a specified timestamp (#353)
1 parent 23f201a commit f958238

File tree

2 files changed

+194
-136
lines changed

2 files changed

+194
-136
lines changed

‎src/include/api/feature_vector_array.h‎

Lines changed: 60 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,13 @@ class FeatureVectorArray {
8383
const tiledb::Context& ctx,
8484
const std::string& uri,
8585
const std::string& ids_uri = "",
86-
size_t num_vectors = 0) {
87-
auto array = tiledb_helpers::open_array(tdb_func__, ctx, uri, TILEDB_READ);
86+
size_t num_vectors = 0,
87+
size_t timestamp = 0) {
88+
auto temporal_policy =
89+
timestamp == 0 ? tiledb::TemporalPolicy() :
90+
tiledb::TemporalPolicy(tiledb::TimeTravel, timestamp);
91+
auto array = tiledb_helpers::open_array(
92+
tdb_func__, ctx, uri, TILEDB_READ, temporal_policy);
8893
feature_type_ = get_array_datatype(*array);
8994
array->close(); // @todo create Matrix constructor that takes opened array
9095
feature_size_ = datatype_to_size(feature_type_);
@@ -102,7 +107,7 @@ class FeatureVectorArray {
102107
throw std::runtime_error("Unsupported features attribute type");
103108
}
104109
vector_array = tdb_col_major_matrix_dispatch_table.at(feature_type_)(
105-
ctx, uri, num_vectors);
110+
ctx, uri, num_vectors, timestamp);
106111
} else {
107112
auto ids_array =
108113
tiledb_helpers::open_array(tdb_func__, ctx, ids_uri, TILEDB_READ);
@@ -117,7 +122,7 @@ class FeatureVectorArray {
117122
"Unsupported attribute type for feature vector with ids");
118123
}
119124
vector_array = tdb_col_major_matrix_with_ids_dispatch_table.at(type)(
120-
ctx, uri, ids_uri, num_vectors);
125+
ctx, uri, ids_uri, num_vectors, timestamp);
121126
}
122127
(void)vector_array->load();
123128
}
@@ -233,15 +238,19 @@ class FeatureVectorArray {
233238
// : impl_vector_array(t) {
234239
}
235240
vector_array_impl(
236-
const tiledb::Context& ctx, const std::string& uri, size_t num_vectors)
237-
: impl_vector_array(ctx, uri, num_vectors) {
241+
const tiledb::Context& ctx,
242+
const std::string& uri,
243+
size_t num_vectors,
244+
size_t timestamp)
245+
: impl_vector_array(ctx, uri, num_vectors, timestamp) {
238246
}
239247
vector_array_impl(
240248
const tiledb::Context& ctx,
241249
const std::string& uri,
242250
const std::string& ids_uri,
243-
size_t num_vectors)
244-
: impl_vector_array(ctx, uri, ids_uri, num_vectors) {
251+
size_t num_vectors,
252+
size_t timestamp)
253+
: impl_vector_array(ctx, uri, ids_uri, num_vectors, timestamp) {
245254
}
246255
vector_array_impl(size_t rows, size_t cols)
247256
: impl_vector_array(rows, cols) {
@@ -273,39 +282,23 @@ class FeatureVectorArray {
273282
};
274283

275284
private:
276-
using col_major_matrix_constructor_function =
277-
std::function<std::unique_ptr<vector_array_base>(size_t, size_t)>;
278-
using col_major_matrix_table_type =
279-
std::map<tiledb_datatype_t, col_major_matrix_constructor_function>;
285+
// clang-format off
286+
using col_major_matrix_constructor_function = std::function<std::unique_ptr<vector_array_base>(size_t, size_t)>;
287+
using col_major_matrix_table_type = std::map<tiledb_datatype_t, col_major_matrix_constructor_function>;
280288
static const col_major_matrix_table_type col_major_matrix_dispatch_table;
281289

282-
using tdb_col_major_matrix_constructor_function =
283-
std::function<std::unique_ptr<vector_array_base>(
284-
const tiledb::Context&, const std::string&, size_t)>;
285-
using tdb_col_major_matrix_table_type =
286-
std::map<tiledb_datatype_t, tdb_col_major_matrix_constructor_function>;
287-
static const tdb_col_major_matrix_table_type
288-
tdb_col_major_matrix_dispatch_table;
289-
290-
using col_major_matrix_with_ids_constructor_function =
291-
std::function<std::unique_ptr<vector_array_base>(size_t, size_t)>;
292-
using col_major_matrix_with_ids_table_type = std::map<
293-
std::tuple<tiledb_datatype_t, tiledb_datatype_t>,
294-
col_major_matrix_with_ids_constructor_function>;
295-
static const col_major_matrix_with_ids_table_type
296-
col_major_matrix_with_ids_dispatch_table;
297-
298-
using tdb_col_major_matrix_with_ids_constructor_function =
299-
std::function<std::unique_ptr<vector_array_base>(
300-
const tiledb::Context&,
301-
const std::string&,
302-
const std::string&,
303-
size_t)>;
304-
using tdb_col_major_matrix_with_ids_table_type = std::map<
305-
std::tuple<tiledb_datatype_t, tiledb_datatype_t>,
306-
tdb_col_major_matrix_with_ids_constructor_function>;
307-
static const tdb_col_major_matrix_with_ids_table_type
308-
tdb_col_major_matrix_with_ids_dispatch_table;
290+
using tdb_col_major_matrix_constructor_function = std::function<std::unique_ptr<vector_array_base>(const tiledb::Context&, const std::string&, size_t, size_t)>;
291+
using tdb_col_major_matrix_table_type = std::map<tiledb_datatype_t, tdb_col_major_matrix_constructor_function>;
292+
static const tdb_col_major_matrix_table_type tdb_col_major_matrix_dispatch_table;
293+
294+
using col_major_matrix_with_ids_constructor_function = std::function<std::unique_ptr<vector_array_base>(size_t, size_t)>;
295+
using col_major_matrix_with_ids_table_type = std::map<std::tuple<tiledb_datatype_t, tiledb_datatype_t>, col_major_matrix_with_ids_constructor_function>;
296+
static const col_major_matrix_with_ids_table_type col_major_matrix_with_ids_dispatch_table;
297+
298+
using tdb_col_major_matrix_with_ids_constructor_function = std::function<std::unique_ptr<vector_array_base>(const tiledb::Context&, const std::string&, const std::string&, size_t, size_t)>;
299+
using tdb_col_major_matrix_with_ids_table_type = std::map<std::tuple<tiledb_datatype_t, tiledb_datatype_t>, tdb_col_major_matrix_with_ids_constructor_function>;
300+
static const tdb_col_major_matrix_with_ids_table_type tdb_col_major_matrix_with_ids_dispatch_table;
301+
// clang-format on
309302

310303
tiledb_datatype_t feature_type_{TILEDB_ANY};
311304
size_t feature_size_{0};
@@ -317,94 +310,25 @@ class FeatureVectorArray {
317310
std::unique_ptr</*const*/ vector_array_base> vector_array;
318311
};
319312

320-
const FeatureVectorArray::col_major_matrix_table_type
321-
FeatureVectorArray::col_major_matrix_dispatch_table = {
322-
{TILEDB_FLOAT32,
323-
[](size_t rows, size_t cols) {
324-
return std::make_unique<
325-
FeatureVectorArray::vector_array_impl<ColMajorMatrix<float>>>(
326-
rows, cols);
327-
}},
328-
{TILEDB_UINT8,
329-
[](size_t rows, size_t cols) {
330-
return std::make_unique<
331-
FeatureVectorArray::vector_array_impl<ColMajorMatrix<uint8_t>>>(
332-
rows, cols);
333-
}},
334-
{TILEDB_INT32,
335-
[](size_t rows, size_t cols) {
336-
return std::make_unique<
337-
FeatureVectorArray::vector_array_impl<ColMajorMatrix<int32_t>>>(
338-
rows, cols);
339-
}},
340-
{TILEDB_UINT32,
341-
[](size_t rows, size_t cols) {
342-
return std::make_unique<
343-
FeatureVectorArray::vector_array_impl<ColMajorMatrix<uint32_t>>>(
344-
rows, cols);
345-
}},
346-
{TILEDB_INT64,
347-
[](size_t rows, size_t cols) {
348-
return std::make_unique<
349-
FeatureVectorArray::vector_array_impl<ColMajorMatrix<int64_t>>>(
350-
rows, cols);
351-
}},
352-
{TILEDB_UINT64,
353-
[](size_t rows, size_t cols) {
354-
return std::make_unique<
355-
FeatureVectorArray::vector_array_impl<ColMajorMatrix<uint64_t>>>(
356-
rows, cols);
357-
}},
313+
// clang-format off
314+
const FeatureVectorArray::col_major_matrix_table_type FeatureVectorArray::col_major_matrix_dispatch_table = {
315+
{TILEDB_FLOAT32, [](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrix<float >>>(rows, cols);}},
316+
{TILEDB_UINT8, [](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrix<uint8_t >>>(rows, cols);}},
317+
{TILEDB_INT32, [](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrix<int32_t >>>(rows, cols);}},
318+
{TILEDB_UINT32, [](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrix<uint32_t>>>(rows, cols);}},
319+
{TILEDB_INT64, [](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrix<int64_t >>>(rows, cols);}},
320+
{TILEDB_UINT64, [](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrix<uint64_t>>>(rows, cols);}},
358321
};
359322

360-
const FeatureVectorArray::tdb_col_major_matrix_table_type
361-
FeatureVectorArray::tdb_col_major_matrix_dispatch_table = {
362-
{TILEDB_FLOAT32,
363-
[](const tiledb::Context& ctx,
364-
const std::string& uri,
365-
size_t num_vectors) {
366-
return std::make_unique<
367-
FeatureVectorArray::vector_array_impl<tdbColMajorMatrix<float>>>(
368-
ctx, uri, num_vectors);
369-
}},
370-
{TILEDB_UINT8,
371-
[](const tiledb::Context& ctx,
372-
const std::string& uri,
373-
size_t num_vectors) {
374-
return std::make_unique<FeatureVectorArray::vector_array_impl<
375-
tdbColMajorMatrix<uint8_t>>>(ctx, uri, num_vectors);
376-
}},
377-
{TILEDB_INT32,
378-
[](const tiledb::Context& ctx,
379-
const std::string& uri,
380-
size_t num_vectors) {
381-
return std::make_unique<FeatureVectorArray::vector_array_impl<
382-
tdbColMajorMatrix<int32_t>>>(ctx, uri, num_vectors);
383-
}},
384-
{TILEDB_UINT32,
385-
[](const tiledb::Context& ctx,
386-
const std::string& uri,
387-
size_t num_vectors) {
388-
return std::make_unique<FeatureVectorArray::vector_array_impl<
389-
tdbColMajorMatrix<uint32_t>>>(ctx, uri, num_vectors);
390-
}},
391-
{TILEDB_INT64,
392-
[](const tiledb::Context& ctx,
393-
const std::string& uri,
394-
size_t num_vectors) {
395-
return std::make_unique<FeatureVectorArray::vector_array_impl<
396-
tdbColMajorMatrix<int64_t>>>(ctx, uri, num_vectors);
397-
}},
398-
{TILEDB_UINT64,
399-
[](const tiledb::Context& ctx,
400-
const std::string& uri,
401-
size_t num_vectors) {
402-
return std::make_unique<FeatureVectorArray::vector_array_impl<
403-
tdbColMajorMatrix<uint64_t>>>(ctx, uri, num_vectors);
404-
}},
323+
const FeatureVectorArray::tdb_col_major_matrix_table_type FeatureVectorArray::tdb_col_major_matrix_dispatch_table = {
324+
{TILEDB_FLOAT32, [](const tiledb::Context& ctx, const std::string& uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrix<float >>>(ctx, uri, num_vectors, timestamp); }},
325+
{TILEDB_UINT8, [](const tiledb::Context& ctx, const std::string& uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrix<uint8_t >>>(ctx, uri, num_vectors, timestamp); }},
326+
{TILEDB_INT32, [](const tiledb::Context& ctx, const std::string& uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrix<int32_t >>>(ctx, uri, num_vectors, timestamp); }},
327+
{TILEDB_UINT32, [](const tiledb::Context& ctx, const std::string& uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrix<uint32_t>>>(ctx, uri, num_vectors, timestamp); }},
328+
{TILEDB_INT64, [](const tiledb::Context& ctx, const std::string& uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrix<int64_t >>>(ctx, uri, num_vectors, timestamp); }},
329+
{TILEDB_UINT64, [](const tiledb::Context& ctx, const std::string& uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrix<uint64_t>>>(ctx, uri, num_vectors, timestamp); }},
405330
};
406331

407-
// clang-format off
408332
const FeatureVectorArray::col_major_matrix_with_ids_table_type FeatureVectorArray::col_major_matrix_with_ids_dispatch_table = {
409333
{{TILEDB_FLOAT32, TILEDB_UINT32},[](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrixWithIds<float, uint32_t>>>(rows, cols); }},
410334
{{TILEDB_UINT8, TILEDB_UINT32},[](size_t rows, size_t cols) { return std::make_unique<FeatureVectorArray::vector_array_impl<ColMajorMatrixWithIds<uint8_t, uint32_t>>>(rows, cols); }},
@@ -422,19 +346,19 @@ const FeatureVectorArray::col_major_matrix_with_ids_table_type FeatureVectorArra
422346
};
423347

424348
const FeatureVectorArray::tdb_col_major_matrix_with_ids_table_type FeatureVectorArray::tdb_col_major_matrix_with_ids_dispatch_table = {
425-
{{TILEDB_FLOAT32, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<float, uint32_t>>>(ctx, uri, ids_uri, num_vectors);}},
426-
{{TILEDB_UINT8, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint8_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors);}},
427-
{{TILEDB_INT32, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int32_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors);}},
428-
{{TILEDB_UINT32, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint32_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors);}},
429-
{{TILEDB_INT64, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int64_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors);}},
430-
{{TILEDB_UINT64, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint64_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors);}},
431-
432-
{{TILEDB_FLOAT32, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<float, uint64_t>>>(ctx, uri, ids_uri, num_vectors);}},
433-
{{TILEDB_UINT8, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint8_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors);}},
434-
{{TILEDB_INT32, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int32_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors);}},
435-
{{TILEDB_UINT32, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint32_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors);}},
436-
{{TILEDB_INT64, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int64_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors);}},
437-
{{TILEDB_UINT64, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint64_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors);}},
349+
{{TILEDB_FLOAT32, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<float, uint32_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
350+
{{TILEDB_UINT8, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint8_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
351+
{{TILEDB_INT32, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int32_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
352+
{{TILEDB_UINT32, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint32_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
353+
{{TILEDB_INT64, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int64_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
354+
{{TILEDB_UINT64, TILEDB_UINT32},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint64_t, uint32_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
355+
356+
{{TILEDB_FLOAT32, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<float, uint64_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
357+
{{TILEDB_UINT8, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) { return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint8_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
358+
{{TILEDB_INT32, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int32_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
359+
{{TILEDB_UINT32, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint32_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
360+
{{TILEDB_INT64, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<int64_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
361+
{{TILEDB_UINT64, TILEDB_UINT64},[](const tiledb::Context& ctx, const std::string& uri, const std::string& ids_uri, size_t num_vectors, size_t timestamp) {return std::make_unique<FeatureVectorArray::vector_array_impl<tdbColMajorMatrixWithIds<uint64_t, uint64_t>>>(ctx, uri, ids_uri, num_vectors, timestamp);}},
438362
};
439363
// clang-format on
440364

0 commit comments

Comments
 (0)