Remove array_types.h [skip ci]

Author: lums658

src/include/array_types.h

@@ -41,10 +41,13 @@
 #include "flat_query.h"
 #include "linalg.h"
 
-extern double global_time_of_interest;
 
 namespace detail::ivf {
-template <typename T = shuffled_db_type>
+
+/**
+ * Overload for already opened arrays.  Since the array is already opened, we don't need
+ * to specify its type with a template parameter.
+ */
 auto qv_query_heap_infinite_ram(
     auto&& shuffled_db,
     auto&& centroids,
@@ -56,7 +59,12 @@ auto qv_query_heap_infinite_ram(
     bool nth,
     size_t nthreads);
 
-template <typename T = shuffled_db_type>
+/**
+ * Overload for case where we need to open the vector and id arrays.  We can't
+ * do any template argument deduction here because we need to know the type of
+ * the vector array.
+ */
+template <typename T>
 auto qv_query_heap_infinite_ram(
     tiledb::Context& ctx,
     const std::string& part_uri,
@@ -69,13 +77,15 @@ auto qv_query_heap_infinite_ram(
     bool nth,
     size_t nthreads);
 
+
 /**
  * @brief Query a (small) set of query vectors against a vector database.
  * This version loads the entire partition array into memory and then
  * queries each vector in the query set against the appropriate partitions.
+ *
+ * For now that type of the array needs to be passed as a template argument.
  */
-
-template <typename T>
+template <typename T, class shuffled_ids_type>
 auto qv_query_heap_infinite_ram(
     tiledb::Context& ctx,
     const std::string& part_uri,
@@ -95,7 +105,7 @@ auto qv_query_heap_infinite_ram(
   auto shuffled_db = tdbColMajorMatrix<T>(ctx, part_uri);
   auto shuffled_ids = read_vector<shuffled_ids_type>(ctx, id_uri);
 
-  return qv_query_heap_infinite_ram<T>(
+  return qv_query_heap_infinite_ram(
       shuffled_db,
       centroids,
       q,
@@ -107,7 +117,7 @@ auto qv_query_heap_infinite_ram(
       nthreads);
 }
 
-template <typename T = shuffled_db_type>
+
 auto qv_query_heap_infinite_ram(
     const std::string& part_uri,
     auto&& centroids,
@@ -119,7 +129,7 @@ auto qv_query_heap_infinite_ram(
     bool nth,
     size_t nthreads) {
   tiledb::Context ctx;
-  return qv_query_heap_infinite_ram<T>(
+  return qv_query_heap_infinite_ram(
       ctx,
       part_uri,
       centroids,
@@ -132,7 +142,6 @@ auto qv_query_heap_infinite_ram(
       nthreads);
 }
 
-template <typename T>
 auto qv_query_heap_infinite_ram(
     auto&& shuffled_db,
     auto&& centroids,
@@ -209,7 +218,7 @@ auto qv_query_heap_infinite_ram(
   return top_k;
 }
 
-template <typename T = shuffled_db_type>
+template <typename T, class shuffled_ids_type>
 auto qv_query_heap_finite_ram(
     tiledb::Context& ctx,
     const std::string& part_uri,
@@ -224,6 +233,9 @@ auto qv_query_heap_finite_ram(
     size_t nthreads) {
   scoped_timer _{tdb_func__};
 
+  using parts_type = typename std::remove_reference_t<decltype(centroids)>::value_type;
+  using indices_type = typename std::remove_reference_t<decltype(indices)>::value_type;
+
   size_t num_queries = size(q);
 
   // get closest centroid for each query vector
@@ -275,7 +287,7 @@ auto qv_query_heap_finite_ram(
                          indices[active_partitions[i]];
   }
 
-  auto shuffled_db = tdbColMajorPartitionedMatrix<T>(
+  auto shuffled_db = tdbColMajorPartitionedMatrix<T, shuffled_ids_type, indices_type, parts_type>(
       ctx,
       part_uri,
       std::move(indices),

src/include/detail/ivf/qv.h

@@ -59,6 +59,7 @@ class tdbMatrix : public Matrix<T, LayoutPolicy, I> {
   using Base::Base;
 
  public:
+  using value_type = typename Base::value_type;
   using index_type = typename Base::index_type;
   using size_type = typename Base::size_type;
   using reference = typename Base::reference;

src/include/detail/linalg/tdb_matrix.h

@@ -27,12 +27,12 @@
  *
  * @section DESCRIPTION
  *
- * Class the provides a matrix view to a partitioned TileDB array (as partitioned by
- * IVF indexing).
+ * Class the provides a matrix view to a partitioned TileDB array (as
+ * partitioned by IVF indexing).
  *
  * The class requires the URI of a partitioned TileDB array and partioned set of
- * vector identifiers.  The class will provide a view of the requested partitions
- * and the corresponding vector identifiers.
+ * vector identifiers.  The class will provide a view of the requested
+ * partitions and the corresponding vector identifiers.
  *
  * Also provides support for out-of-core operation.
  *
@@ -55,7 +55,6 @@
 
 #include "detail/linalg/tdb_defs.h"
 
-#include "array_types.h"
 #include "utils/timer.h"
 
 namespace stdx {
@@ -67,7 +66,20 @@ extern bool global_verbose;
 extern bool global_debug;
 extern std::string global_region;
 
-template <class T, class LayoutPolicy = stdx::layout_right, class I = size_t>
+/**
+ *
+ * @note The template parameters indices_type and parts_type are deduced using
+ * CTAD.  However, with the uri-based constructor, the type of the indices and
+ * the shuffled_db array cannot be deduced.  Therefore, the user must specify
+ * the type of the indices and the shuffled_ids array.
+ */
+template <
+    class T,
+    class shuffled_ids_type,
+    class indices_type,
+    class parts_type,
+    class LayoutPolicy = stdx::layout_right,
+    class I = size_t>
 class tdbPartitionedMatrix : public Matrix<T, LayoutPolicy, I> {
   /****************************************************************************
    *
@@ -120,9 +132,9 @@ class tdbPartitionedMatrix : public Matrix<T, LayoutPolicy, I> {
    ****************************************************************************/
   tiledb::Array ids_array_;
   tiledb::ArraySchema ids_schema_;
-  std::vector<shuffled_ids_type> indices_;  // @todo pointer and span?
-  std::vector<parts_type> parts_;           // @todo pointer and span?
-  std::vector<shuffled_ids_type> ids_;      // @todo pointer and span?
+  std::vector<indices_type> indices_;   // @todo pointer and span?
+  std::vector<parts_type> parts_;       // @todo pointer and span?
+  std::vector<shuffled_ids_type> ids_;  // @todo pointer and span?
 
   std::tuple<index_type, index_type> row_part_view_;
   std::tuple<index_type, index_type> col_part_view_;
@@ -174,7 +186,6 @@ class tdbPartitionedMatrix : public Matrix<T, LayoutPolicy, I> {
       , parts_{in_parts}
       , row_part_view_{0, 0}
       , col_part_view_{0, 0} {
-
     constructor_timer.stop();
 
     total_num_parts_ = size(parts_);
@@ -268,7 +279,7 @@ class tdbPartitionedMatrix : public Matrix<T, LayoutPolicy, I> {
        */
       std::get<0>(col_view_) = std::get<1>(col_view_);  // # columns
       std::get<0>(col_part_view_) =
-          std::get<1>(col_part_view_);  // # partitions
+          std::get<1>(col_part_view_);                  // # partitions
 
       std::get<1>(col_part_view_) = std::get<0>(col_part_view_);
       for (size_t i = std::get<0>(col_part_view_); i < total_num_parts_; ++i) {
@@ -420,15 +431,25 @@ class tdbPartitionedMatrix : public Matrix<T, LayoutPolicy, I> {
 /**
  * Convenience class for row-major matrices.
  */
-template <class T, class I = size_t>
+template <
+    class T,
+    class shuffled_ids_type,
+    class indices_type,
+    class parts_type,
+    class I = size_t>
 using tdbRowMajorPartitionedMatrix =
-    tdbPartitionedMatrix<T, stdx::layout_right, I>;
+    tdbPartitionedMatrix<T, shuffled_ids_type, indices_type, parts_type, stdx::layout_right, I>;
 
 /**
  * Convenience class for column-major matrices.
  */
-template <class T, class I = size_t>
+template <
+    class T,
+    class shuffled_ids_type,
+    class indices_type,
+    class parts_type,
+    class I = size_t>
 using tdbColMajorPartitionedMatrix =
-    tdbPartitionedMatrix<T, stdx::layout_left, I>;
+    tdbPartitionedMatrix<T, shuffled_ids_type, indices_type, parts_type, stdx::layout_left, I>;
 
 #endif  // TILEDB_PARTITIONED_MATRIX_H

src/include/detail/linalg/tdb_partitioned_matrix.h

@@ -65,7 +65,6 @@
 
 #include <docopt.h>
 
-#include "array_types.h"
 #include "config.h"
 #include "defs.h"
 #include "ivf_query.h"
@@ -81,6 +80,27 @@ using json = nlohmann::json;
 bool global_verbose = false;
 bool global_debug = false;
 
+
+#include <cstdint>
+
+/**
+ * Specify some types for the demo.
+ */
+#if 1
+using db_type = uint8_t;
+#else
+using db_type = float;
+#endif
+
+using groundtruth_type = int32_t;
+using centroids_type = float;
+
+using shuffled_ids_type = uint64_t;
+
+// @todo Are these the same?
+using indices_type = uint64_t;
+using parts_type = uint64_t;
+
 static constexpr const char USAGE[] =
     R"(ivf_hack: demo hack feature vector search with kmeans index.
 Usage:
@@ -171,12 +191,12 @@ int main(int argc, char* argv[]) {
     auto indices = read_vector<indices_type>(ctx, index_uri);
     debug_matrix(indices, "indices");
 
-    auto q = tdbColMajorMatrix<q_type>(ctx, query_uri, nqueries);
+    auto q = tdbColMajorMatrix<db_type, shuffled_ids_type>(ctx, query_uri, nqueries);
     debug_matrix(q, "q");
 
     auto top_k = [&]() {
       if (finite) {
-        return detail::ivf::qv_query_heap_finite_ram(
+        return detail::ivf::qv_query_heap_finite_ram<db_type, shuffled_ids_type>(
             ctx,
             part_uri,
             centroids,
@@ -189,7 +209,7 @@ int main(int argc, char* argv[]) {
             nth,
             nthreads);
       } else {
-        return detail::ivf::qv_query_heap_infinite_ram(
+        return detail::ivf::qv_query_heap_infinite_ram<db_type, shuffled_ids_type>(
             ctx,
             part_uri,
             centroids,