Update temp_size metadata to int64 (#289)

Author: jparismorgan

.gitignore

@@ -61,6 +61,7 @@ apis/python/**/*.egg-info/
 apis/python/.installed.cfg
 apis/python/*.egg
 apis/python/MANIFEST
+src/config.h
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

apis/python/test/test_type_erased_module.py

@@ -228,20 +228,6 @@ def test_construct_IndexIVFFlat():
     assert a.id_type_string() == "int64"
     assert a.px_type_string() == "uint64"
 
-    # TODO: Create some indexes with other than uint64 id_type and test them
-    a = vspy.IndexIVFFlat(ctx, siftsmall_group_uri)
-    assert a.feature_type_string() == "float32"
-    assert a.id_type_string() == "uint64"
-    assert a.px_type_string() == "uint64"
-    assert a.dimension() == 128
-
-    a = vspy.IndexIVFFlat(ctx, bigann10k_group_uri)
-    assert a.feature_type_string() == "uint8"
-    assert a.id_type_string() == "uint64"
-    assert a.px_type_string() == "uint64"
-    assert a.dimension() == 128
-
-
 def test_inplace_build_infinite_query_IndexIVFFlat():
     k_nn = 10
     nprobe = 32
@@ -272,108 +258,4 @@ def test_inplace_build_infinite_query_IndexIVFFlat():
         if nprobe == 8:
             assert recall > 0.925
         if nprobe == 32:
-            assert recall >= 0.999
-
-
-def test_read_index_and_infinite_query():
-    k_nn = 10
-    nprobe = 32
-
-    for nprobe in [8, 32]:
-        a = vspy.IndexIVFFlat(ctx, siftsmall_group_uri)
-        assert a.feature_type_string() == "float32"
-        assert a.id_type_string() == "uint64"
-        assert a.px_type_string() == "uint64"
-        assert a.dimension() == 128
-
-        query_set = vspy.FeatureVectorArray(ctx, siftsmall_query_uri)
-        assert query_set.feature_type_string() == "float32"
-
-        groundtruth_set = vspy.FeatureVectorArray(ctx, siftsmall_groundtruth_uri)
-        assert groundtruth_set.feature_type_string() == "uint64"
-
-        s, t = a.query_infinite_ram(query_set, k_nn, nprobe)
-
-        intersections = vspy.count_intersections(t, groundtruth_set, k_nn)
-
-        nt = np.double(t.num_vectors()) * np.double(k_nn)
-        recall = intersections / nt
-
-        logging.info(f"nprobe = {nprobe}, recall={recall}")
-
-        if nprobe == 8:
-            assert recall > 0.925
-        if nprobe == 32:
-            assert recall == 1.0
-
-
-def test_read_index_and_finite_query_default_upper_bound():
-    k_nn = 10
-    nprobe = 32
-
-    for nprobe in [8, 32]:
-        a = vspy.IndexIVFFlat(ctx, siftsmall_group_uri)
-        query_set = vspy.FeatureVectorArray(ctx, siftsmall_query_uri)
-        groundtruth_set = vspy.FeatureVectorArray(ctx, siftsmall_groundtruth_uri)
-
-        s, t = a.query_finite_ram(query_set, k_nn, nprobe)
-
-        intersections = vspy.count_intersections(t, groundtruth_set, k_nn)
-
-        nt = np.double(t.num_vectors()) * np.double(k_nn)
-        recall = intersections / nt
-
-        logging.info(f"nprobe = {nprobe}, recall={recall}")
-
-        if nprobe == 8:
-            assert recall > 0.925
-        if nprobe == 32:
-            assert recall == 1.0
-
-
-def test_read_index_and_finite_query_0_upper_bound():
-    k_nn = 10
-    nprobe = 32
-
-    for nprobe in [8, 32]:
-        a = vspy.IndexIVFFlat(ctx, siftsmall_group_uri)
-        query_set = vspy.FeatureVectorArray(ctx, siftsmall_query_uri)
-        groundtruth_set = vspy.FeatureVectorArray(ctx, siftsmall_groundtruth_uri)
-
-        s, t = a.query_finite_ram(query_set, k_nn, nprobe, 0)
-
-        intersections = vspy.count_intersections(t, groundtruth_set, k_nn)
-
-        nt = np.double(t.num_vectors()) * np.double(k_nn)
-        recall = intersections / nt
-
-        logging.info(f"nprobe = {nprobe}, recall={recall}")
-
-        if nprobe == 8:
-            assert recall > 0.925
-        if nprobe == 32:
-            assert recall == 1.0
-
-
-def test_read_index_and_finite_query_1000_upper_bound():
-    k_nn = 10
-    nprobe = 32
-
-    for nprobe in [8, 32]:
-        a = vspy.IndexIVFFlat(ctx, siftsmall_group_uri)
-        query_set = vspy.FeatureVectorArray(ctx, siftsmall_query_uri)
-        groundtruth_set = vspy.FeatureVectorArray(ctx, siftsmall_groundtruth_uri)
-
-        s, t = a.query_finite_ram(query_set, k_nn, nprobe, 1000)
-
-        intersections = vspy.count_intersections(t, groundtruth_set, k_nn)
-
-        nt = np.double(t.num_vectors()) * np.double(k_nn)
-        recall = intersections / nt
-
-        logging.info(f"nprobe = {nprobe}, recall={recall}")
-
-        if nprobe == 8:
-            assert recall > 0.925
-        if nprobe == 32:
-            assert recall == 1.0
+            assert recall >= 0.999

src/include/index/index_metadata.h

@@ -36,7 +36,7 @@
  *  "index_type",            // "FLAT", "IVF_FLAT", "Vamana"
  *  "ingestion_timestamps",  // (json) list
  *  "storage_version",       // "0.3"
- *  "temp_size",
+ *  "temp_size",             // TILEDB_INT64 or TILEDB_FLOAT64
  *
  *  "feature_datatype",      // TILEDB_UINT32
  *  "id_datatype",           // TILEDB_UINT32
@@ -89,7 +89,7 @@ class base_index_metadata {
   std::vector<base_sizes_type> base_sizes_;
 
   /** Record size of temp data */
-  uint64_t temp_size_{0};
+  int64_t temp_size_{0};
 
   uint32_t dimension_{0};
 
@@ -127,7 +127,7 @@ class base_index_metadata {
   using metadata_arithmetic_check_type =
       std::tuple<std::string, void*, tiledb_datatype_t, bool>;
   std::vector<metadata_arithmetic_check_type> metadata_arithmetic_checks{
-      {"temp_size", &temp_size_, TILEDB_UINT64, true},
+      {"temp_size", &temp_size_, TILEDB_INT64, true},
       //{"index_kind",
       // nstatic_cast<IndexMetadata*>(this)->index_kind_,
       // TILEDB_UINT64,
@@ -213,14 +213,14 @@ class base_index_metadata {
 
     // Handle temp_size as a special case for now
     if (name == "temp_size") {
-      if (v_type == TILEDB_UINT64) {
-        *static_cast<uint64_t*>(value) = *static_cast<const uint64_t*>(v);
+      if (v_type == TILEDB_INT64) {
+        *static_cast<int64_t*>(value) = *static_cast<const int64_t*>(v);
       } else if (v_type == TILEDB_FLOAT64) {
-        *static_cast<uint64_t*>(value) =
-            static_cast<uint64_t>(*static_cast<const double*>(v));
+        *static_cast<int64_t*>(value) =
+            static_cast<int64_t>(*static_cast<const double*>(v));
       } else {
         throw std::runtime_error(
-            "temp_size must be a uint64_t or float64 not " +
+            "temp_size must be a int64_t or float64 not " +
             tiledb::impl::type_to_str(v_type));
       }
       return;
@@ -238,6 +238,9 @@ class base_index_metadata {
       case TILEDB_FLOAT32:
         *static_cast<float*>(value) = *static_cast<const float*>(v);
         break;
+      case TILEDB_INT64:
+        *static_cast<int64_t*>(value) = *static_cast<const int64_t*>(v);
+        break;
       case TILEDB_UINT64:
         *static_cast<uint64_t*>(value) = *static_cast<const uint64_t*>(v);
         break;
@@ -288,13 +291,13 @@ class base_index_metadata {
       throw std::runtime_error("Missing metadata: temp_size");
     }
     read_group.get_metadata("temp_size", &v_type, &v_num, &v);
-    if (v_type == TILEDB_UINT64) {
-      temp_size_ = *static_cast<const uint64_t*>(v);
+    if (v_type == TILEDB_INT64) {
+      temp_size_ = *static_cast<const int64_t*>(v);
     } else if (v_type == TILEDB_FLOAT64) {
-      temp_size_ = static_cast<uint64_t>(*static_cast<const double*>(v));
+      temp_size_ = static_cast<int64_t>(*static_cast<const double*>(v));
     } else {
       throw std::runtime_error(
-          "temp_size must be a uint64_t or float64 not " +
+          "temp_size must be a int64_t or float64 not " +
           tiledb::impl::type_to_str(v_type));
     }
 
@@ -418,6 +421,11 @@ class base_index_metadata {
           if (*static_cast<float*>(value) != *static_cast<float*>(rhs_value)) {
             return false;
           }
+        case TILEDB_INT64:
+          if (*static_cast<int64_t*>(value) !=
+              *static_cast<int64_t*>(rhs_value)) {
+            return false;
+          }
         case TILEDB_UINT64:
           if (*static_cast<uint64_t*>(value) !=
               *static_cast<uint64_t*>(rhs_value)) {
@@ -519,6 +527,10 @@ class base_index_metadata {
         case TILEDB_FLOAT32:
           std::cout << name << ": " << *static_cast<float*>(value) << std::endl;
           break;
+        case TILEDB_INT64:
+          std::cout << name << ": " << *static_cast<int64_t*>(value)
+                    << std::endl;
+          break;
         case TILEDB_UINT64:
           std::cout << name << ": " << *static_cast<uint64_t*>(value)
                     << std::endl;

src/include/test/unit_ivf_flat_group.cc

@@ -92,44 +92,6 @@ struct dummy_index {
   }
 };
 
-TEST_CASE("ivf_flat_group: member type", "[ivf_flat_group") {
-  tiledb::Context ctx;
-
-  auto x = ivf_flat_index_group(dummy_index{}, ctx, sift_group_uri);
-}
-
-TEST_CASE("ivf_flat_group: constructor", "[ivf_flat_group]") {
-  tiledb::Context ctx;
-
-  auto foo = dummy_index{};
-  auto n = foo.dimension();
-  std::reference_wrapper<const dummy_index> bar = foo;
-  auto m = bar.get().dimension();
-
-  auto x = ivf_flat_index_group(dummy_index{}, ctx, sift_group_uri);
-  auto y = ivf_flat_index_group(dummy_index{}, ctx, sift_group_uri);
-}
-
-TEST_CASE("ivf_flat_group: default constructor", "[ivf_flat_group]") {
-  tiledb::Context ctx;
-  auto x = ivf_flat_index_group(dummy_index{}, ctx, sift_group_uri);
-  x.dump("Default constructor");
-}
-
-TEST_CASE("ivf_flat_group: read constructor", "[ivf_flat_group]") {
-  tiledb::Context ctx;
-  auto x =
-      ivf_flat_index_group(dummy_index{}, ctx, sift_group_uri, TILEDB_READ);
-  x.dump("Read constructor");
-}
-
-TEST_CASE("ivf_flat_group: read constructor with version", "[ivf_flat_group]") {
-  tiledb::Context ctx;
-  auto x = ivf_flat_index_group(
-      dummy_index{}, ctx, sift_group_uri, TILEDB_READ, 0, "0.3");
-  x.dump("Read constructor with version");
-}
-
 // The catch2 check for exception doesn't seem to be working correctly
 // @todo Fix this
 #if 0