Incremental indexing: unification, performance, maintenance APIs (#11178)

Just the public facing API changes. All the important stuff is
implemented by the sibling PR:
* rerun-io/dataplatform#1708

Breaking Protobuf changes only apply to the JSON representation, which
isn't used anywhere.

Author: teh-cmc

crates/store/re_protos/proto/rerun/v1alpha1/cloud.proto

@@ -84,9 +84,6 @@ service RerunCloudService {
   // Creates a custom index for a specific column (vector search, full-text search, etc).
   rpc CreateIndex(CreateIndexRequest) returns (CreateIndexResponse) {}
 
-  // Recreate an index with the same configuration but (potentially) new data.
-  rpc ReIndex(ReIndexRequest) returns (ReIndexResponse) {}
-
   /* Queries */
 
   // Search a previously created index.
@@ -291,15 +288,6 @@ message CreateIndexResponse {
   rerun.common.v1alpha1.DataframePart data = 1;
 }
 
-message ReIndexRequest {
-  reserved 1;
-  reserved "dataset_id";
-}
-
-message ReIndexResponse {
-  rerun.common.v1alpha1.DataframePart data = 1;
-}
-
 message IndexConfig {
   // what kind of index do we want to create and what are its index specific properties.
   IndexProperties properties = 1;
@@ -623,28 +611,38 @@ message ScanTableResponse {
 message DoMaintenanceRequest {
   rerun.common.v1alpha1.EntryId dataset_id = 1;
 
-  // Create the acceleration structures for temporal queries.
+  // Optimize all builtin and user-defined indexes on this dataset.
+  //
+  // This merges all individual index deltas back in the main index, improving runtime performance
+  // of all indexes.
+  bool optimize_indexes = 2;
+
+  // Retrain all user-defined indexes on this dataset from scratch.
   //
-  // This will recreate all scalar indexes from scratch everytime.
+  // This retrains all user-defined indexes from scratch for optimal runtime performance.
+  // This is faster than re-creating the indexes, and automatically keeps track of their configurations.
   //
-  // TODO(cmc): support incremental scalar indexing & index compaction
-  bool build_scalar_indexes = 2;
+  // This implies `optimize_indexes`.
+  bool retrain_indexes = 6;
 
   // Compact the underlying Lance fragments, for all Rerun Manifests.
   //
   // Hardcoded to the default (optimal) settings.
   bool compact_fragments = 3;
 
   // If set, all Lance fragments older than this date will be removed, for all Rerun Manifests.
+  //
   // In case requested date is more recent than 1 hour, it will be ignored and 1 hour ago
   // timestamp will be used. This is to prevent still used files (like recent transaction files)
   // to be removed and cause Lance Dataset update issues.
+  //
   // See https://docs.rs/lance/latest/lance/dataset/cleanup/index.html
   // and https://docs.rs/lance/latest/lance/dataset/cleanup/fn.cleanup_old_versions.html
   google.protobuf.Timestamp cleanup_before = 4;
 
   // Override default platform behavior and allow cleanup of recent files. This will respect
   // the value of `cleanup_before` timestamp even if it's more recent than 1 hour.
+  //
   // ⚠️ Do not ever use this unless you know exactly what you're doing. Improper use will lead to data loss.
   bool unsafe_allow_recent_cleanup = 5;
 }

crates/store/re_protos/src/v1alpha1/rerun.cloud.v1alpha1.ext.rs

@@ -113,7 +113,8 @@ impl TryFrom<crate::cloud::v1alpha1::GetChunksRequest> for GetChunksRequest {
 #[derive(Debug, Clone)]
 pub struct DoMaintenanceRequest {
     pub dataset_id: Option<crate::common::v1alpha1::EntryId>,
-    pub build_scalar_indexes: bool,
+    pub optimize_indexes: bool,
+    pub retrain_indexes: bool,
     pub compact_fragments: bool,
     pub cleanup_before: Option<jiff::Timestamp>,
     pub unsafe_allow_recent_cleanup: bool,
@@ -123,7 +124,8 @@ impl From<DoMaintenanceRequest> for crate::cloud::v1alpha1::DoMaintenanceRequest
     fn from(value: DoMaintenanceRequest) -> Self {
         Self {
             dataset_id: value.dataset_id,
-            build_scalar_indexes: value.build_scalar_indexes,
+            optimize_indexes: value.optimize_indexes,
+            retrain_indexes: value.retrain_indexes,
             compact_fragments: value.compact_fragments,
             cleanup_before: value.cleanup_before.map(|ts| prost_types::Timestamp {
                 seconds: ts.as_second(),

crates/store/re_protos/src/v1alpha1/rerun.cloud.v1alpha1.rs

@@ -357,7 +357,8 @@ where
     pub async fn do_maintenance(
         &mut self,
         dataset_id: EntryId,
-        build_scalar_indexes: bool,
+        optimize_indexes: bool,
+        retrain_indexes: bool,
         compact_fragments: bool,
         cleanup_before: Option<jiff::Timestamp>,
         unsafe_allow_recent_cleanup: bool,
@@ -366,7 +367,8 @@ where
             .do_maintenance(tonic::Request::new(
                 re_protos::cloud::v1alpha1::ext::DoMaintenanceRequest {
                     dataset_id: Some(dataset_id.into()),
-                    build_scalar_indexes,
+                    optimize_indexes,
+                    retrain_indexes,
                     compact_fragments,
                     cleanup_before,
                     unsafe_allow_recent_cleanup,

crates/store/re_redap_client/src/connection_client.rs

@@ -594,16 +594,6 @@ impl RerunCloudService for RerunCloudHandler {
         Err(tonic::Status::unimplemented("create_index not implemented"))
     }
 
-    async fn re_index(
-        &self,
-        _request: tonic::Request<re_protos::cloud::v1alpha1::ReIndexRequest>,
-    ) -> std::result::Result<
-        tonic::Response<re_protos::cloud::v1alpha1::ReIndexResponse>,
-        tonic::Status,
-    > {
-        Err(tonic::Status::unimplemented("re_index not implemented"))
-    }
-
     /* Queries */
 
     type SearchDatasetStream = SearchDatasetResponseStream;

crates/store/re_server/src/rerun_cloud.rs

@@ -1517,7 +1517,8 @@ class DatasetEntry(Entry):
 
     def do_maintenance(
         self,
-        build_scalar_index: bool = False,
+        optimize_indexes: bool = False,
+        retrain_indexes: bool = False,
         compact_fragments: bool = False,
         cleanup_before: datetime | None = None,
         unsafe_allow_recent_cleanup: bool = False,

rerun_py/rerun_bindings/rerun_bindings.pyi

@@ -300,12 +300,13 @@ impl ConnectionHandle {
     }
 
     #[tracing::instrument(level = "info", skip_all)]
-    #[allow(clippy::fn_params_excessive_bools)]
+    #[allow(clippy::fn_params_excessive_bools, clippy::too_many_arguments)]
     pub fn do_maintenance(
         &self,
         py: Python<'_>,
         dataset_id: EntryId,
-        build_scalar_indexes: bool,
+        optimize_indexes: bool,
+        retrain_indexes: bool,
         compact_fragments: bool,
         cleanup_before: Option<jiff::Timestamp>,
         unsafe_allow_recent_cleanup: bool,
@@ -317,7 +318,8 @@ impl ConnectionHandle {
                     .await?
                     .do_maintenance(
                         dataset_id,
-                        build_scalar_indexes,
+                        optimize_indexes,
+                        retrain_indexes,
                         compact_fragments,
                         cleanup_before,
                         unsafe_allow_recent_cleanup,

rerun_py/src/catalog/connection_handle.rs

@@ -729,7 +729,8 @@ impl PyDatasetEntry {
 
     /// Perform maintenance tasks on the datasets.
     #[pyo3(signature = (
-            build_scalar_index = false,
+            optimize_indexes = false,
+            retrain_indexes = false,
             compact_fragments = false,
             cleanup_before = None,
             unsafe_allow_recent_cleanup = false,
@@ -739,7 +740,8 @@ impl PyDatasetEntry {
     fn do_maintenance(
         self_: PyRef<'_, Self>,
         py: Python<'_>,
-        build_scalar_index: bool,
+        optimize_indexes: bool,
+        retrain_indexes: bool,
         compact_fragments: bool,
         cleanup_before: Option<Bound<'_, PyAny>>,
         unsafe_allow_recent_cleanup: bool,
@@ -766,7 +768,8 @@ impl PyDatasetEntry {
         connection.do_maintenance(
             py,
             dataset_id,
-            build_scalar_index,
+            optimize_indexes,
+            retrain_indexes,
             compact_fragments,
             cleanup_before,
             unsafe_allow_recent_cleanup,