remove GetChunks Cloud API and all its usage (#11449)

Author: zehiko

crates/store/re_datafusion/src/dataframe_query_common.rs

@@ -23,8 +23,8 @@ use re_dataframe::external::re_chunk_store::ChunkStore;
 use re_dataframe::{Index, QueryExpression};
 use re_log_encoding::codec::wire::decoder::Decode as _;
 use re_log_types::EntryId;
-use re_protos::cloud::v1alpha1::DATASET_MANIFEST_ID_FIELD_NAME;
 use re_protos::cloud::v1alpha1::ext::{Query, QueryLatestAt, QueryRange};
+use re_protos::cloud::v1alpha1::{DATASET_MANIFEST_ID_FIELD_NAME, QueryDatasetResponse};
 use re_protos::cloud::v1alpha1::{GetDatasetSchemaRequest, QueryDatasetRequest};
 use re_protos::common::v1alpha1::ext::ScanParameters;
 use re_protos::headers::RerunHeadersInjectorExt as _;
@@ -106,10 +106,10 @@ impl DataframeQueryTableProvider {
         let query = query_from_query_expression(query_expression);
 
         let fields_of_interest = [
-            "chunk_partition_id",
-            "chunk_id",
-            "rerun_partition_layer",
-            "chunk_key",
+            QueryDatasetResponse::PARTITION_ID,
+            QueryDatasetResponse::CHUNK_ID,
+            QueryDatasetResponse::PARTITION_LAYER,
+            QueryDatasetResponse::CHUNK_KEY,
         ]
         .into_iter()
         .map(String::from)
@@ -346,7 +346,7 @@ fn compute_schema_for_query(
     // Create the actual filter to apply to the column descriptors
     let filter = ChunkStore::create_component_filter_from_query(query_expression);
 
-    // When we call GetChunks we will not return row_id, so we only select indices and
+    // When we call QueryDataset we will not return row_id, so we only select indices and
     // components from the column descriptors.
     let filtered_fields = column_descriptors
         .filter_components(filter)

crates/store/re_datafusion/src/dataframe_query_provider_wasm.rs

@@ -99,7 +99,7 @@ impl DataframePartitionStream {
                 chunks_and_partition_ids.map_err(|err| exec_datafusion_err!("{err}"))?;
 
             let _span = tracing::trace_span!(
-                "get_chunks::batch_insert",
+                "fetch_chunks::batch_insert",
                 num_chunks = chunks_and_partition_ids.len()
             )
             .entered();

crates/store/re_protos/proto/rerun/v1alpha1/cloud.proto

@@ -110,25 +110,13 @@ service RerunCloudService {
   // * Latest-at, range and dataframe queries.
   // * Arbitrary Lance filters.
   //
-  // To fetch the actual chunks themselves, see `GetChunks`.
+  // To fetch the actual chunks themselves, see `FetchChunks`.
   //
   // Passing chunk IDs to this method effectively acts as a IF_EXIST filter.
   //
   // This endpoint requires the standard dataset headers.
   rpc QueryDataset(QueryDatasetRequest) returns (stream QueryDatasetResponse) {}
 
-  // Perform Rerun-native queries on a dataset, returning the underlying chunks.
-  //
-  // These Rerun-native queries include:
-  // * Filtering by specific partition and chunk IDs.
-  // * Latest-at, range and dataframe queries.
-  // * Arbitrary Lance filters.
-  //
-  // To fetch only the actual chunk IDs rather than the chunks themselves, see `QueryDataset`.
-  //
-  // This endpoint requires the standard dataset headers.
-  rpc GetChunks(GetChunksRequest) returns (stream GetChunksResponse) {}
-
   // Fetch specific chunks from Rerun Cloud. In a 2-step query process, result of 1st phase,
   // that is, the result of `QueryDataset` should include all the necessary information to send
   // the actual chunk requests, which is the 2nd step of the query process.
@@ -530,63 +518,6 @@ message QueryRange {
   reserved "fuzzy_descriptors";
 }
 
-message GetChunksRequest {
-  // Client can specify from which partitions to get chunks. If left unspecified (empty list),
-  // data from all partition (that match other query parameters) will be included.
-  repeated rerun.common.v1alpha1.PartitionId partition_ids = 2;
-
-  // Client can specify chunk ids to include. If left unspecified (empty list),
-  // all chunks (that match other query parameters) will be included.
-  repeated rerun.common.v1alpha1.Tuid chunk_ids = 3;
-
-  // Which entity paths are we interested in? Leave empty, and set `select_all_entity_paths`,
-  // in order to query all of them.
-  repeated rerun.common.v1alpha1.EntityPath entity_paths = 4;
-
-  // If set, the query will cover all existing entity paths.
-  //
-  // `entity_paths` must be empty, otherwise an error will be raised.
-  //
-  // Truth table:
-  // ```text
-  // select_all_entity_paths | entity_paths   | result
-  // ------------------------+----------------+--------
-  // false                   | []             | valid query, empty results (no entity paths selected)
-  // false                   | ['foo', 'bar'] | valid query, 'foo' & 'bar' selected
-  // true                    | []             | valid query, all entity paths selected
-  // true                    | ['foo', 'bar'] | invalid query, error
-  // ```
-  bool select_all_entity_paths = 6;
-
-  // Which components are we interested in?
-  //
-  // If left unspecified, all existing components are considered of interest.
-  //
-  // This will perform a basic fuzzy match on the available columns' descriptors.
-  // The fuzzy logic is a simple case-sensitive `contains()` query.
-  // For example, given a `log_tick__SeriesLines:width` index, all of the following
-  // would match: `SeriesLines:width`, `Width`, `SeriesLines`, etc.
-  repeated string fuzzy_descriptors = 9;
-
-  // If set, static data will be excluded from the results.
-  bool exclude_static_data = 7;
-
-  // If set, temporal data will be excluded from the results.
-  bool exclude_temporal_data = 8;
-
-  // Query details
-  Query query = 5;
-
-  reserved 1;
-  reserved "dataset_id";
-}
-
-message GetChunksResponse {
-  // Every gRPC response, even within the confines of a stream, involves HTTP2 overhead, which isn't
-  // cheap by any means, which is why we're returning a batch of `ArrowMsg` rather than a single one.
-  repeated rerun.log_msg.v1alpha1.ArrowMsg chunks = 1;
-}
-
 message FetchChunksRequest {
   // Information about the chunks to fetch. These dataframes have to include the following columns:
   // * `chunk_id` - Chunk unique identifier

crates/store/re_protos/src/v1alpha1/rerun.cloud.v1alpha1.ext.rs

@@ -11,7 +11,7 @@ use re_chunk::TimelineName;
 use re_log_types::{EntityPath, EntryId, TimeInt};
 use re_sorbet::ComponentColumnDescriptor;
 
-use crate::cloud::v1alpha1::{EntryKind, QueryTasksResponse};
+use crate::cloud::v1alpha1::{EntryKind, QueryDatasetResponse, QueryTasksResponse};
 use crate::cloud::v1alpha1::{
     GetDatasetSchemaResponse, RegisterWithDatasetResponse, ScanPartitionTableResponse,
     VectorDistanceMetric,
@@ -121,61 +121,18 @@ impl TryFrom<crate::cloud::v1alpha1::QueryDatasetRequest> for QueryDatasetReques
     }
 }
 
-// --- GetChunksRequest --
+// --- QueryDatasetResponse ---
 
-#[derive(Debug, Clone)]
-pub struct GetChunksRequest {
-    pub partition_ids: Vec<crate::common::v1alpha1::ext::PartitionId>,
-    pub chunk_ids: Vec<re_chunk::ChunkId>,
-    pub entity_paths: Vec<EntityPath>,
-    pub select_all_entity_paths: bool,
-    pub fuzzy_descriptors: Vec<String>,
-    pub exclude_static_data: bool,
-    pub exclude_temporal_data: bool,
-    pub query: Option<Query>,
+impl QueryDatasetResponse {
+    pub const PARTITION_ID: &str = "chunk_partition_id";
+    pub const CHUNK_ID: &str = "chunk_id";
+    pub const PARTITION_LAYER: &str = RegisterWithDatasetResponse::PARTITION_LAYER;
+    pub const CHUNK_KEY: &str = "chunk_key";
+    pub const CHUNK_IS_STATIC: &str = "chunk_is_static";
+    pub const CHUNK_ENTITY_PATH: &str = "chunk_entity_path";
 }
 
-impl TryFrom<crate::cloud::v1alpha1::GetChunksRequest> for GetChunksRequest {
-    type Error = tonic::Status;
-
-    fn try_from(value: crate::cloud::v1alpha1::GetChunksRequest) -> Result<Self, Self::Error> {
-        Ok(Self {
-            partition_ids: value
-                .partition_ids
-                .into_iter()
-                .map(TryInto::try_into)
-                .collect::<Result<Vec<_>, _>>()?,
-
-            chunk_ids: value
-                .chunk_ids
-                .into_iter()
-                .map(|tuid| {
-                    let id: re_tuid::Tuid = tuid.try_into()?;
-                    Ok::<_, tonic::Status>(re_chunk::ChunkId::from_u128(id.as_u128()))
-                })
-                .collect::<Result<Vec<_>, _>>()?,
-
-            entity_paths: value
-                .entity_paths
-                .into_iter()
-                .map(|path| {
-                    path.try_into().map_err(|err| {
-                        tonic::Status::invalid_argument(format!("invalid entity path: {err}"))
-                    })
-                })
-                .collect::<Result<Vec<_>, _>>()?,
-
-            select_all_entity_paths: value.select_all_entity_paths,
-
-            fuzzy_descriptors: value.fuzzy_descriptors,
-
-            exclude_static_data: value.exclude_static_data,
-            exclude_temporal_data: value.exclude_temporal_data,
-
-            query: value.query.map(|q| q.try_into()).transpose()?,
-        })
-    }
-}
+// --- DoMaintenanceRequest ---
 
 #[derive(Debug, Clone)]
 pub struct DoMaintenanceRequest {