Data Explorer: Implement SearchSchema RPC, refactor and improve data explorer test suite (#889)

Addresses posit-dev/positron#8805, follows backend API change in posit-dev/positron#8810. Also pulls in variables comm changes which can be implemented in a follow up PR

Author: wesm

crates/amalthea/src/comm/data_explorer_comm.rs

@@ -21,11 +21,9 @@ pub struct OpenDatasetResult {
 /// Result in Methods
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct SearchSchemaResult {
-	/// A schema containing matching columns up to the max_results limit
-	pub matches: TableSchema,
-
-	/// The total number of columns matching the filter
-	pub total_num_matches: i64
+	/// The column indices of the matching column indices in the indicated
+	/// sort order
+	pub matches: Vec<i64>
 }
 
 /// Exported result
@@ -691,6 +689,22 @@ pub struct ColumnSelection {
 	pub spec: ArraySelection
 }
 
+/// Possible values for SortOrder in SearchSchema
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, strum_macros::Display)]
+pub enum SearchSchemaSortOrder {
+	#[serde(rename = "original")]
+	#[strum(to_string = "original")]
+	Original,
+
+	#[serde(rename = "ascending")]
+	#[strum(to_string = "ascending")]
+	Ascending,
+
+	#[serde(rename = "descending")]
+	#[strum(to_string = "descending")]
+	Descending
+}
+
 /// Possible values for ColumnDisplayType
 #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, strum_macros::Display)]
 pub enum ColumnDisplayType {
@@ -1059,15 +1073,12 @@ pub struct GetSchemaParams {
 /// Parameters for the SearchSchema method.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct SearchSchemaParams {
-	/// Column filters to apply when searching
+	/// Column filters to apply when searching, can be empty
 	pub filters: Vec<ColumnFilter>,
 
-	/// Index (starting from zero) of first result to fetch (for paging)
-	pub start_index: i64,
-
-	/// Maximum number of resulting column schemas to fetch from the start
-	/// index
-	pub max_results: i64,
+	/// How to sort results: original in-schema order, alphabetical ascending
+	/// or descending
+	pub sort_order: SearchSchemaSortOrder,
 }
 
 /// Parameters for the GetDataValues method.
@@ -1181,10 +1192,9 @@ pub enum DataExplorerBackendRequest {
 	#[serde(rename = "get_schema")]
 	GetSchema(GetSchemaParams),
 
-	/// Search full, unfiltered table schema with column filters
+	/// Search table schema with column filters, optionally sort results
 	///
-	/// Search full, unfiltered table schema for column names matching one or
-	/// more column filters
+	/// Search table schema with column filters, optionally sort results
 	#[serde(rename = "search_schema")]
 	SearchSchema(SearchSchemaParams),
 

crates/amalthea/src/comm/variables_comm.rs

@@ -44,6 +44,22 @@ pub struct FormattedVariable {
 	pub content: String
 }
 
+/// Result of the summarize operation
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+pub struct QueryTableSummaryResult {
+	/// The total number of rows in the table.
+	pub num_rows: i64,
+
+	/// The total number of columns in the table.
+	pub num_columns: i64,
+
+	/// The column schemas in the table.
+	pub column_schemas: Vec<String>,
+
+	/// The column profiles in the table.
+	pub column_profiles: Vec<String>
+}
+
 /// A single variable in the runtime.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct Variable {
@@ -196,6 +212,16 @@ pub struct ViewParams {
 	pub path: Vec<String>,
 }
 
+/// Parameters for the QueryTableSummary method.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+pub struct QueryTableSummaryParams {
+	/// The path to the table to summarize, as an array of access keys.
+	pub path: Vec<String>,
+
+	/// A list of query types.
+	pub query_types: Vec<String>,
+}
+
 /// Parameters for the Update method.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct UpdateParams {
@@ -271,6 +297,12 @@ pub enum VariablesBackendRequest {
 	#[serde(rename = "view")]
 	View(ViewParams),
 
+	/// Query table summary
+	///
+	/// Request a data summary for a table variable.
+	#[serde(rename = "query_table_summary")]
+	QueryTableSummary(QueryTableSummaryParams),
+
 }
 
 /**
@@ -297,6 +329,9 @@ pub enum VariablesBackendReply {
 	/// The ID of the viewer that was opened.
 	ViewReply(Option<String>),
 
+	/// Result of the summarize operation
+	QueryTableSummaryReply(QueryTableSummaryResult),
+
 }
 
 /**

crates/ark/src/data_explorer/r_data_explorer.rs

@@ -14,6 +14,9 @@ use amalthea::comm::data_explorer_comm::BackendState;
 use amalthea::comm::data_explorer_comm::CodeSyntaxName;
 use amalthea::comm::data_explorer_comm::ColumnDisplayType;
 use amalthea::comm::data_explorer_comm::ColumnFilter;
+use amalthea::comm::data_explorer_comm::ColumnFilterParams;
+use amalthea::comm::data_explorer_comm::ColumnFilterType;
+use amalthea::comm::data_explorer_comm::ColumnFilterTypeSupportStatus;
 use amalthea::comm::data_explorer_comm::ColumnProfileType;
 use amalthea::comm::data_explorer_comm::ColumnProfileTypeSupportStatus;
 use amalthea::comm::data_explorer_comm::ColumnSchema;
@@ -53,6 +56,7 @@ use amalthea::comm::data_explorer_comm::TableRowLabels;
 use amalthea::comm::data_explorer_comm::TableSchema;
 use amalthea::comm::data_explorer_comm::TableSelection;
 use amalthea::comm::data_explorer_comm::TableShape;
+use amalthea::comm::data_explorer_comm::TextSearchType;
 use amalthea::comm::event::CommManagerEvent;
 use amalthea::socket::comm::CommInitiator;
 use amalthea::socket::comm::CommSocket;
@@ -532,9 +536,7 @@ impl RDataExplorer {
                 return Err(anyhow!("Data Explorer: Not yet supported"));
             },
 
-            DataExplorerBackendRequest::SearchSchema(_) => {
-                return Err(anyhow!("Data Explorer: Not yet supported"));
-            },
+            DataExplorerBackendRequest::SearchSchema(params) => self.search_schema(params),
 
             DataExplorerBackendRequest::SetColumnFilters(_) => {
                 return Err(anyhow!("Data Explorer: Not yet supported"));
@@ -859,6 +861,104 @@ impl RDataExplorer {
         self.view_indices = Some(view_indices);
     }
 
+    /// Search the schema for columns matching the given filters and sort order.
+    ///
+    /// - `params`: The search parameters including filters and sort order.
+    fn search_schema(
+        &self,
+        params: amalthea::comm::data_explorer_comm::SearchSchemaParams,
+    ) -> anyhow::Result<DataExplorerBackendReply> {
+        let all_columns = &self.shape.columns;
+
+        // Apply column filters to find matching columns using iterator chaining
+        let mut matching_indices: Vec<i64> = all_columns
+            .iter()
+            .enumerate()
+            .filter_map(|(index, column)| {
+                let column_index = index as i64;
+
+                // Check if column matches all filters
+                let matches = params
+                    .filters
+                    .iter()
+                    .all(|filter| self.column_matches_filter(column, filter));
+
+                if matches {
+                    Some(column_index)
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        // Apply sort order
+        match params.sort_order {
+            amalthea::comm::data_explorer_comm::SearchSchemaSortOrder::Original => {
+                // matching_indices is already in original order
+            },
+            order => {
+                let ascending = matches!(
+                    order,
+                    amalthea::comm::data_explorer_comm::SearchSchemaSortOrder::Ascending
+                );
+                matching_indices.sort_by(|&a, &b| {
+                    let ord = all_columns[a as usize]
+                        .column_name
+                        .cmp(&all_columns[b as usize].column_name);
+                    if ascending { ord } else { ord.reverse() }
+                });
+            }
+        }
+
+        Ok(DataExplorerBackendReply::SearchSchemaReply(
+            amalthea::comm::data_explorer_comm::SearchSchemaResult {
+                matches: matching_indices,
+            },
+        ))
+    }
+
+    /// Check if a column matches a given column filter.
+    fn column_matches_filter(&self, column: &ColumnSchema, filter: &ColumnFilter) -> bool {
+        match filter.filter_type {
+            ColumnFilterType::TextSearch => {
+                if let ColumnFilterParams::TextSearch(text_search) = &filter.params {
+                    let column_name = if text_search.case_sensitive {
+                        column.column_name.to_owned()
+                    } else {
+                        column.column_name.to_lowercase()
+                    };
+
+                    let search_term = if text_search.case_sensitive {
+                        text_search.term.to_owned()
+                    } else {
+                        text_search.term.to_lowercase()
+                    };
+
+                    match text_search.search_type {
+                        TextSearchType::Contains => column_name.contains(&search_term),
+                        TextSearchType::NotContains => !column_name.contains(&search_term),
+                        TextSearchType::StartsWith => column_name.starts_with(&search_term),
+                        TextSearchType::EndsWith => column_name.ends_with(&search_term),
+                        TextSearchType::RegexMatch => {
+                            // For regex matching, we use simple string matching as a fallback
+                            // A full regex implementation would require additional dependencies
+                            column_name.contains(&search_term)
+                        },
+                    }
+                } else {
+                    false
+                }
+            },
+            ColumnFilterType::MatchDataTypes => {
+                if let ColumnFilterParams::MatchDataTypes(type_filter) = &filter.params {
+                    type_filter.display_types.contains(&column.type_display)
+                } else {
+                    false
+                }
+            },
+        }
+    }
+
     /// Get the schema for a vector of columns in the data object.
     ///
     /// - `column_indices`: The vector of columns in the data object.
@@ -950,8 +1050,17 @@ impl RDataExplorer {
                     ],
                 },
                 search_schema: SearchSchemaFeatures {
-                    support_status: SupportStatus::Unsupported,
-                    supported_types: vec![],
+                    support_status: SupportStatus::Supported,
+                    supported_types: vec![
+                        ColumnFilterTypeSupportStatus {
+                            column_filter_type: ColumnFilterType::TextSearch,
+                            support_status: SupportStatus::Supported,
+                        },
+                        ColumnFilterTypeSupportStatus {
+                            column_filter_type: ColumnFilterType::MatchDataTypes,
+                            support_status: SupportStatus::Supported,
+                        },
+                    ],
                 },
                 set_row_filters: SetRowFiltersFeatures {
                     support_status: SupportStatus::Supported,

crates/ark/src/variables/r_variables.rs

@@ -18,6 +18,7 @@ use amalthea::comm::variables_comm::VariablesBackendReply;
 use amalthea::comm::variables_comm::VariablesBackendRequest;
 use amalthea::comm::variables_comm::VariablesFrontendEvent;
 use amalthea::socket::comm::CommSocket;
+use anyhow::anyhow;
 use crossbeam::channel::select;
 use crossbeam::channel::unbounded;
 use crossbeam::channel::Sender;
@@ -294,6 +295,9 @@ impl RVariables {
                 let viewer_id = self.view(&params.path)?;
                 Ok(VariablesBackendReply::ViewReply(viewer_id))
             },
+            VariablesBackendRequest::QueryTableSummary(_) => {
+                return Err(anyhow!("Variables: QueryTableSummary not yet supported"));
+            },
         }
     }