fix(cube-orchestrator): Deterministic column sequence in output (#10086)

* fix: Deterministic column sequence in output

* fix: tests fixed

* fix: Removed unused import

* chore(ci): trigger workflow rerun

---------

Co-authored-by: usman.yasin <[email protected]>

Author: UsmanYasin

packages/cubejs-backend-native/Cargo.lock

@@ -10,6 +10,7 @@ serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.133"
 anyhow = "1.0"
 itertools = "0.13.0"
+indexmap = { version = "2.0", features = ["serde"] }
 
 [dependencies.neon]
 version = "=1"

rust/cubeorchestrator/Cargo.toml

@@ -3,8 +3,8 @@ use crate::{
     transport::JsRawData,
 };
 use cubeshared::codegen::{root_as_http_message, HttpCommand};
+use indexmap::IndexMap;
 use neon::prelude::Finalize;
-use std::collections::HashMap;
 
 #[derive(Debug)]
 pub enum ParseError {
@@ -35,7 +35,7 @@ impl std::error::Error for ParseError {}
 pub struct QueryResult {
     pub columns: Vec<String>,
     pub rows: Vec<Vec<DBResponseValue>>,
-    pub columns_pos: HashMap<String, usize>,
+    pub columns_pos: IndexMap<String, usize>,
 }
 
 impl Finalize for QueryResult {}
@@ -45,7 +45,7 @@ impl QueryResult {
         let mut result = QueryResult {
             columns: vec![],
             rows: vec![],
-            columns_pos: HashMap::new(),
+            columns_pos: IndexMap::new(),
         };
 
         let http_message =
@@ -69,7 +69,7 @@ impl QueryResult {
                         return Err(ParseError::ColumnNameNotDefined);
                     }
 
-                    let (columns, columns_pos): (Vec<_>, HashMap<_, _>) = result_set_columns
+                    let (columns, columns_pos): (Vec<_>, IndexMap<_, _>) = result_set_columns
                         .iter()
                         .enumerate()
                         .map(|(index, column_name)| {
@@ -111,13 +111,13 @@ impl QueryResult {
             return Ok(QueryResult {
                 columns: vec![],
                 rows: vec![],
-                columns_pos: HashMap::new(),
+                columns_pos: IndexMap::new(),
             });
         }
 
         let first_row = &js_raw_data[0];
         let columns: Vec<String> = first_row.keys().cloned().collect();
-        let columns_pos: HashMap<String, usize> = columns
+        let columns_pos: IndexMap<String, usize> = columns
             .iter()
             .enumerate()
             .map(|(index, column)| (column.clone(), index))

rust/cubeorchestrator/src/query_message_parser.rs

@@ -7,6 +7,7 @@ use crate::{
 };
 use anyhow::{bail, Context, Result};
 use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
+use indexmap::IndexMap;
 use itertools::multizip;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -171,7 +172,12 @@ pub fn get_members(
     alias_to_member_name_map: &HashMap<String, String>,
     annotation: &HashMap<String, ConfigItem>,
 ) -> Result<(MembersMap, Vec<String>)> {
-    let mut members_map: MembersMap = HashMap::new();
+    let mut members_map: MembersMap = IndexMap::new();
+    // IndexMap maintains insertion order, ensuring deterministic column ordering.
+    // The order comes from db_data.columns which now preserves the database result order
+    // (since JsRawData uses IndexMap instead of HashMap).
+    // Not sure if it solves the original comment below.
+    // Original Comment:
     // Hashmaps don't guarantee the order of the elements while iterating
     // this fires in get_compact_row because members map doesn't hold the members for
     // date range queries, which are added later and thus columns in final recordset are not
@@ -267,13 +273,13 @@ pub fn get_members(
 
 /// Convert DB response object to the compact output format.
 pub fn get_compact_row(
-    members_to_alias_map: &HashMap<String, String>,
+    members_to_alias_map: &IndexMap<String, String>,
     annotation: &HashMap<String, ConfigItem>,
     query_type: &QueryType,
     members: &[String],
     time_dimensions: Option<&Vec<QueryTimeDimension>>,
     db_row: &[DBResponseValue],
-    columns_pos: &HashMap<String, usize>,
+    columns_pos: &IndexMap<String, usize>,
 ) -> Result<Vec<DBResponsePrimitive>> {
     let mut row: Vec<DBResponsePrimitive> = Vec::with_capacity(members.len());
 
@@ -322,9 +328,9 @@ pub fn get_vanilla_row(
     query_type: &QueryType,
     query: &NormalizedQuery,
     db_row: &[DBResponseValue],
-    columns_pos: &HashMap<String, usize>,
-) -> Result<HashMap<String, DBResponsePrimitive>> {
-    let mut row = HashMap::new();
+    columns_pos: &IndexMap<String, usize>,
+) -> Result<IndexMap<String, DBResponsePrimitive>> {
+    let mut row = IndexMap::new();
 
     // FIXME: For now custom granularities are not supported, only common ones.
     // There is no granularity type/class implementation in rust yet.
@@ -527,7 +533,7 @@ pub enum TransformedData {
         members: Vec<String>,
         dataset: Vec<Vec<DBResponsePrimitive>>,
     },
-    Vanilla(Vec<HashMap<String, DBResponsePrimitive>>),
+    Vanilla(Vec<IndexMap<String, DBResponsePrimitive>>),
 }
 
 impl TransformedData {
@@ -2180,7 +2186,7 @@ mod tests {
             &QueryResult {
                 columns: vec![],
                 rows: vec![],
-                columns_pos: HashMap::new(),
+                columns_pos: IndexMap::new(),
             },
             alias_to_member_name_map,
             annotation,
@@ -2209,7 +2215,7 @@ mod tests {
             alias_to_member_name_map,
             annotation,
         )?;
-        let members_map_expected: MembersMap = HashMap::from([
+        let members_map_expected: MembersMap = IndexMap::from([
             (
                 "ECommerceRecordsUs2021.postalCode".to_string(),
                 "e_commerce_records_us2021__postal_code".to_string(),
@@ -2241,7 +2247,7 @@ mod tests {
             &QueryResult {
                 columns: vec![],
                 rows: vec![],
-                columns_pos: HashMap::new(),
+                columns_pos: IndexMap::new(),
             },
             alias_to_member_name_map,
             annotation,
@@ -2270,7 +2276,7 @@ mod tests {
             alias_to_member_name_map,
             annotation,
         )?;
-        let members_map_expected: MembersMap = HashMap::from([
+        let members_map_expected: MembersMap = IndexMap::from([
             (
                 "ECommerceRecordsUs2021.orderDate.day".to_string(),
                 "e_commerce_records_us2021__order_date_day".to_string(),
@@ -2313,7 +2319,7 @@ mod tests {
             &QueryResult {
                 columns: vec![],
                 rows: vec![],
-                columns_pos: HashMap::new(),
+                columns_pos: IndexMap::new(),
             },
             alias_to_member_name_map,
             annotation,
@@ -2345,7 +2351,7 @@ mod tests {
             alias_to_member_name_map,
             annotation,
         )?;
-        let members_map_expected: HashMap<String, String> = HashMap::from([
+        let members_map_expected: MembersMap = IndexMap::from([
             (
                 "ECommerceRecordsUs2021.orderDate.month".to_string(),
                 "e_commerce_records_us2021__order_date_month".to_string(),
@@ -2640,7 +2646,7 @@ mod tests {
             &raw_data.rows[0],
             &raw_data.columns_pos,
         )?;
-        let expected = HashMap::from([
+        let expected = IndexMap::from([
             (
                 "ECommerceRecordsUs2021.city".to_string(),
                 DBResponsePrimitive::String("Missouri City".to_string()),

rust/cubeorchestrator/src/query_result_transform.rs

@@ -1,4 +1,5 @@
 use crate::query_result_transform::DBResponsePrimitive;
+use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::{collections::HashMap, fmt::Display};
@@ -153,7 +154,7 @@ pub struct QueryTimeDimension {
 
 pub type AliasToMemberMap = HashMap<String, String>;
 
-pub type MembersMap = HashMap<String, String>;
+pub type MembersMap = IndexMap<String, String>;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct GranularityMeta {
@@ -317,4 +318,4 @@ pub struct TransformDataRequest {
     pub res_type: Option<ResultType>,
 }
 
-pub type JsRawData = Vec<HashMap<String, DBResponsePrimitive>>;
+pub type JsRawData = Vec<IndexMap<String, DBResponsePrimitive>>;