refactor(query): optimize visibility checker for large-scale deployments improved 10x (#18954)

* refactor(query): optimize grant object visibility checker for large-scale deployments

  This commit improves the performance of grant object visibility checking,
  which is critical for SHOW TABLES/DATABASES operations and privilege
  verification in large deployments with hundreds of thousands of objects.

  Key optimizations:
  - Skip ownership loading for account_admin role (has unrestricted access)
  - Replace HashMap/HashSet with FastHashMap/FastHashSet for integer keys
  - Introduce CatalogIdPool to map catalog names to u32 IDs, avoiding
    repeated string hashing in hot paths
  - Pre-allocate hash table capacity based on ownership count
  - Optimize database/table ownership lookups using composite integer keys

  Benchmark results show significant improvements for large-scale scenarios:
  - 100K objects: construction time reduced from O(n log n) to O(n)
  - 1M tables: SHOW TABLES performance improved by ~6x
  - 100K tables: SHOW TABLES performance improved by ~10x
  - Multi-catalog scenarios: reduced memory overhead and faster lookups

* fix

* extract method

* use BUILTIN_ROLE_ACCOUNT_ADMIN replace account_admin

Author: TCeason

Cargo.lock

@@ -457,6 +457,7 @@ rmp-serde = "1.1.1"
 roaring = { version = "^0.10", features = ["serde"] }
 rotbl = { version = "0.2.9", features = [] }
 rust_decimal = "1.26"
+rustc-hash = "2.1.0"
 rustix = { version = "0.38.37", features = ["fs"] }
 rustls = { version = "0.23.27", features = ["ring", "tls12"], default-features = false }
 rustls-pemfile = "2"

Cargo.toml

@@ -15,6 +15,7 @@
 use std::sync::Arc;
 
 use databend_common_exception::Result;
+use databend_common_meta_app::principal::BUILTIN_ROLE_ACCOUNT_ADMIN;
 use databend_common_version::BUILD_INFO;
 use databend_query::sessions::BuildInfoRef;
 use databend_query::sessions::QueryContext;
@@ -63,11 +64,17 @@ impl PySessionContext {
                 databend_common_meta_app::principal::UserPrivilegeSet::available_privileges_on_global(),
             );
             // Grant account_admin role
-            user_info.grants.grant_role("account_admin".to_string());
-            user_info.option.set_default_role(Some("account_admin".to_string()));
+            user_info
+                .grants
+                .grant_role(BUILTIN_ROLE_ACCOUNT_ADMIN.to_string());
+            user_info
+                .option
+                .set_default_role(Some(BUILTIN_ROLE_ACCOUNT_ADMIN.to_string()));
             // Set all flags to bypass various checks
             user_info.option.set_all_flag();
-            session.set_authed_user(user_info, Some("account_admin".to_string())).await?;
+            session
+                .set_authed_user(user_info, Some(BUILTIN_ROLE_ACCOUNT_ADMIN.to_string()))
+                .await?;
 
             Ok::<Arc<Session>, Box<dyn std::error::Error + Send + Sync>>(session)
         });

src/bendpy/src/context.rs

@@ -38,6 +38,13 @@ mod user_stage;
 mod ownership_object;
 
 mod auto_increment;
+
+/// Built-in role name with full administrative privileges.
+pub const BUILTIN_ROLE_ACCOUNT_ADMIN: &str = "account_admin";
+
+/// Built-in role that every user inherits by default.
+pub const BUILTIN_ROLE_PUBLIC: &str = "public";
+
 pub mod client_session;
 pub mod client_session_ident;
 pub mod connection_ident;

src/meta/app/src/principal/mod.rs

@@ -29,6 +29,7 @@ use crate::principal::AuthInfo;
 use crate::principal::UserGrantSet;
 use crate::principal::UserIdentity;
 use crate::principal::UserQuota;
+use crate::principal::BUILTIN_ROLE_ACCOUNT_ADMIN;
 
 #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Default)]
 #[serde(default)]
@@ -103,7 +104,9 @@ impl UserInfo {
     }
 
     pub fn is_account_admin(&self) -> bool {
-        self.grants.roles().contains(&"account_admin".to_string())
+        self.grants
+            .roles()
+            .contains(&BUILTIN_ROLE_ACCOUNT_ADMIN.to_string())
     }
 
     pub fn has_option_flag(&self, flag: UserOptionFlag) -> bool {

src/meta/app/src/principal/user_info.rs

@@ -45,7 +45,7 @@ fn test_decode_v136_add_task() -> anyhow::Result<()> {
         when_condition: Some("c1 > 1".to_string()),
         after: vec!["task_a".to_string(), "task_b".to_string()],
         comment: Some("comment".to_string()),
-        owner: "public".to_string(),
+        owner: mt::BUILTIN_ROLE_PUBLIC.to_string(),
         owner_user: "me".to_string(),
         schedule_options: Some(ScheduleOptions {
             interval: Some(11),
@@ -82,7 +82,7 @@ fn test_decode_v136_task_message() -> anyhow::Result<()> {
         when_condition: Some("c1 > 1".to_string()),
         after: vec!["task_a".to_string(), "task_b".to_string()],
         comment: Some("comment".to_string()),
-        owner: "public".to_string(),
+        owner: mt::BUILTIN_ROLE_PUBLIC.to_string(),
         owner_user: "me".to_string(),
         schedule_options: Some(ScheduleOptions {
             interval: Some(11),

src/meta/proto-conv/tests/it/v136_add_task.rs

@@ -31,7 +31,7 @@ fn test_decode_v140_task_message() -> anyhow::Result<()> {
         when_condition: Some("c1 > 1".to_string()),
         after: vec!["task_a".to_string(), "task_b".to_string()],
         comment: Some("comment".to_string()),
-        owner: "public".to_string(),
+        owner: mt::BUILTIN_ROLE_PUBLIC.to_string(),
         owner_user: "me".to_string(),
         schedule_options: Some(ScheduleOptions {
             interval: Some(11),

src/meta/proto-conv/tests/it/v140_task_message.rs

@@ -33,6 +33,7 @@ use databend_common_meta_app::principal::RoleIdent;
 use databend_common_meta_app::principal::RoleInfo;
 use databend_common_meta_app::principal::TenantOwnershipObjectIdent;
 use databend_common_meta_app::principal::UserPrivilegeType;
+use databend_common_meta_app::principal::BUILTIN_ROLE_ACCOUNT_ADMIN;
 use databend_common_meta_app::tenant::Tenant;
 use databend_common_meta_app::tenant_key::errors::ExistError;
 use databend_common_meta_app::KeyWithTenant;
@@ -68,8 +69,6 @@ use crate::serialize_struct;
 
 static TXN_MAX_RETRY_TIMES: u32 = 60;
 
-static BUILTIN_ROLE_ACCOUNT_ADMIN: &str = "account_admin";
-
 #[derive(Clone)]
 pub struct RoleMgr {
     kv_api: Arc<dyn kvapi::KVApi<Error = MetaError> + Send + Sync>,

src/query/management/src/role/role_mgr.rs

@@ -237,6 +237,28 @@ impl PrivilegeAccess {
         Ok(())
     }
 
+    async fn get_role_names_and_ownerships(
+        &self,
+        tenant: &Tenant,
+    ) -> Result<(Vec<String>, Vec<SeqV<OwnershipInfo>>)> {
+        let roles = self.ctx.get_all_effective_roles().await?;
+        let roles_name = roles
+            .iter()
+            .map(|role| role.name.to_string())
+            .collect::<Vec<_>>();
+
+        if roles_name
+            .iter()
+            .any(|role_name| role_name == BUILTIN_ROLE_ACCOUNT_ADMIN)
+        {
+            return Ok((roles_name, Vec::new()));
+        }
+
+        let user_api = UserApiProvider::instance();
+        let ownerships = user_api.role_api(tenant).list_ownerships().await?;
+        Ok((roles_name, ownerships))
+    }
+
     async fn validate_db_access(
         &self,
         catalog_name: &str,
@@ -961,13 +983,8 @@ impl AccessChecker for PrivilegeAccess {
                             return Ok(());
                         }
 
-                        let user_api = UserApiProvider::instance();
-                        let ownerships = user_api
-                            .role_api(&tenant)
-                            .list_ownerships()
-                            .await?;
-                        let roles = self.ctx.get_all_effective_roles().await?;
-                        let roles_name: Vec<String> = roles.iter().map(|role| role.name.to_string()).collect();
+                        let (roles_name, ownerships) =
+                            self.get_role_names_and_ownerships(&tenant).await?;
                         check_db_tb_ownership_access(&identity, catalog, database, show_db_id, &ownerships, &roles_name)?;
                     }
                     Some(RewriteKind::ShowStreams(database)) => {
@@ -979,13 +996,8 @@ impl AccessChecker for PrivilegeAccess {
                         if has_priv(&tenant, database, None, show_db_id, table_id, grant_set, true).await? {
                             return Ok(());
                         }
-                        let user_api = UserApiProvider::instance();
-                        let ownerships = user_api
-                            .role_api(&tenant)
-                            .list_ownerships()
-                            .await?;
-                        let roles = self.ctx.get_all_effective_roles().await?;
-                        let roles_name: Vec<String> = roles.iter().map(|role| role.name.to_string()).collect();
+                        let (roles_name, ownerships) =
+                            self.get_role_names_and_ownerships(&tenant).await?;
                         check_db_tb_ownership_access(&identity, &ctl_name, database, show_db_id, &ownerships, &roles_name)?;
                     }
                     Some(RewriteKind::ShowColumns(catalog_name, database, table)) => {
@@ -1140,13 +1152,8 @@ impl AccessChecker for PrivilegeAccess {
                 if has_priv(&tenant, &plan.database, None, show_db_id, None, grant_set, true).await? {
                     return Ok(());
                 }
-                let user_api = UserApiProvider::instance();
-                let ownerships = user_api
-                    .role_api(&tenant)
-                    .list_ownerships()
-                    .await?;
-                let roles = self.ctx.get_all_effective_roles().await?;
-                let roles_name: Vec<String> = roles.iter().map(|role| role.name.to_string()).collect();
+                let (roles_name, ownerships) =
+                    self.get_role_names_and_ownerships(&tenant).await?;
                 check_db_tb_ownership_access(&identity, &ctl_name, &plan.database, show_db_id, &ownerships, &roles_name)?;
             }
 
@@ -1709,7 +1716,10 @@ fn check_db_tb_ownership_access(
 ) -> Result<()> {
     // If contains account_admin even though the current role is not account_admin,
     // It also as a admin user.
-    if roles_name.contains(&"account_admin".to_string()) {
+    if roles_name
+        .iter()
+        .any(|role_name| role_name == BUILTIN_ROLE_ACCOUNT_ADMIN)
+    {
         return Ok(());
     }
 

src/query/service/src/interpreters/access/privilege_access.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use databend_common_exception::Result;
+use databend_common_users::BUILTIN_ROLE_PUBLIC;
 use poem::error::InternalServerError;
 use poem::error::Result as PoemResult;
 use poem::web::Json;
@@ -22,8 +23,6 @@ use serde::Serialize;
 
 use crate::servers::http::v1::HttpQueryContext;
 
-const PUBLIC_ROLE: &str = "public";
-
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ListRolesResponse {
     pub roles: Vec<RoleInfo>,
@@ -46,11 +45,11 @@ async fn handle(ctx: &HttpQueryContext) -> Result<ListRolesResponse> {
     let current_role = ctx
         .session
         .get_current_role()
-        .map_or(PUBLIC_ROLE.to_string(), |role| role.name);
+        .map_or(BUILTIN_ROLE_PUBLIC.to_string(), |role| role.name);
     let default_role = current_user
         .option
         .default_role()
-        .map_or(PUBLIC_ROLE.to_string(), |role| role.to_string());
+        .map_or(BUILTIN_ROLE_PUBLIC.to_string(), |role| role.to_string());
     let mut roles = vec![];
     for role in all_roles {
         let is_current = role.name == current_role;
@@ -63,7 +62,7 @@ async fn handle(ctx: &HttpQueryContext) -> Result<ListRolesResponse> {
     }
     if roles.is_empty() {
         roles.push(RoleInfo {
-            name: PUBLIC_ROLE.to_string(),
+            name: BUILTIN_ROLE_PUBLIC.to_string(),
             is_current: true,
             is_default: true,
         });