Skip to content

Commit 393c857

Browse files
TCeasonTCeason
authored
Merge branch 'main' into tag_object-2
2 parents 04b3613 + d7c8bc6 commit 393c857

File tree

87 files changed

+1973
-2102
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+1973
-2102
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/bendpy/src/lib.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,9 @@ fn create_embedded_config(
8989

9090
// Query configuration
9191
conf.query.tenant_id = Tenant::new_literal("python_binding");
92-
conf.query.embedded_mode = true;
93-
conf.query.cluster_id = "".to_string();
94-
conf.query.warehouse_id = "".to_string();
92+
conf.query.common.embedded_mode = true;
93+
conf.query.common.cluster_id = "".to_string();
94+
conf.query.common.warehouse_id = "".to_string();
9595
conf.query.node_id = "embedded_node".to_string();
9696

9797
// Logging configuration
@@ -136,8 +136,8 @@ fn create_embedded_config(
136136
let total_memory = system.total_memory();
137137

138138
// Set max server memory usage to 80% of system memory
139-
conf.query.max_server_memory_usage = (total_memory as f64 * 0.8) as u64;
140-
conf.query.max_memory_limit_enabled = true;
139+
conf.query.common.max_server_memory_usage = (total_memory as f64 * 0.8) as u64;
140+
conf.query.common.max_memory_limit_enabled = true;
141141

142142
// Enable spill when memory usage exceeds 60% of system memory
143143
conf.spill.global_bytes_limit = (total_memory as f64 * 0.6) as u64;
@@ -211,8 +211,8 @@ fn create_python_binding_telemetry_payload(config: &InnerConfig) -> serde_json::
211211
"disk_cache_max_bytes": config.cache.disk_cache_config.max_bytes
212212
},
213213
"memory_management": {
214-
"max_server_memory_usage": config.query.max_server_memory_usage,
215-
"max_memory_limit_enabled": config.query.max_memory_limit_enabled,
214+
"max_server_memory_usage": config.query.common.max_server_memory_usage,
215+
"max_memory_limit_enabled": config.query.common.max_memory_limit_enabled,
216216
"spill_enabled": config.spill.global_bytes_limit > 0,
217217
"spill_threshold_bytes": config.spill.global_bytes_limit,
218218
"spill_threshold_percent": 60.0,

‎src/binaries/query/entry.rs‎

Lines changed: 33 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ pub async fn init_services(conf: &InnerConfig, ee_mode: bool) -> Result<(), Main
101101
async fn precheck_services(conf: &InnerConfig) -> Result<(), MainError> {
102102
let make_error = || "failed to precheck";
103103

104-
if conf.query.max_memory_limit_enabled {
105-
let size = conf.query.max_server_memory_usage as i64;
104+
if conf.query.common.max_memory_limit_enabled {
105+
let size = conf.query.common.max_server_memory_usage as i64;
106106
info!("Set memory limit: {}", size);
107107
GLOBAL_MEM_STAT.set_limit(size, false);
108108
}
@@ -136,13 +136,13 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
136136
.with_context(make_error)?;
137137
info!(
138138
"Databend query has been registered:{:?}/{:?} to metasrv:{:?}.",
139-
conf.query.warehouse_id, conf.query.cluster_id, conf.meta.endpoints
139+
conf.query.common.warehouse_id, conf.query.common.cluster_id, conf.meta.endpoints
140140
);
141141
}
142142

143143
// RPC API service.
144144
{
145-
let address = conf.query.flight_api_address.clone();
145+
let address = conf.query.common.flight_api_address.clone();
146146
let mut srv = FlightService::create(conf.clone()).with_context(make_error)?;
147147
let listening = srv
148148
.start(address.parse().with_context(make_error)?)
@@ -154,12 +154,12 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
154154

155155
// MySQL handler.
156156
{
157-
let hostname = conf.query.mysql_handler_host.clone();
158-
let listening = format!("{}:{}", hostname, conf.query.mysql_handler_port);
159-
let tcp_keepalive_timeout_secs = conf.query.mysql_handler_tcp_keepalive_timeout_secs;
157+
let hostname = conf.query.common.mysql_handler_host.clone();
158+
let listening = format!("{}:{}", hostname, conf.query.common.mysql_handler_port);
159+
let tcp_keepalive_timeout_secs = conf.query.common.mysql_handler_tcp_keepalive_timeout_secs;
160160
let tls_config = MySQLTlsConfig::new(
161-
conf.query.mysql_tls_server_cert.clone(),
162-
conf.query.mysql_tls_server_key.clone(),
161+
conf.query.common.mysql_tls_server_cert.clone(),
162+
conf.query.common.mysql_tls_server_key.clone(),
163163
);
164164

165165
let mut handler = MySQLHandler::create(tcp_keepalive_timeout_secs, tls_config)
@@ -180,8 +180,11 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
180180

181181
// ClickHouse HTTP handler.
182182
{
183-
let hostname = conf.query.clickhouse_http_handler_host.clone();
184-
let listening = format!("{}:{}", hostname, conf.query.clickhouse_http_handler_port);
183+
let hostname = conf.query.common.clickhouse_http_handler_host.clone();
184+
let listening = format!(
185+
"{}:{}",
186+
hostname, conf.query.common.clickhouse_http_handler_port
187+
);
185188

186189
let mut srv = HttpHandler::create(HttpHandlerKind::Clickhouse);
187190
let listening = srv
@@ -199,8 +202,8 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
199202

200203
// Databend HTTP handler.
201204
{
202-
let hostname = conf.query.http_handler_host.clone();
203-
let listening = format!("{}:{}", hostname, conf.query.http_handler_port);
205+
let hostname = conf.query.common.http_handler_host.clone();
206+
let listening = format!("{}:{}", hostname, conf.query.common.http_handler_port);
204207

205208
let mut srv = HttpHandler::create(HttpHandlerKind::Query);
206209
let listening = srv
@@ -219,7 +222,7 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
219222
// Metric API service.
220223
{
221224
set_system_version("query", DATABEND_GIT_SEMVER, VERGEN_GIT_SHA);
222-
let address = conf.query.metric_api_address.clone();
225+
let address = conf.query.common.metric_api_address.clone();
223226
let mut srv = MetricService::create();
224227
let listening = srv
225228
.start(address.parse().with_context(make_error)?)
@@ -231,7 +234,7 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
231234

232235
// Admin HTTP API service.
233236
{
234-
let address = conf.query.admin_api_address.clone();
237+
let address = conf.query.common.admin_api_address.clone();
235238
let mut srv = AdminService::create(conf);
236239
let listening = srv
237240
.start(address.parse().with_context(make_error)?)
@@ -245,7 +248,7 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
245248
{
246249
let address = format!(
247250
"{}:{}",
248-
conf.query.flight_sql_handler_host, conf.query.flight_sql_handler_port
251+
conf.query.common.flight_sql_handler_host, conf.query.common.flight_sql_handler_port
249252
);
250253
let mut srv =
251254
FlightSQLServer::create(conf.clone(), &BUILD_INFO).with_context(make_error)?;
@@ -307,10 +310,10 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
307310
println!();
308311
println!("Memory:");
309312
println!(" limit: {}", {
310-
if conf.query.max_memory_limit_enabled {
313+
if conf.query.common.max_memory_limit_enabled {
311314
format!(
312315
"Memory: server memory limit to {} (bytes)",
313-
conf.query.max_server_memory_usage
316+
conf.query.common.max_server_memory_usage
314317
)
315318
} else {
316319
"unlimited".to_string()
@@ -336,10 +339,10 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
336339
println!();
337340
println!("Storage: {}", conf.storage.params);
338341
println!("Disk cache:");
339-
println!(" storage: {}", conf.cache.data_cache_storage);
342+
println!(" storage: {:?}", conf.cache.data_cache_storage);
340343
println!(" path: {:?}", conf.cache.disk_cache_config);
341344
println!(
342-
" reload policy: {}",
345+
" reload policy: {:?}",
343346
conf.cache.data_cache_key_reload_policy
344347
);
345348

@@ -368,38 +371,38 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
368371

369372
println!();
370373
println!("Admin");
371-
println!(" listened at {}", conf.query.admin_api_address);
374+
println!(" listened at {}", conf.query.common.admin_api_address);
372375
println!("MySQL");
373376
println!(
374377
" listened at {}:{}",
375-
conf.query.mysql_handler_host, conf.query.mysql_handler_port
378+
conf.query.common.mysql_handler_host, conf.query.common.mysql_handler_port
376379
);
377380
println!(
378381
" connect via: mysql -u${{USER}} -p${{PASSWORD}} -h{} -P{}",
379-
conf.query.mysql_handler_host, conf.query.mysql_handler_port
382+
conf.query.common.mysql_handler_host, conf.query.common.mysql_handler_port
380383
);
381384
println!("Databend");
382385
println!(
383386
" listened at {}:{}",
384-
conf.query.http_handler_host, conf.query.http_handler_port
387+
conf.query.common.http_handler_host, conf.query.common.http_handler_port
385388
);
386389

387390
println!(
388391
" usage with args: bendsql -u ${{USER}} -p ${{PASSWORD}} -h {} -P {}",
389-
conf.query.http_handler_host, conf.query.http_handler_port
392+
conf.query.common.http_handler_host, conf.query.common.http_handler_port
390393
);
391394

392395
println!(
393396
" usage with dsn: bendsql --dsn \"databend://${{USER}}:${{PASSWORD}}@{}:{}?sslmode=disable\"",
394-
conf.query.http_handler_host, conf.query.http_handler_port
397+
conf.query.common.http_handler_host, conf.query.common.http_handler_port
395398
);
396399

397400
println!(
398401
" http: {}",
399402
HttpHandlerKind::Query.usage(
400403
format!(
401404
"{}:{}",
402-
conf.query.http_handler_host, conf.query.http_handler_port
405+
conf.query.common.http_handler_host, conf.query.common.http_handler_port
403406
)
404407
.parse()
405408
.with_context(make_error)?
@@ -421,8 +424,9 @@ pub async fn start_services(conf: &InnerConfig) -> Result<(), MainError> {
421424
start_time.elapsed().as_secs_f32()
422425
);
423426

424-
let graceful_shutdown_timeout =
425-
Some(Duration::from_millis(conf.query.shutdown_wait_timeout_ms));
427+
let graceful_shutdown_timeout = Some(Duration::from_millis(
428+
conf.query.common.shutdown_wait_timeout_ms,
429+
));
426430
shutdown_handle
427431
.wait_for_termination_request(graceful_shutdown_timeout)
428432
.await;

‎src/meta/app/src/principal/user_grant.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use crate::principal::UserPrivilegeType;
2424
// some statements like `SELECT 1`, `SHOW USERS`, `SHOW ROLES`, `SHOW TABLES` will be
2525
// rewritten to the queries on the system tables, we need to skip the privilege check on
2626
// these tables.
27-
pub const SYSTEM_TABLES_ALLOW_LIST: [&str; 21] = [
27+
pub const SYSTEM_TABLES_ALLOW_LIST: [&str; 22] = [
2828
"catalogs",
2929
"columns",
3030
"databases",
@@ -44,6 +44,7 @@ pub const SYSTEM_TABLES_ALLOW_LIST: [&str; 21] = [
4444
"one",
4545
"processes",
4646
"user_functions",
47+
"procedures",
4748
"functions",
4849
"indexes",
4950
];

‎src/meta/app/src/principal/user_privilege.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ impl UserPrivilegeSet {
239239
let wh_privs_without_ownership = Self::available_privileges_on_warehouse(false);
240240
let connection_privs_without_ownership = Self::available_privileges_on_connection(false);
241241
let seq_privs_without_ownership = Self::available_privileges_on_sequence(false);
242+
let procedure_privs_without_ownership = Self::available_privileges_on_procedure(false);
242243
let mask_privs = Self::available_privileges_on_masking_policy(false);
243244
let row_access_privs = Self::available_privileges_on_row_access_policy(false);
244245
let privs = make_bitflags!(UserPrivilegeType::{ Usage | Super | CreateUser | DropUser | CreateRole | DropRole | CreateDatabase | Grant | CreateDataMask | CreateMaskingPolicy | CreateRowAccessPolicy | CreateWarehouse | CreateConnection | CreateSequence | CreateProcedure });
@@ -248,6 +249,7 @@ impl UserPrivilegeSet {
248249
| wh_privs_without_ownership.privileges
249250
| connection_privs_without_ownership.privileges
250251
| seq_privs_without_ownership.privileges
252+
| procedure_privs_without_ownership.privileges
251253
| udf_privs_without_ownership.privileges
252254
| mask_privs.privileges
253255
| row_access_privs.privileges)

‎src/query/catalog/src/catalog/manager.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ use std::sync::Arc;
1818
use chrono::Utc;
1919
use databend_common_base::base::BuildInfoRef;
2020
use databend_common_base::base::GlobalInstance;
21-
use databend_common_config::CatalogConfig;
2221
use databend_common_config::InnerConfig;
2322
use databend_common_exception::ErrorCode;
2423
use databend_common_exception::Result;
@@ -107,11 +106,19 @@ impl CatalogManager {
107106
// init external catalogs.
108107
let mut external_catalogs = HashMap::default();
109108
for (name, ctl_cfg) in conf.catalogs.iter() {
110-
let CatalogConfig::Hive(hive_ctl_cfg) = ctl_cfg;
111109
let creator = catalog_creators.get(&CatalogType::Hive).ok_or_else(|| {
112110
ErrorCode::BadArguments(format!("unknown catalog type: {:?}", CatalogType::Hive))
113111
})?;
114112

113+
if ctl_cfg.ty.as_str() != "hive" {
114+
return Err(ErrorCode::CatalogNotSupported(format!(
115+
"got unsupported catalog type in config: {}",
116+
ctl_cfg.ty
117+
)));
118+
}
119+
120+
let hive_ctl_cfg = &ctl_cfg.hive;
121+
115122
let ctl_info = CatalogInfo {
116123
id: CatalogIdIdent::new(&tenant, 0).into(),
117124
name_ident: CatalogNameIdent::new(tenant.clone(), name).into(),

‎src/query/config/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ serfig = { workspace = true }
3434
toml = { workspace = true }
3535

3636
[dev-dependencies]
37+
temp-env = { workspace = true }
3738

3839
[lints]
3940
workspace = true

0 commit comments

Comments
 (0)