Skip to content

feat: enable datasource support for gcs storage #7258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Standing-Man wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

feat: enable datasource support for gcs storage #7258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e933791
feat: enable datasource support for gcs storage
Standing-Man Nov 19, 2025
6a2ee74
fix: apply suggestion
killme2008 Dec 10, 2025
70202cb
fix: style
killme2008 Dec 10, 2025
7449008
fix: typo
killme2008 Dec 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions src/cli/src/data/export.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,40 @@ pub struct ExportCommand {
/// if oss is set, this is required
#[clap(long)]
oss_access_key_secret: Option<String>,

/// if export data to gcs
#[clap(long)]
gcs: bool,

/// The gcs bucket name
/// if gcs is set, this is required
#[clap(long)]
gcs_bucket: Option<String>,

/// The gcs root path
/// if gcs is set, this is required
#[clap(long)]
gcs_root: Option<String>,

/// The gcs endpoint
/// if gcs is set, this is required
#[clap(long)]
gcs_endpoint: Option<String>,

/// The gcs credential
/// if gcs is set, this is required
#[clap(long)]
gcs_credential: Option<String>,

/// The gcs predefined_acl
/// if gcs is set, this is required
#[clap(long, alias = "gcs-acl")]
gcs_predefined_acl: Option<String>,

/// The gcs default_storage_class
/// if gcs is set, this is required
#[clap(long, alias = "gcs-class")]
gcs_default_storage_class: Option<String>,
}

impl ExportCommand {
Expand Down
11 changes: 10 additions & 1 deletion src/common/datasource/src/object_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.

pub mod fs;
pub mod gcs;
pub mod oss;
pub mod s3;

Expand All @@ -25,14 +26,16 @@ use snafu::{OptionExt, ResultExt};
use url::{ParseError, Url};

use self::fs::build_fs_backend;
use self::gcs::build_gcs_backend;
use self::oss::build_oss_backend;
use self::s3::build_s3_backend;
use crate::error::{self, Result};
use crate::object_store::oss::build_oss_backend;
use crate::util::find_dir_and_filename;

pub const FS_SCHEMA: &str = "FS";
pub const S3_SCHEMA: &str = "S3";
pub const OSS_SCHEMA: &str = "OSS";
pub const GCS_SCHEMA: &str = "GCS";

/// Returns `(schema, Option<host>, path)`
pub fn parse_url(url: &str) -> Result<(String, Option<String>, String)> {
Expand Down Expand Up @@ -75,6 +78,12 @@ pub fn build_backend(url: &str, connection: &HashMap<String, String>) -> Result<
Ok(build_oss_backend(&host, &root, connection)?)
}
FS_SCHEMA => Ok(build_fs_backend(&root)?),
GCS_SCHEMA => {
let host = host.context(error::EmptyHostPathSnafu {
url: url.to_string(),
})?;
Ok(build_gcs_backend(&host, &root, connection)?)
}

_ => error::UnsupportedBackendProtocolSnafu {
protocol: schema,
Expand Down
111 changes: 111 additions & 0 deletions src/common/datasource/src/object_store/gcs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::HashMap;

use object_store::ObjectStore;
use object_store::services::Gcs;
use snafu::ResultExt;

use crate::error::{self, Result};

const ROOT: &str = "root";
const BUCKET: &str = "bucket";
const ENDPOINT: &str = "endpoint";
const CREDENTIAL: &str = "credential";
const PREDEFINED_ACL: &str = "predefined_acl";
const DEFAULT_STORAGE_CLASS: &str = "default_storage_class";

pub fn is_supported_in_gcs(key: &str) -> bool {
[
ENDPOINT,
ROOT,
BUCKET,
CREDENTIAL,
PREDEFINED_ACL,
DEFAULT_STORAGE_CLASS,
]
.contains(&key)
}

/// Build a gcs backend using the provided bucket, root, and connection parameters.
pub fn build_gcs_backend(
bucket: &str,
root: &str,
connection: &HashMap<String, String>,
) -> Result<ObjectStore> {
let mut builder = Gcs::default().bucket(bucket).root(root);

if let Some(endpoint) = connection.get(ENDPOINT) {
builder = builder.endpoint(endpoint);
}

if let Some(credential) = connection.get(CREDENTIAL) {
builder = builder.credential(credential);
}

if let Some(predefined_acl) = connection.get(PREDEFINED_ACL) {
builder = builder.predefined_acl(predefined_acl);
}

if let Some(default_storage_class) = connection.get(DEFAULT_STORAGE_CLASS) {
builder = builder.default_storage_class(default_storage_class);
}

let op = ObjectStore::new(builder)
.context(error::BuildBackendSnafu)?
.layer(
object_store::layers::RetryLayer::new()
.with_jitter()
.with_notify(object_store::util::PrintDetailedError),
)
.layer(object_store::layers::LoggingLayer::default())
.layer(object_store::layers::TracingLayer)
.layer(object_store::layers::build_prometheus_metrics_layer(true))
.finish();

Ok(op)
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_is_supported_in_gcs() {
assert!(is_supported_in_gcs(ENDPOINT));
assert!(is_supported_in_gcs(ROOT));
assert!(is_supported_in_gcs(BUCKET));
assert!(is_supported_in_gcs(CREDENTIAL));
assert!(is_supported_in_gcs(PREDEFINED_ACL));
assert!(is_supported_in_gcs(DEFAULT_STORAGE_CLASS));
assert!(!is_supported_in_gcs("hahahhah"))
}

#[test]
fn test_build_gcs_backend() {
let mut connection = HashMap::new();

connection.insert(
ENDPOINT.to_string(),
"https://storage.googleapis.com".to_string(),
);
connection.insert(CREDENTIAL.to_string(), "authentication token".to_string());
connection.insert(PREDEFINED_ACL.to_string(), "publicRead".to_string());
connection.insert(DEFAULT_STORAGE_CLASS.to_string(), "STANDARD".to_string());

let result = build_gcs_backend("my-bucket", "my-root", &connection);
assert!(result.is_ok());
}
}
5 changes: 5 additions & 0 deletions src/table/src/requests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use std::fmt;
use std::str::FromStr;

use common_base::readable_size::ReadableSize;
use common_datasource::object_store::gcs::is_supported_in_gcs;
use common_datasource::object_store::oss::is_supported_in_oss;
use common_datasource::object_store::s3::is_supported_in_s3;
use common_query::AddColumnLocation;
Expand Down Expand Up @@ -107,6 +108,10 @@ pub fn validate_table_option(key: &str) -> bool {
return true;
}

if is_supported_in_gcs(key) {
return true;
}

if is_mito_engine_option_key(key) {
return true;
}
Expand Down