From 5de7d02d810af546ef3c80b9fafeddf6a43fc7a5 Mon Sep 17 00:00:00 2001 From: BohuTANG Date: Wed, 5 Nov 2025 19:29:01 +0800 Subject: [PATCH 1/2] docs: refresh connection parameter guides --- .../00-sql-reference/51-connect-parameters.md | 225 ++++++++++-------- .../00-ddl/13-connection/create-connection.md | 127 +++++++--- .../00-sql-reference/51-connect-parameters.md | 123 ++++++---- .../00-ddl/13-connection/create-connection.md | 103 ++++++-- 4 files changed, 388 insertions(+), 190 deletions(-) diff --git a/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md b/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md index aa6c5127dd..eadd1546c9 100644 --- a/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md +++ b/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md @@ -2,188 +2,217 @@ title: 连接参数 --- import FunctionDescription from '@site/src/components/FunctionDescription'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; -连接参数是用于建立与外部存储服务（如 Amazon S3）安全连接的键值对。这些参数对于创建 Stage、将数据复制到 Databend 以及查询外部文件等任务至关重要。 - -有关每个存储服务的具体连接详细信息，请参见下表。 +运行 `CREATE CONNECTION` 时需要给出一组键值对作为连接参数，用来定义可复用的外部连接。创建好连接后，可以在 Stage、COPY 等语句中通过 `CONNECTION = (CONNECTION_NAME = '')` 直接引用。完整语法请参见 [CREATE CONNECTION](../10-sql-commands/00-ddl/13-connection/create-connection.md)。 -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; +不同存储服务的连接参数见下表。 -下表列出了用于访问类似 Amazon S3 的存储服务的连接参数： +下表列出了访问 Amazon S3 及其兼容存储服务所需的连接参数： | Parameter | Required? | Description | |--------------------------- |----------- |-------------------------------------------------------------- | -| endpoint_url | Yes | 类似 Amazon S3 的存储服务的 Endpoint URL。 | -| access_key_id | Yes | 用于标识请求者的 Access Key ID。 | -| secret_access_key | Yes | 用于身份验证的 Secret Access Key。 | -| enable_virtual_host_style | No | 是否使用虚拟主机样式的 URL。默认为 *false*。 | -| master_key | No | 用于高级数据加密的可选 Master Key。 | -| region | No | Bucket 所在的 AWS 区域。 | -| security_token | No | 用于临时凭证的安全令牌。 | +| endpoint_url | Yes | Amazon S3 兼容存储的 Endpoint URL。 | +| access_key_id | Yes | 请求方的 Access Key ID。 | +| secret_access_key | Yes | 用于认证的 Secret Access Key。 | +| enable_virtual_host_style | No | 是否使用虚拟主机样式 URL，默认 *false*。 | +| master_key | No | 用于高级加密的 Master Key。 | +| region | No | Bucket 所在的 AWS 区域。 | +| security_token | No | 临时凭证的安全令牌。 | :::note -- 如果命令中未指定 **endpoint_url** 参数，则 Databend 默认在 Amazon S3 上创建 Stage。因此，当您在 S3 兼容的对象存储或其他对象存储解决方案上创建外部 Stage 时，请务必包含 **endpoint_url** 参数。 +- 如果命令里没有 **endpoint_url**，Databend 会默认把 Stage 建在 Amazon S3。因此在 S3 兼容对象存储或其他对象存储上建外部 Stage 时，记得显式填写 **endpoint_url**。 -- **region** 参数不是必需的，因为 Databend 可以自动检测区域信息。通常不需要手动为此参数指定值。如果自动检测失败，Databend 将默认使用 'us-east-1' 作为区域。当使用 MinIO 部署 Databend 且未配置区域信息时，它将自动默认使用 'us-east-1'，并且可以正常工作。但是，如果您收到诸如 "region is missing" 或 "The bucket you are trying to access requires a specific endpoint. Please direct all future requests to this particular endpoint" 之类的错误消息，则需要确定您的区域名称并将其显式分配给 **region** 参数。 +- **region** 通常不需手动填写，Databend 会自动识别区域信息；若识别失败，则回落到 `us-east-1`。例如在 MinIO 未配置区域时仍能正常工作。如果看到 “region is missing” 或 “The bucket you are trying to access requires a specific endpoint...” 之类的错误提示，请确认实际区域并为 **region** 指定正确值。 ::: -```sql title='Examples' +```sql title='示例' +-- 创建可复用的 Amazon S3 连接 +CREATE CONNECTION my_s3_conn + STORAGE_TYPE = 's3' + ACCESS_KEY_ID = '' + SECRET_ACCESS_KEY = ''; + +-- 创建 Stage 时引用连接 CREATE STAGE my_s3_stage - 's3://my-bucket' - CONNECTION = ( - ACCESS_KEY_ID = '', - SECRET_ACCESS_KEY = '' - ); + URL = 's3://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_s3_conn'); +-- 为 MinIO 等 S3 兼容服务创建连接 +CREATE CONNECTION my_minio_conn + STORAGE_TYPE = 's3' + ENDPOINT_URL = 'http://localhost:9000' + ACCESS_KEY_ID = 'ROOTUSER' + SECRET_ACCESS_KEY = 'CHANGEME123'; + CREATE STAGE my_minio_stage - 's3://databend' - CONNECTION = ( - ENDPOINT_URL = 'http://localhost:9000', - ACCESS_KEY_ID = 'ROOTUSER', - SECRET_ACCESS_KEY = 'CHANGEME123' - ); + URL = 's3://databend' + CONNECTION = (CONNECTION_NAME = 'my_minio_conn'); ``` +也可以改用 AWS IAM Role 和 External ID 认证，以更细粒度地控制可访问的 S3 Bucket，同时增加额外的安全校验。更多背景请参考 https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html。 -要访问您的 Amazon S3 Bucket，您还可以指定 AWS IAM 角色和外部 ID 进行身份验证。通过指定 AWS IAM 角色和外部 ID，您可以更精细地控制用户可以访问哪些 S3 Bucket。这意味着，如果 IAM 角色已被授予仅访问特定 S3 Bucket 的权限，则用户将只能访问这些 Bucket。外部 ID 可以通过提供额外的验证层来进一步增强安全性。有关更多信息，请参见 https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html - -下表列出了使用 AWS IAM 角色身份验证访问 Amazon S3 存储服务的连接参数： +下表列出了使用 AWS IAM 角色访问 Amazon S3 的连接参数： | Parameter | Required? | Description | |-------------- |----------- |------------------------------------------------------- | | endpoint_url | No | Amazon S3 的 Endpoint URL。 | -| role_arn | Yes | 用于授权访问 S3 的 AWS IAM 角色的 ARN。 | -| external_id | No | 用于增强角色承担安全性的外部 ID。 | - -```sql title='Examples' -CREATE STAGE my_s3_stage - 's3://my-bucket' - CONNECTION = ( - ROLE_ARN = 'arn:aws:iam::123456789012:role/my-role', - EXTERNAL_ID = 'my-external-id' - ); +| role_arn | Yes | 用于授权访问 S3 的 AWS IAM 角色 ARN。 | +| external_id | No | 用于增强角色授权安全性的 External ID。 | + +```sql title='示例' +-- 使用 IAM 角色创建连接 +CREATE CONNECTION my_iam_conn + STORAGE_TYPE = 's3' + ROLE_ARN = 'arn:aws:iam::123456789012:role/my-role' + EXTERNAL_ID = 'my-external-id'; + +-- 创建 Stage 时引用连接 +CREATE STAGE my_iam_stage + URL = 's3://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_iam_conn'); ``` -下表列出了用于访问 Azure Blob Storage 的连接参数： +下表列出了访问 Azure Blob Storage 的连接参数： | Parameter | Required? | Description | |----------------|-------------|-------------------------------------------------------| | endpoint_url | Yes | Azure Blob Storage 的 Endpoint URL。 | -| account_key | Yes | 用于身份验证的 Azure Blob Storage 帐户密钥。 | -| account_name | Yes | 用于标识的 Azure Blob Storage 帐户名称。 | - -```sql title='Examples' +| account_key | Yes | Azure Blob Storage 帐户密钥。 | +| account_name | Yes | Azure Blob Storage 帐户名称。 | + +```sql title='示例' +-- 创建 Azure Blob Storage 连接 +CREATE CONNECTION my_azure_conn + STORAGE_TYPE = 'azblob' + ACCOUNT_NAME = 'myaccount' + ACCOUNT_KEY = 'myaccountkey' + ENDPOINT_URL = 'https://.blob.core.windows.net'; + +-- 创建 Stage 并引用该连接 CREATE STAGE my_azure_stage - 'azblob://my-container' - CONNECTION = ( - ACCOUNT_NAME = 'myaccount', - ACCOUNT_KEY = 'myaccountkey', - ENDPOINT_URL = 'https://.blob.core.windows.net' - ); + URL = 'azblob://my-container' + CONNECTION = (CONNECTION_NAME = 'my_azure_conn'); ``` -下表列出了用于访问 Google Cloud Storage 的连接参数： +下表列出了访问 Google Cloud Storage 的连接参数： | Parameter | Required? | Description | |----------------|-------------|-------------------------------------------------------| -| credential | Yes | 用于身份验证的 Google Cloud Storage 凭据。 | +| credential | Yes | 用于认证的 Google Cloud Storage 凭证。 | -要获取 `credential`，您可以按照 Google 文档中的主题 [创建服务帐户密钥](https://cloud.google.com/iam/docs/keys-create-delete#creating) -创建一个服务帐户密钥文件并下载。下载服务帐户密钥文件后，您可以 -通过以下命令将其转换为 base64 字符串： +可以按照 Google 文档 [Create a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating) 生成服务账号密钥文件，并用下面的命令把它转成 base64： ``` -base64 -i -o ~/Desktop/base64-encoded-key.txt +base64 -i -o ~/Desktop/base64-encoded-key.txt ``` -```sql title='Examples' +```sql title='示例' +-- 创建 GCS 连接 +CREATE CONNECTION my_gcs_conn + STORAGE_TYPE = 'gcs' + CREDENTIAL = ''; + +-- 创建 Stage 时引用连接 CREATE STAGE my_gcs_stage - 'gcs://my-bucket' - CONNECTION = ( - CREDENTIAL = '' - ); + URL = 'gcs://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_gcs_conn'); ``` -下表列出了用于访问阿里云 OSS 的连接参数： +下表列出了访问阿里云 OSS 的连接参数： | Parameter | Required? | Description | |---------------------- |----------- |--------------------------------------------------------- | -| access_key_id | Yes | 用于身份验证的阿里云 OSS Access Key ID。 | -| access_key_secret | Yes | 用于身份验证的阿里云 OSS Access Key Secret。 | +| access_key_id | Yes | 阿里云 OSS Access Key ID。 | +| access_key_secret | Yes | 阿里云 OSS Access Key Secret。 | | endpoint_url | Yes | 阿里云 OSS 的 Endpoint URL。 | -| presign_endpoint_url | No | 用于预签名阿里云 OSS URL 的 Endpoint URL。 | +| presign_endpoint_url | No | 用于预签名 URL 的 Endpoint。 | + +```sql title='示例' +-- 创建阿里云 OSS 连接 +CREATE CONNECTION my_oss_conn + STORAGE_TYPE = 'oss' + ACCESS_KEY_ID = '' + ACCESS_KEY_SECRET = '' + ENDPOINT_URL = 'https://.[-internal].aliyuncs.com'; -```sql title='Examples' +-- 创建 Stage 并引用该连接 CREATE STAGE my_oss_stage - 'oss://my-bucket' - CONNECTION = ( - ACCESS_KEY_ID = '', - ACCESS_KEY_SECRET = '', - ENDPOINT_URL = 'https://.[-internal].aliyuncs.com' - ); + URL = 'oss://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_oss_conn'); ``` -下表列出了用于访问腾讯云对象存储（COS）的连接参数： +下表列出了访问腾讯云对象存储（COS）的连接参数： | Parameter | Required? | Description | |-------------- |----------- |------------------------------------------------------------- | -| endpoint_url | Yes | 腾讯云对象存储的 Endpoint URL。 | -| secret_id | Yes | 用于身份验证的腾讯云对象存储 Secret ID。 | -| secret_key | Yes | 用于身份验证的腾讯云对象存储 Secret Key。 | - -```sql title='Examples' +| endpoint_url | Yes | 腾讯云 COS 的 Endpoint URL。 | +| secret_id | Yes | 腾讯云 COS Secret ID。 | +| secret_key | Yes | 腾讯云 COS Secret Key。 | + +```sql title='示例' +-- 创建腾讯云 COS 连接 +CREATE CONNECTION my_cos_conn + STORAGE_TYPE = 'cos' + SECRET_ID = '' + SECRET_KEY = '' + ENDPOINT_URL = ''; + +-- 创建 Stage 并引用该连接 CREATE STAGE my_cos_stage - 'cos://my-bucket' - CONNECTION = ( - SECRET_ID = '', - SECRET_KEY = '', - ENDPOINT_URL = '' - ); + URL = 'cos://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_cos_conn'); ``` -下表列出了用于访问 Hugging Face 的连接参数： +下表列出了访问 Hugging Face 的连接参数： | Parameter | Required? | Description | |-----------|-----------------------|-----------------------------------------------------------------------------------------------------------------| -| repo_type | No (default: dataset) | Hugging Face 仓库的类型。可以是 `dataset` 或 `model`。 | -| revision | No (default: main) | Hugging Face URI 的修订版本。可以是仓库的分支、标签或提交。 | -| token | No | 来自 Hugging Face 的 API 令牌，可能需要用于访问私有仓库或某些资源。 | - -```sql title='Examples' +| repo_type | No (default: dataset) | Hugging Face 仓库类型，可取 `dataset` 或 `model`。 | +| revision | No (default: main) | Hugging Face URI 的版本，可为分支、标签或提交。 | +| token | No | Hugging Face API Token，访问私有仓库或部分资源时需要。 | + +```sql title='示例' +-- 创建 Hugging Face 连接 +CREATE CONNECTION my_hf_conn + STORAGE_TYPE = 'hf' + REPO_TYPE = 'dataset' + REVISION = 'main' + TOKEN = ''; + +-- 创建 Stage 并引用该连接 CREATE STAGE my_huggingface_stage - 'hf://opendal/huggingface-testdata/' - CONNECTION = ( - REPO_TYPE = 'dataset' - REVISION = 'main' - ); + URL = 'hf://opendal/huggingface-testdata/' + CONNECTION = (CONNECTION_NAME = 'my_hf_conn'); ``` - +访问公开仓库可以省略 `TOKEN`，访问私有或受限仓库时再补上。 - \ No newline at end of file + + diff --git a/docs/cn/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md b/docs/cn/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md index dee18ccd06..e55ca97780 100644 --- a/docs/cn/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md +++ b/docs/cn/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md @@ -3,6 +3,8 @@ title: CREATE CONNECTION sidebar_position: 1 --- import FunctionDescription from '@site/src/components/FunctionDescription'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; @@ -20,20 +22,33 @@ CREATE [ OR REPLACE ] CONNECTION [ IF NOT EXISTS ] [ ] ``` -| 参数 | 描述 | -|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| -| STORAGE_TYPE | 存储服务类型。可选值包括：`s3`、`azblob`、`gcs`、`oss` 和 `cos`。 | -| storage_params | 根据存储类型和认证方式而异，详见下文常用认证方式。 | +| 参数 | 描述 | +|-----------------|---------------------------------------------------------------------------------------------------| +| STORAGE_TYPE | 存储服务类型。可选值包括 `s3`、`azblob`、`gcs`、`oss`、`cos` 等。 | +| storage_params | 根据存储类型和认证方式而变化。完整参数请参阅 [连接参数](../../../00-sql-reference/51-connect-parameters.md)。 | -其他存储类型及附加参数，请参见 [连接参数](../../../00-sql-reference/51-connect-parameters.md)。 +## 连接参数 -### Amazon S3 的认证方式 +连接用于封装外部存储的凭据和配置。创建连接时请选择合适的 `STORAGE_TYPE` 并填写所需参数。 -Databend 支持两种 Amazon S3 连接认证方式： +| STORAGE_TYPE | 常见参数 | 说明 | +|--------------|----------|------| +| `s3` | `ACCESS_KEY_ID`/`SECRET_ACCESS_KEY`、或 `ROLE_ARN`/`EXTERNAL_ID`，可选 `ENDPOINT_URL`、`REGION` | Amazon S3 及兼容服务（MinIO、Cloudflare R2 等）。 | +| `azblob` | `ACCOUNT_NAME`、`ACCOUNT_KEY`、`ENDPOINT_URL` | Azure Blob Storage。 | +| `gcs` | `CREDENTIAL`（Base64 编码的服务账号密钥） | Google Cloud Storage。 | +| `oss` | `ACCESS_KEY_ID`、`ACCESS_KEY_SECRET`、`ENDPOINT_URL` | 阿里云对象存储 OSS。 | +| `cos` | `SECRET_ID`、`SECRET_KEY`、`ENDPOINT_URL` | 腾讯云对象存储 COS。 | +| `hf` | `REPO_TYPE`、`REVISION`，可选 `TOKEN` | Hugging Face Hub 数据集与模型。 | -#### 1. 访问密钥认证 +展开下列选项卡以查看各存储类型示例： -使用 AWS 访问密钥进行认证，即传统的 Access Key ID 与 Secret Access Key 方式。 + + + +Amazon S3 及兼容服务支持以下两种认证方式： + + + ```sql CREATE CONNECTION @@ -43,13 +58,12 @@ CREATE CONNECTION ``` | 参数 | 描述 | -|-----------|-------------| +|------|------| | ACCESS_KEY_ID | AWS Access Key ID。 | | SECRET_ACCESS_KEY | AWS Secret Access Key。 | -#### 2. IAM 角色认证 - -使用 AWS IAM 角色进行认证，无需 Access Key，更安全地访问 S3 存储桶。 + + ```sql CREATE CONNECTION @@ -58,32 +72,87 @@ CREATE CONNECTION ``` | 参数 | 描述 | -|-----------|-------------| -| ROLE_ARN | Databend 将扮演的 IAM 角色的 Amazon Resource Name (ARN)。 | +|------|------| +| ROLE_ARN | Databend 将扮演的 IAM 角色的 Amazon Resource Name (ARN)。 | + + -## 访问控制要求 + + -| 权限 | 对象类型 | 描述 | -|:------------------|:------------|:----------------------| -| CREATE CONNECTION | 全局 | 创建连接。 | +```sql +CREATE CONNECTION + STORAGE_TYPE = 'azblob' + ACCOUNT_NAME = '' + ACCOUNT_KEY = '' + ENDPOINT_URL = 'https://.blob.core.windows.net'; +``` + + -创建连接时，执行操作的用户或 [current_role](/guides/security/access-control/roles) 必须拥有 CREATE CONNECTION [权限](/guides/security/access-control/privileges)。 +```sql +CREATE CONNECTION + STORAGE_TYPE = 'gcs' + CREDENTIAL = ''; +``` -:::note + + -`enable_experimental_connection_rbac_check` 设置用于控制连接级访问，默认禁用。 -创建连接仅需超级用户权限，绕过详细 RBAC 检查。启用后，将在建立连接时执行细粒度权限验证。 -此为实验性功能，未来可能默认启用。 +```sql +CREATE CONNECTION + STORAGE_TYPE = 'oss' + ACCESS_KEY_ID = '' + ACCESS_KEY_SECRET = '' + ENDPOINT_URL = 'https://.[-internal].aliyuncs.com'; +``` -::: + + + +```sql +CREATE CONNECTION + STORAGE_TYPE = 'cos' + SECRET_ID = '' + SECRET_KEY = '' + ENDPOINT_URL = ''; +``` + + + + +```sql +CREATE CONNECTION + STORAGE_TYPE = 'hf' + REPO_TYPE = 'dataset' + REVISION = 'main' + TOKEN = ''; +``` + +访问公开仓库时可以省略 `TOKEN`，访问私有或受限资源再补上即可。 + + + + +## 访问控制要求 + +| 权限 | 对象类型 | 描述 | +|:------------------|:---------|:---------| +| CREATE CONNECTION | 全局 | 创建连接 | + +创建连接时，执行操作的用户或 [current_role](/guides/security/access-control/roles) 必须拥有 CREATE CONNECTION [权限](/guides/security/access-control/privileges)。 + +## 更新已存在表的连接 + +若要将现有外部表切换到新的连接，可使用 [`ALTER TABLE ... CONNECTION`](/sql/sql-commands/ddl/table/alter-table-connection) 命令，无需重新创建表。 ## 示例 ### 使用访问密钥 -以下示例创建名为 toronto 的 Amazon S3 连接，并建立名为 my_s3_stage 的外部 Stage，指向 `s3://databend-toronto`，使用 toronto 连接。更多示例请参见 [使用示例](index.md#usage-examples)。 +示例：创建名为 toronto 的 Amazon S3 连接，并建立外部 Stage `my_s3_stage` 指向 `s3://databend-toronto`。更多与连接相关的示例请参阅 [使用示例](index.md#usage-examples)。 ```sql CREATE CONNECTION toronto @@ -98,7 +167,7 @@ CREATE STAGE my_s3_stage ### 使用 AWS IAM 角色 -以下示例使用 IAM 角色创建 Amazon S3 连接，并创建使用该连接的 Stage。无需在 Databend 中存储访问密钥，更加安全。 +示例：使用 IAM Role 创建 Amazon S3 连接，并让 Stage 复用该连接，无需在 Databend 中存储访问密钥。 ```sql CREATE CONNECTION databend_test @@ -114,5 +183,5 @@ SELECT * FROM @databend_test/test.parquet LIMIT 1; ``` :::info -在 Databend Cloud 中使用 IAM 角色，需在 AWS 账户与 Databend Cloud 之间建立信任关系。详见 [使用 AWS IAM 角色创建外部 Stage](/guides/load-data/stage/aws-iam-role)。 -::: \ No newline at end of file +在 Databend Cloud 中使用 IAM 角色，需要在 AWS 账户与 Databend Cloud 之间建立信任关系。详见 [使用 AWS IAM 角色创建外部 Stage](/guides/load-data/stage/aws-iam-role)。 +::: diff --git a/docs/en/sql-reference/00-sql-reference/51-connect-parameters.md b/docs/en/sql-reference/00-sql-reference/51-connect-parameters.md index a8a4a58e1c..f730d5fc5c 100644 --- a/docs/en/sql-reference/00-sql-reference/51-connect-parameters.md +++ b/docs/en/sql-reference/00-sql-reference/51-connect-parameters.md @@ -5,9 +5,9 @@ import FunctionDescription from '@site/src/components/FunctionDescription'; -Connection parameters are key-value pairs used to establish secure links to external storage services like Amazon S3. These parameters are crucial for tasks such as creating stages, copying data into Databend, and querying external files. +Connection parameters are key-value pairs you supply when creating reusable connections with `CREATE CONNECTION`. After a connection is created, reference it from stages, COPY commands, and other SQL features by using `CONNECTION = (CONNECTION_NAME = '')`. For full syntax and usage, see [CREATE CONNECTION](../10-sql-commands/00-ddl/13-connection/create-connection.md). -For specific connection details per storage service, see the tables below. +For storage-specific connection details, see the tables below. import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; @@ -34,20 +34,27 @@ The following table lists connection parameters for accessing an Amazon S3-like ::: ```sql title='Examples' +-- Create a reusable connection for Amazon S3 +CREATE CONNECTION my_s3_conn + STORAGE_TYPE = 's3' + ACCESS_KEY_ID = '' + SECRET_ACCESS_KEY = ''; + +-- Use the connection when creating a stage CREATE STAGE my_s3_stage - 's3://my-bucket' - CONNECTION = ( - ACCESS_KEY_ID = '', - SECRET_ACCESS_KEY = '' - ); + URL = 's3://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_s3_conn'); +-- Create a reusable connection for an S3-compatible service such as MinIO +CREATE CONNECTION my_minio_conn + STORAGE_TYPE = 's3' + ENDPOINT_URL = 'http://localhost:9000' + ACCESS_KEY_ID = 'ROOTUSER' + SECRET_ACCESS_KEY = 'CHANGEME123'; + CREATE STAGE my_minio_stage - 's3://databend' - CONNECTION = ( - ENDPOINT_URL = 'http://localhost:9000', - ACCESS_KEY_ID = 'ROOTUSER', - SECRET_ACCESS_KEY = 'CHANGEME123' - ); + URL = 's3://databend' + CONNECTION = (CONNECTION_NAME = 'my_minio_conn'); ``` @@ -62,12 +69,16 @@ The following table lists connection parameters for accessing Amazon S3 storage | external_id | No | External ID for enhanced security in role assumption. | ```sql title='Examples' -CREATE STAGE my_s3_stage - 's3://my-bucket' - CONNECTION = ( - ROLE_ARN = 'arn:aws:iam::123456789012:role/my-role', - EXTERNAL_ID = 'my-external-id' - ); +-- Create the connection using IAM role authentication +CREATE CONNECTION my_iam_conn + STORAGE_TYPE = 's3' + ROLE_ARN = 'arn:aws:iam::123456789012:role/my-role' + EXTERNAL_ID = 'my-external-id'; + +-- Reference the connection when creating a stage +CREATE STAGE my_iam_stage + URL = 's3://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_iam_conn'); ``` @@ -83,13 +94,17 @@ The following table lists connection parameters for accessing Azure Blob Storage | account_name | Yes | Azure Blob Storage account name for identification. | ```sql title='Examples' +-- Create a connection for Azure Blob Storage +CREATE CONNECTION my_azure_conn + STORAGE_TYPE = 'azblob' + ACCOUNT_NAME = 'myaccount' + ACCOUNT_KEY = 'myaccountkey' + ENDPOINT_URL = 'https://.blob.core.windows.net'; + +-- Create a stage that uses the connection CREATE STAGE my_azure_stage - 'azblob://my-container' - CONNECTION = ( - ACCOUNT_NAME = 'myaccount', - ACCOUNT_KEY = 'myaccountkey', - ENDPOINT_URL = 'https://.blob.core.windows.net' - ); + URL = 'azblob://my-container' + CONNECTION = (CONNECTION_NAME = 'my_azure_conn'); ``` @@ -111,11 +126,15 @@ base64 -i -o ~/Desktop/base64-encoded-key.txt ``` ```sql title='Examples' +-- Create the connection with the base64-encoded credential +CREATE CONNECTION my_gcs_conn + STORAGE_TYPE = 'gcs' + CREDENTIAL = ''; + +-- Use the connection when creating a stage CREATE STAGE my_gcs_stage - 'gcs://my-bucket' - CONNECTION = ( - CREDENTIAL = '' - ); + URL = 'gcs://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_gcs_conn'); ``` @@ -132,13 +151,17 @@ The following table lists connection parameters for accessing Alibaba Cloud OSS: | presign_endpoint_url | No | Endpoint URL for presigning Alibaba Cloud OSS URLs. | ```sql title='Examples' +-- Create a connection for Alibaba Cloud OSS +CREATE CONNECTION my_oss_conn + STORAGE_TYPE = 'oss' + ACCESS_KEY_ID = '' + ACCESS_KEY_SECRET = '' + ENDPOINT_URL = 'https://.[-internal].aliyuncs.com'; + +-- Create a stage using the connection CREATE STAGE my_oss_stage - 'oss://my-bucket' - CONNECTION = ( - ACCESS_KEY_ID = '', - ACCESS_KEY_SECRET = '', - ENDPOINT_URL = 'https://.[-internal].aliyuncs.com' - ); + URL = 'oss://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_oss_conn'); ``` @@ -154,13 +177,17 @@ The following table lists connection parameters for accessing Tencent Cloud Obje | secret_key | Yes | Tencent Cloud Object Storage secret key for authentication. | ```sql title='Examples' +-- Create a connection for Tencent COS +CREATE CONNECTION my_cos_conn + STORAGE_TYPE = 'cos' + SECRET_ID = '' + SECRET_KEY = '' + ENDPOINT_URL = ''; + +-- Create a stage that uses the connection CREATE STAGE my_cos_stage - 'cos://my-bucket' - CONNECTION = ( - SECRET_ID = '', - SECRET_KEY = '', - ENDPOINT_URL = '' - ); + URL = 'cos://my-bucket' + CONNECTION = (CONNECTION_NAME = 'my_cos_conn'); ``` @@ -176,12 +203,16 @@ The following table lists connection parameters for accessing Hugging Face: | token | No | The API token from Hugging Face, which may be required for accessing private repositories or certain resources. | ```sql title='Examples' +-- Create a connection for Hugging Face +CREATE CONNECTION my_hf_conn + STORAGE_TYPE = 'hf' + REPO_TYPE = 'dataset' + REVISION = 'main'; + +-- Create a stage that uses the connection CREATE STAGE my_huggingface_stage - 'hf://opendal/huggingface-testdata/' - CONNECTION = ( - REPO_TYPE = 'dataset' - REVISION = 'main' - ); + URL = 'hf://opendal/huggingface-testdata/' + CONNECTION = (CONNECTION_NAME = 'my_hf_conn'); ``` diff --git a/docs/en/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md b/docs/en/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md index 5539629e89..edea830491 100644 --- a/docs/en/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md +++ b/docs/en/sql-reference/10-sql-commands/00-ddl/13-connection/create-connection.md @@ -3,6 +3,8 @@ title: CREATE CONNECTION sidebar_position: 1 --- import FunctionDescription from '@site/src/components/FunctionDescription'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; @@ -23,17 +25,30 @@ CREATE [ OR REPLACE ] CONNECTION [ IF NOT EXISTS ] | Parameter | Description | |------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| | STORAGE_TYPE | Type of storage service. Possible values include: `s3`, `azblob`, `gcs`, `oss`, and `cos`. | -| storage_params | Vary based on storage type and authentication method. See details below for common authentication methods. | +| storage_params | Vary based on storage type and authentication method. See [Connection Parameters](../../../00-sql-reference/51-connect-parameters.md) for the complete list. | -For other storage types and additional parameters, see [Connection Parameters](../../../00-sql-reference/51-connect-parameters.md) for details. +## Connection Parameters -### Authentication Methods for Amazon S3 +Connections encapsulate the credentials and configuration for a specific storage backend. Choose the appropriate `STORAGE_TYPE` and provide the required parameters when creating the connection. The table highlights common options: -Databend supports two primary authentication methods for Amazon S3 connections: +| STORAGE_TYPE | Typical parameters | Description | +|--------------|-------------------|-------------| +| `s3` | `ACCESS_KEY_ID`/`SECRET_ACCESS_KEY`, or `ROLE_ARN`/`EXTERNAL_ID`, optional `ENDPOINT_URL`, `REGION` | Amazon S3 and S3-compatible services (MinIO, Cloudflare R2, etc.). | +| `azblob` | `ACCOUNT_NAME`, `ACCOUNT_KEY`, `ENDPOINT_URL` | Azure Blob Storage. | +| `gcs` | `CREDENTIAL` (base64-encoded service account key) | Google Cloud Storage. | +| `oss` | `ACCESS_KEY_ID`, `ACCESS_KEY_SECRET`, `ENDPOINT_URL` | Alibaba Cloud Object Storage Service. | +| `cos` | `SECRET_ID`, `SECRET_KEY`, `ENDPOINT_URL` | Tencent Cloud Object Storage. | +| `hf` | `REPO_TYPE`, `REVISION`, optional `TOKEN` | Hugging Face Hub datasets and models. | -#### 1. Access Keys Authentication +For parameter meanings, optional flags, and additional storage types, refer to [Connection Parameters](../../../00-sql-reference/51-connect-parameters.md). Expand the tabs below to see storage-specific examples: -Use AWS access keys for authentication. This is the traditional method using an access key ID and secret access key. + + + +Choose an authentication method for Amazon S3 and S3-compatible services: + + + ```sql CREATE CONNECTION @@ -47,9 +62,8 @@ CREATE CONNECTION | ACCESS_KEY_ID | Your AWS access key ID. | | SECRET_ACCESS_KEY | Your AWS secret access key. | -#### 2. IAM Role Authentication - -Use AWS IAM roles for authentication instead of access keys. This provides a more secure way to access your S3 buckets without managing credentials directly in Databend. + + ```sql CREATE CONNECTION @@ -61,6 +75,67 @@ CREATE CONNECTION |-----------|-------------| | ROLE_ARN | The Amazon Resource Name (ARN) of the IAM role that Databend will assume to access your S3 resources. | + + + + + + +```sql +CREATE CONNECTION + STORAGE_TYPE = 'azblob' + ACCOUNT_NAME = '' + ACCOUNT_KEY = '' + ENDPOINT_URL = 'https://.blob.core.windows.net'; +``` + + + + +```sql +CREATE CONNECTION + STORAGE_TYPE = 'gcs' + CREDENTIAL = ''; +``` + + + + +```sql +CREATE CONNECTION + STORAGE_TYPE = 'oss' + ACCESS_KEY_ID = '' + ACCESS_KEY_SECRET = '' + ENDPOINT_URL = 'https://.[-internal].aliyuncs.com'; +``` + + + + +```sql +CREATE CONNECTION + STORAGE_TYPE = 'cos' + SECRET_ID = '' + SECRET_KEY = '' + ENDPOINT_URL = ''; +``` + + + + +```sql +CREATE CONNECTION + STORAGE_TYPE = 'hf' + REPO_TYPE = 'dataset' + REVISION = 'main' + TOKEN = ''; +``` + +Omit `TOKEN` for public repositories; include it for private or rate-limited assets. + + + + ## Access control requirements @@ -71,15 +146,9 @@ CREATE CONNECTION To create a connection, the user performing the operation or the [current_role](/guides/security/access-control/roles) must have the CREATE CONNECTION [privilege](/guides/security/access-control/privileges). -:::note +## Update Table Connections -The enable_experimental_connection_rbac_check settings governs connection-level access control. It is disabled by default. -Connection creation solely requires the user to possess superuser privileges, bypassing detailed RBAC checks. -When enabled, granular permission verification is enforced during connection establishment. - -This is an experimental feature and may be enabled by default in the future. - -::: +To switch an existing table to a new connection, use [`ALTER TABLE ... CONNECTION`](/sql/sql-commands/ddl/table/alter-table-connection). This command rebinds external tables to a different connection without recreating the table. ## Examples From d08dcc85e3bc970037aac9d7d9c230ac4924d3a7 Mon Sep 17 00:00:00 2001 From: BohuTANG Date: Wed, 5 Nov 2025 19:30:45 +0800 Subject: [PATCH 2/2] fix: correct aws doc link punctuation --- docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md b/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md index eadd1546c9..653856342b 100644 --- a/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md +++ b/docs/cn/sql-reference/00-sql-reference/51-connect-parameters.md @@ -56,7 +56,7 @@ CREATE STAGE my_minio_stage CONNECTION = (CONNECTION_NAME = 'my_minio_conn'); ``` -也可以改用 AWS IAM Role 和 External ID 认证，以更细粒度地控制可访问的 S3 Bucket，同时增加额外的安全校验。更多背景请参考 https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html。 +也可以改用 AWS IAM Role 和 External ID 认证，以更细粒度地控制可访问的 S3 Bucket，同时增加额外的安全校验。更多背景请参考 [AWS 文档](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html)。 下表列出了使用 AWS IAM 角色访问 Amazon S3 的连接参数：