Merge pull request #3797 from ClickHouse/kp/clickpipes-azure-blob-storage

Blargian · web-flow · commit 6bc4eabdfb69 · 2025-05-19T12:23:51.000+02:00
Add note of azure blob storage clickpipe private preview
diff --git a/docs/integrations/data-ingestion/clickpipes/index.md b/docs/integrations/data-ingestion/clickpipes/index.md
@@ -14,6 +14,7 @@ import S3svg from '@site/static/images/integrations/logos/amazon_s3_logo.svg';
 import Amazonkinesis from '@site/static/images/integrations/logos/amazon_kinesis_logo.svg';
 import Gcssvg from '@site/static/images/integrations/logos/gcs.svg';
 import DOsvg from '@site/static/images/integrations/logos/digitalocean.svg';
+import ABSsvg from '@site/static/images/integrations/logos/azureblobstorage.svg';
 import Postgressvg from '@site/static/images/integrations/logos/postgresql.svg';
 import Mysqlsvg from '@site/static/images/integrations/logos/mysql.svg';
 import redpanda_logo from '@site/static/images/integrations/logos/logo_redpanda.png';
@@ -42,7 +43,7 @@ import Image from '@theme/IdealImage';
 | Amazon S3            | <S3svg class="image" alt="Amazon S3 logo" style={{width: '3rem', height: 'auto'}}/>              |Object Storage| Stable           | Configure ClickPipes to ingest large volumes of data from object storage.                            |
 | Google Cloud Storage | <Gcssvg class="image" alt="Google Cloud Storage logo" style={{width: '3rem', height: 'auto'}}/>  |Object Storage| Stable           | Configure ClickPipes to ingest large volumes of data from object storage.                            |
 | DigitalOcean Spaces | <DOsvg class="image" alt="Digital Ocean logo" style={{width: '3rem', height: 'auto'}}/> | Object Storage | Stable | Configure ClickPipes to ingest large volumes of data from object storage.
-
+| Azure Blob Storage | <ABSsvg class="image" alt="Azure Blob Storage logo" style={{width: '3rem', height: 'auto'}}/> | Object Storage | Private Beta | Configure ClickPipes to ingest large volumes of data from object storage.
 | Amazon Kinesis       | <Amazonkinesis class="image" alt="Amazon Kenesis logo" style={{width: '3rem', height: 'auto'}}/> |Streaming| Stable           | Configure ClickPipes and start ingesting streaming data from Amazon Kinesis into ClickHouse cloud.   |
 | Postgres             | <Postgressvg class="image" alt="Postgres logo" style={{width: '3rem', height: 'auto'}}/>         |DBMS| Public Beta      | Configure ClickPipes and start ingesting data from Postgres into ClickHouse Cloud.                   |
 | MySQL                | <Mysqlsvg class="image" alt="MySQL logo" style={{width: '3rem', height: 'auto'}}/>               |DBMS| Private Beta | Configure ClickPipes and start ingesting data from MySQL into ClickHouse Cloud.                      |
diff --git a/docs/integrations/data-ingestion/clickpipes/object-storage.md b/docs/integrations/data-ingestion/clickpipes/object-storage.md
@@ -8,6 +8,7 @@ title: 'Integrating Object Storage with ClickHouse Cloud'
 import S3svg from '@site/static/images/integrations/logos/amazon_s3_logo.svg';
 import Gcssvg from '@site/static/images/integrations/logos/gcs.svg';
 import DOsvg from '@site/static/images/integrations/logos/digitalocean.svg';
+import ABSsvg from '@site/static/images/integrations/logos/azureblobstorage.svg';
 import cp_step0 from '@site/static/images/integrations/data-ingestion/clickpipes/cp_step0.png';
 import cp_step1 from '@site/static/images/integrations/data-ingestion/clickpipes/cp_step1.png';
 import cp_step2_object_storage from '@site/static/images/integrations/data-ingestion/clickpipes/cp_step2_object_storage.png';
@@ -23,7 +24,7 @@ import cp_overview from '@site/static/images/integrations/data-ingestion/clickpi
 import Image from '@theme/IdealImage';
 
 # Integrating Object Storage with ClickHouse Cloud
-Object Storage ClickPipes provide a simple and resilient way to ingest data from Amazon S3, Google Cloud Storage, and DigitalOcean Spaces into ClickHouse Cloud. Both one-time and continuous ingestion are supported with exactly-once semantics.
+Object Storage ClickPipes provide a simple and resilient way to ingest data from Amazon S3, Google Cloud Storage, Azure Blob Storage, and DigitalOcean Spaces into ClickHouse Cloud. Both one-time and continuous ingestion are supported with exactly-once semantics.
 
 
 ## Prerequisite {#prerequisite}
@@ -95,6 +96,7 @@ Image
 | Amazon S3            |<S3svg class="image" alt="Amazon S3 logo" style={{width: '3rem', height: 'auto'}}/>|Object Storage| Stable          | Configure ClickPipes to ingest large volumes of data from object storage.                            |
 | Google Cloud Storage |<Gcssvg class="image" alt="Google Cloud Storage logo" style={{width: '3rem', height: 'auto'}}/>|Object Storage| Stable          | Configure ClickPipes to ingest large volumes of data from object storage.                            |
 | DigitalOcean Spaces | <DOsvg class="image" alt="Digital Ocean logo" style={{width: '3rem', height: 'auto'}}/> | Object Storage | Stable | Configure ClickPipes to ingest large volumes of data from object storage.
+| Azure Blob Storage | <ABSsvg class="image" alt="Azure Blob Storage logo" style={{width: '3rem', height: 'auto'}}/> | Object Storage | Private Beta | Configure ClickPipes to ingest large volumes of data from object storage.
 
 More connectors will get added to ClickPipes, you can find out more by [contacting us](https://clickhouse.com/company/contact?loc=clickpipes).
 
@@ -126,13 +128,13 @@ To increase the throughput on large ingest jobs, we recommend scaling the ClickH
 - There are limitations on the types of views that are supported. Please read the section on [exactly-once semantics](#exactly-once-semantics) and [view support](#view-support) for more information.
 - Role authentication is not available for S3 ClickPipes for ClickHouse Cloud instances deployed into GCP or Azure. It is only supported for AWS ClickHouse Cloud instances.
 - ClickPipes will only attempt to ingest objects at 10GB or smaller in size. If a file is greater than 10GB an error will be appended to the ClickPipes dedicated error table.
-- S3 / GCS ClickPipes **does not** share a listing syntax with the [S3 Table Function](/sql-reference/table-functions/s3).
+- S3 / GCS ClickPipes **does not** share a listing syntax with the [S3 Table Function](/sql-reference/table-functions/s3), nor Azure with the [AzureBlobStorage Table function](/sql-reference/table-functions/azureBlobStorage).
   - `?` — Substitutes any single character
   - `*` — Substitutes any number of any characters except / including empty string
   - `**` — Substitutes any number of any character include / including empty string
 
 :::note
-This is a valid path:
+This is a valid path (for S3):
 
 https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz
 
@@ -143,7 +145,7 @@ https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/{documents-01,doc
 :::
 
 ## Continuous Ingest {#continuous-ingest}
-ClickPipes supports continuous ingestion from S3, GCS, and DigitalOcean Spaces. When enabled, ClickPipes will continuously ingest data from the specified path, it will poll for new files at a rate of once every 30 seconds. However, new files must be lexically greater than the last ingested file, meaning they must be named in a way that defines the ingestion order. For instance, files named `file1`, `file2`, `file3`, etc., will be ingested sequentially. If a new file is added with a name like `file0`, ClickPipes will not ingest it because it is not lexically greater than the last ingested file.
+ClickPipes supports continuous ingestion from S3, GCS, Azure Blob Storage, and DigitalOcean Spaces. When enabled, ClickPipes continuously ingests data from the specified path, and polls for new files at a rate of once every 30 seconds. However, new files must be lexically greater than the last ingested file. This means that they must be named in a way that defines the ingestion order. For instance, files named `file1`, `file2`, `file3`, etc., will be ingested sequentially. If a new file is added with a name like `file0`, ClickPipes will not ingest it because it is not lexically greater than the last ingested file.
 
 ## Archive table {#archive-table}
 ClickPipes will create a table next to your destination table with the postfix `s3_clickpipe_<clickpipe_id>_archive`. This table will contain a list of all the files that have been ingested by the ClickPipe. This table is used to track files during ingestion and can be used to verify files have been ingested. The archive table has a [TTL](/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl) of 7 days.
@@ -167,6 +169,8 @@ The Service Account permissions attached to the HMAC credentials should be `stor
 ### DigitalOcean Spaces {#dospaces}
 Currently only protected buckets are supported for DigitalOcean spaces. You require an "Access Key" and a "Secret Key" to access the bucket and its files. You can read [this guide](https://docs.digitalocean.com/products/spaces/how-to/manage-access/) on how to create access keys.
 
+### Azure Blob Storage {#azureblobstorage}
+Currently only protected buckets are supported for Azure Blob Storage. Authentication is done via a connection string, which supports access keys and shared keys. For more information, read [this guide](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string).
 
 ## F.A.Q. {#faq}
 
diff --git a/static/images/integrations/logos/azureblobstorage.svg b/static/images/integrations/logos/azureblobstorage.svg
@@ -0,0 +1,15 @@
+<svg width="62" height="50" viewBox="0 0 62 50" fill="none"
+  xmlns="http://www.w3.org/2000/svg">
+  <path d="M0.777344 13.5801H61.2218V47.2796C61.2218 47.8153 61.009 48.3289 60.6303 48.7077C60.2515 49.0864 59.7378 49.2992 59.2022 49.2992H2.7969C2.26128 49.2992 1.7476 49.0864 1.36886 48.7077C0.990118 48.3289 0.777344 47.8153 0.777344 47.2796V13.5801Z" fill="#32CEEE" />
+  <path d="M2.80757 0.7012H59.1916C59.4568 0.7012 59.7194 0.753438 59.9644 0.85493C60.2094 0.956422 60.4321 1.10518 60.6196 1.29271C60.8071 1.48025 60.9559 1.70288 61.0574 1.94791C61.1589 2.19293 61.2111 2.45554 61.2111 2.72076V13.5794H0.777344V2.72076C0.77734 2.45464 0.829929 2.19116 0.932087 1.94543C1.03425 1.69971 1.18396 1.4766 1.37262 1.28893C1.56129 1.10126 1.78518 0.95272 2.03144 0.851858C2.2777 0.750995 2.54146 0.699795 2.80757 0.7012Z" fill="#0078D4" />
+  <path d="M28.6036 18.0625H8.28357C7.72784 18.0625 7.27734 18.513 7.27734 19.0687V29.1736C7.27734 29.7293 7.72784 30.1798 8.28357 30.1798H28.6036C29.1593 30.1798 29.6098 29.7293 29.6098 29.1736V19.0687C29.6098 18.513 29.1593 18.0625 28.6036 18.0625Z" fill="#0078D4" />
+  <path d="M53.5215 18.0625H33.2015C32.6458 18.0625 32.1953 18.513 32.1953 19.0687V29.1736C32.1953 29.7293 32.6458 30.1798 33.2015 30.1798H53.5215C54.0773 30.1798 54.5278 29.7293 54.5278 29.1736V19.0687C54.5278 18.513 54.0773 18.0625 53.5215 18.0625Z" fill="white" />
+  <path d="M28.4903 32.5664H8.17028C7.61456 32.5664 7.16406 33.0169 7.16406 33.5726V43.6775C7.16406 44.2332 7.61456 44.6837 8.17028 44.6837H28.4903C29.046 44.6837 29.4965 44.2332 29.4965 43.6775V33.5726C29.4965 33.0169 29.046 32.5664 28.4903 32.5664Z" fill="#0078D4" />
+  <path d="M53.4083 32.5664H33.0883C32.5325 32.5664 32.082 33.0169 32.082 33.5726V43.6775C32.082 44.2332 32.5325 44.6837 33.0883 44.6837H53.4083C53.964 44.6837 54.4145 44.2332 54.4145 43.6775V33.5726C54.4145 33.0169 53.964 32.5664 53.4083 32.5664Z" fill="#0078D4" />
+  <defs>
+    <linearGradient id="paint0_linear_11640_11692" x1="30.9996" y1="49.2992" x2="30.9996" y2="13.5801" gradientUnits="userSpaceOnUse">
+      <stop stopColor="#32BEDD" />
+      <stop offset="0.775" stopColor="#32D4F5" />
+    </linearGradient>
+  </defs>
+</svg>
diff --git a/styles/ClickHouse/Headings.yml b/styles/ClickHouse/Headings.yml
@@ -33,3 +33,4 @@ exceptions:
   - ReplacingMergeTree
   - AggregatingMergeTree
   - DigitalOcean Spaces
+  - Azure Blob Storage
