diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index e61e73495..877f5218a 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -369,6 +369,10 @@ *** xref:manage:rpk/rpk-install.adoc[] *** xref:manage:rpk/broker-admin.adoc[] *** xref:manage:rpk/config-rpk-profile.adoc[] +** xref:manage:iceberg/index.adoc[Iceberg] +*** xref:manage:iceberg/about-iceberg-topics.adoc[] +*** xref:manage:iceberg/use-iceberg-catalogs.adoc[] +*** xref:manage:iceberg/query-iceberg-topics.adoc[] ** xref:manage:schema-reg/index.adoc[Schema Registry] *** xref:manage:schema-reg/schema-reg-overview.adoc[] *** xref:manage:schema-reg/schema-reg-ui.adoc[] diff --git a/modules/get-started/pages/whats-new-cloud.adoc b/modules/get-started/pages/whats-new-cloud.adoc index fbfa64fad..ebb8198f1 100644 --- a/modules/get-started/pages/whats-new-cloud.adoc +++ b/modules/get-started/pages/whats-new-cloud.adoc @@ -9,6 +9,12 @@ This page lists new features added in Redpanda Cloud. == April 2025 +=== Iceberg topics: beta + +The xref:manage:iceberg/about-iceberg-topics.adoc[Iceberg integration for Redpanda] allows you to store topic data in the cloud in the Iceberg open table format. This makes your streaming data immediately available in downstream analytical systems without setting up and maintaining additional ETL pipelines. You can also integrate your data directly into commonly-used big data processing frameworks, standardizing and simplifying the consumption of streams as tables in a wide variety of data analytics pipelines. + +Iceberg topics is supported in AWS and GCP. + === AI agents Redpanda Cloud is starting to introduce beta versions of xref:develop:agents/about.adoc[AI agents] for enterprise agentic applications driven by a continuous data feed. diff --git a/modules/manage/pages/iceberg/about-iceberg-topics.adoc b/modules/manage/pages/iceberg/about-iceberg-topics.adoc new file mode 100644 index 000000000..0be0f3e87 --- /dev/null +++ b/modules/manage/pages/iceberg/about-iceberg-topics.adoc @@ -0,0 +1,7 @@ += About Iceberg Topics +:description: Learn how Redpanda can integrate topics with Apache Iceberg. +:page-categories: Iceberg, Integration, Management, High Availability, Data Replication +:page-beta: true +:env-cloud: true + +include::25.1@ROOT:manage:partial$iceberg/about-iceberg-topics.adoc[] \ No newline at end of file diff --git a/modules/manage/pages/iceberg/index.adoc b/modules/manage/pages/iceberg/index.adoc new file mode 100644 index 000000000..a9f7d0f21 --- /dev/null +++ b/modules/manage/pages/iceberg/index.adoc @@ -0,0 +1,4 @@ += Integrate Redpanda with Iceberg +:description: Generate Iceberg tables for your Redpanda topics for data lakehouse access. +:page-layout: index +:page-beta: true \ No newline at end of file diff --git a/modules/manage/pages/iceberg/query-iceberg-topics.adoc b/modules/manage/pages/iceberg/query-iceberg-topics.adoc new file mode 100644 index 000000000..6fe888019 --- /dev/null +++ b/modules/manage/pages/iceberg/query-iceberg-topics.adoc @@ -0,0 +1,9 @@ += Query Iceberg Topics +:description: Query Redpanda topic data stored in Iceberg tables, based on the topic Iceberg mode and schema. +:page-categories: Iceberg, Integration, Management, High Availability, Data Replication +:page-beta: true +:env-cloud: true + +When you access Iceberg topics from a data lakehouse or other Iceberg-compatible tools, how you consume the data depends on the xref:manage:iceberg/about-iceberg-topics.adoc#enable-iceberg-integration[Iceberg mode] you've chosen for a topic, and whether you've registered a schema for the topic in the xref:manage:schema-reg/schema-reg-overview.adoc[Redpanda Schema Registry]. You do not need to rely on complex ETL jobs or pipelines to access real-time data from Redpanda. + +include::25.1@ROOT:manage:partial$iceberg/query-iceberg-topics.adoc[] \ No newline at end of file diff --git a/modules/manage/pages/iceberg/use-iceberg-catalogs.adoc b/modules/manage/pages/iceberg/use-iceberg-catalogs.adoc new file mode 100644 index 000000000..eda01dd72 --- /dev/null +++ b/modules/manage/pages/iceberg/use-iceberg-catalogs.adoc @@ -0,0 +1,11 @@ += Use Iceberg Catalogs +:description: Learn how to access Redpanda topic data stored in Iceberg tables, using table metadata or a catalog integration. +:page-categories: Iceberg, Management, High Availability, Data Replication, Integration +:page-beta: true +:env-cloud: true + +include::25.1@ROOT:manage:partial$iceberg/use-iceberg-catalogs.adoc[] + +== Next steps + +* xref:manage:iceberg/query-iceberg-topics.adoc[] \ No newline at end of file diff --git a/modules/manage/partials/controlplane-api.adoc b/modules/manage/partials/controlplane-api.adoc index 9b8df2907..148cd98c0 100644 --- a/modules/manage/partials/controlplane-api.adoc +++ b/modules/manage/partials/controlplane-api.adoc @@ -102,6 +102,7 @@ Create a network by making a request to xref:api:ROOT:cloud-controlplane-api.ado Choose a xref:networking:cidr-ranges.adoc[CIDR range] that does not overlap with your existing VPCs or your Redpanda network. +ifdef::env-dedicated[] [,bash] ---- curl -d \ @@ -114,6 +115,21 @@ curl -d \ "region": "us-west1" }' -H "Authorization: Bearer " -X POST https://api.redpanda.com/v1/networks ---- +endif::[] +ifdef::env-byoc[] +[,bash] +---- +curl -d \ +'{ + "cidr_block": "10.0.0.0/20", + "cloud_provider": "CLOUD_PROVIDER_GCP", + "cluster_type": "TYPE_BYOC", + "name": "", + "resource_group_id": "", + "region": "us-west1" +}' -H "Authorization: Bearer " -X POST https://api.redpanda.com/v1/networks +---- +endif::[] This endpoint returns a <>. @@ -121,6 +137,7 @@ This endpoint returns a <>. After the network is created, make a request to the xref:api:ROOT:cloud-controlplane-api.adoc#post-/v1/clusters[`POST /v1/clusters`] with the resource group ID and network ID in the request body. +ifdef::env-dedicated[] [,bash] ---- curl -d \ @@ -137,9 +154,41 @@ curl -d \ "us-west1-a", "us-west1-b", "us-west1-c" - ] - }' -H "Authorization: Bearer " -X POST https://api.redpanda.com/v1/clusters + ], + "cluster_configuration": { + "custom_properties": { + "audit_enabled":true + } + } +}' -H "Authorization: Bearer " -X POST https://api.redpanda.com/v1/clusters +---- +endif::[] +ifdef::env-byoc[] +[,bash] +---- +curl -d \ +'{ + "cloud_provider": "CLOUD_PROVIDER_GCP", + "connection_type": "CONNECTION_TYPE_PUBLIC", + "name": "my-new-cluster", + "resource_group_id": "", + "network_id": "", + "region": "us-west1", + "throughput_tier": "tier-1-gcp-um4g", + "type": "TYPE_BYOC", + "zones": [ + "us-west1-a", + "us-west1-b", + "us-west1-c" + ], + "cluster_configuration": { + "custom_properties": { + "audit_enabled":true + } + } +}' -H "Authorization: Bearer " -X POST https://api.redpanda.com/v1/clusters ---- +endif::[] The Create Cluster endpoint returns a <>. When the operation completes, you can retrieve cluster details by calling xref:api:ROOT:cloud-controlplane-api.adoc#get-/v1/clusters/-id-[`GET /v1/clusters/\{id}`], and passing the cluster ID as a parameter. @@ -264,6 +313,45 @@ The Create Serverless Cluster endpoint returns a <" \ + -H 'accept: application/json'\ + -H 'content-type: application/json' \ + -d '{ + "cluster_configuration": { + "custom_properties": { + "iceberg_enabled":true, + "iceberg_catalog_type":"rest" + } + } + }' -X PATCH "https://api.cloud.redpanda.com/v1/clusters/" +---- +endif::[] + +ifndef::env-byoc[] +[,bash] +---- +curl -H "Authorization: Bearer " \ + -H 'accept: application/json'\ + -H 'content-type: application/json' \ + -d '{ + "cluster_configuration": { + "custom_properties": { + "audit_enabled":true + } + } + }' -X PATCH "https://api.cloud.redpanda.com/v1/clusters/" +---- +endif::[] + +The Update Cluster endpoint returns a <>. <> to verify that the update is complete. + == Delete a cluster ifndef::env-serverless[] @@ -271,7 +359,7 @@ ifndef::env-serverless[] To delete a cluster, make a request to the xref:api:ROOT:cloud-controlplane-api.adoc#delete-/v1/clusters/-id-[`DELETE /v1/clusters/\{id}`] endpoint, passing the cluster ID as a parameter. This is a <>. ```bash -curl -H "Authorization: Bearer " -X DELETE https://api.redpanda.com/v1/clusters/ +curl -H "Authorization: Bearer " -X DELETE https://api.redpanda.com/v1/clusters/ ``` ifdef::env-byoc[] diff --git a/modules/shared/images/iceberg-integration-optimized.png b/modules/shared/images/iceberg-integration-optimized.png new file mode 100644 index 000000000..f557edd0c Binary files /dev/null and b/modules/shared/images/iceberg-integration-optimized.png differ