diff --git a/modules/manage/pages/disaster-recovery/shadowing/failover-runbook.adoc b/modules/manage/pages/disaster-recovery/shadowing/failover-runbook.adoc index a2314bd17..f5addfb9e 100644 --- a/modules/manage/pages/disaster-recovery/shadowing/failover-runbook.adoc +++ b/modules/manage/pages/disaster-recovery/shadowing/failover-runbook.adoc @@ -94,6 +94,13 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. From the *Shadow Link* page, select the shadow link you want to view. +. The *Overview* tab shows the state of the shadow link and its topics. +-- + rpk:: + -- @@ -159,6 +166,12 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +Capture the status from the *Shadow Link* page. +-- + rpk:: + -- @@ -251,6 +264,13 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. On your *Shadow Link* page, click *Failover All Topics*. +. Click to confirm the failover action. The failover process promotes all topics to writable status. +-- + rpk:: + -- @@ -287,6 +307,13 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. On your *Shadow Link* page, click the *Failover* button for the topics you want to failover. +. Click to confirm the failover action. The failover process promotes the selected topics to writable status. +-- + rpk:: + -- @@ -359,6 +386,13 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. From the *Shadow Link* page, select the shadow link you want to view. +. Click the *Tasks* tab to view all tasks and their status. +-- + rpk:: + -- @@ -427,6 +461,12 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +On your *Shadow Link* page, click *Delete*. +-- + rpk:: + -- @@ -480,6 +520,12 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +All failover actions in the Cloud UI include force delete functionality by default. When you failover a shadow link, all topics are immediately promoted to writable status. +-- + rpk:: + -- diff --git a/modules/manage/pages/disaster-recovery/shadowing/failover.adoc b/modules/manage/pages/disaster-recovery/shadowing/failover.adoc index 7e6369e81..4b87d8913 100644 --- a/modules/manage/pages/disaster-recovery/shadowing/failover.adoc +++ b/modules/manage/pages/disaster-recovery/shadowing/failover.adoc @@ -72,6 +72,14 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. On the *Shadow Link* page, select your shadow link. +. For any of the topics you want to failover, click the corresponding *Failover* button. +. Click to confirm the failover action. The failover process promotes the selected topics to writable status. +-- + rpk:: + -- @@ -114,6 +122,14 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. On the *Shadow Link* page, select your shadow link. +. Click *Failover All Topics*. +. Click to confirm the failover action. The failover process promotes all topics to writable status. +-- + rpk:: + -- @@ -156,6 +172,12 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +All failover actions in the Cloud UI include force delete functionality by default. When you failover a shadow link, all topics are immediately promoted to writable status. +-- + rpk:: + -- @@ -223,6 +245,12 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +Track the progress of failover operations from the *Shadow Link* page in the Cloud UI. +-- + rpk:: + -- diff --git a/modules/manage/pages/disaster-recovery/shadowing/monitor.adoc b/modules/manage/pages/disaster-recovery/shadowing/monitor.adoc index 83c206b8a..6805f297e 100644 --- a/modules/manage/pages/disaster-recovery/shadowing/monitor.adoc +++ b/modules/manage/pages/disaster-recovery/shadowing/monitor.adoc @@ -17,7 +17,7 @@ include::shared:partial$emergency-shadowing-callout.adoc[] == Status commands -To list existing shadow links, run: +To list existing shadow links: ifndef::env-cloud[] // tag::rpk-tab-list-links[] @@ -31,6 +31,12 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +At the organization level of the Cloud UI, navigate to *Shadow Link*. +-- + rpk:: + -- @@ -50,7 +56,7 @@ curl 'https://api.redpanda.com/v1/shadow-links' \ ====== endif::[] -To view shadow link configuration details, run: +To view shadow link configuration details: ifndef::env-cloud[] // tag::rpk-tab-view-link-details[] @@ -66,6 +72,13 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. From the *Shadow Link* page, select the shadow link you want to view. +. Click the *Tasks* tab to view all tasks and their status. +-- + rpk:: + -- @@ -85,7 +98,7 @@ curl 'https://api.redpanda.com/v1/shadow-links/' \ ====== endif::[] -To check your shadow link status and ensure proper operation, run: +To check your shadow link status and ensure proper operation: ifndef::env-cloud[] // tag::rpk-tab-shadow-status[] @@ -99,6 +112,13 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. From the *Shadow Link* page, select the shadow link you want to view. +. Click the *Tasks* tab to view all tasks and their status. +-- + rpk:: + -- @@ -127,7 +147,7 @@ curl "https://$DATAPLANE_API_URL/v1/shadowlinks//topic" \ -- ====== -The responses include the following: +The status includes the following: endif::[] * **Shadow link state**: Overall operational state (`ACTIVE`, `PAUSED`). @@ -197,6 +217,13 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. From the *Shadow Link* page, select the shadow link you want to view. +. Click the *Tasks* tab to view all tasks and their status. +-- + rpk:: + -- diff --git a/modules/manage/pages/disaster-recovery/shadowing/overview.adoc b/modules/manage/pages/disaster-recovery/shadowing/overview.adoc index d2fdf51fc..249367875 100644 --- a/modules/manage/pages/disaster-recovery/shadowing/overview.adoc +++ b/modules/manage/pages/disaster-recovery/shadowing/overview.adoc @@ -16,7 +16,7 @@ include::shared:partial$enterprise-license.adoc[] ==== endif::[] -Shadowing is Redpanda's enterprise-grade disaster recovery solution that establishes asynchronous, offset-preserving replication between two distinct Redpanda clusters. A cluster is able to create a dedicated client that continuously replicates source cluster data, including offsets, timestamps, and cluster metadata. This creates a read-only shadow cluster that you can quickly failover to handle production traffic during a disaster. +Shadowing is Redpanda's enterprise-grade disaster recovery solution that establishes asynchronous, offset-preserving replication between two distinct Redpanda clusters. A cluster is able to create a dedicated client that continuously replicates source cluster data, including offsets, timestamps, and cluster metadata. This creates a read-only shadow cluster that you can quickly failover to handle production traffic during a disaster. Shadowing keeps data flowing, even during regional outages. include::shared:partial$emergency-shadowing-callout.adoc[] @@ -61,7 +61,7 @@ Shadowing for disaster recovery currently has the following limitations: == Shadow link tasks -Shadow linking operates through specialized tasks that handle different aspects of replication. Each task corresponds to a configuration section in your shadow link setup and runs continuously to maintain synchronization with the source cluster. +Shadow linking operates through specialized tasks that handle different aspects of replication. If you use a `shadow-config.yaml` configuration file to create the shadow link, each task corresponds to a section in the file. Tasks run continuously to maintain synchronization with the source cluster. [tabs] ==== @@ -71,7 +71,7 @@ Source Topic Sync:: [#source-topic-sync-task] The **Source Topic Sync task** manages topic discovery and metadata synchronization. This task periodically queries the source cluster to discover available topics, applies your configured topic filters to determine which topics should become shadow topics, and synchronizes topic properties between clusters. -The task is controlled by the `topic_metadata_sync_options` configuration section, which includes: +The task is controlled by the `topic_metadata_sync_options` section in the configuration file. It includes: * **Auto-creation filters**: Determines which source topics automatically become shadow topics * **Property synchronization**: Controls which topic properties replicate from source to shadow @@ -87,7 +87,7 @@ Consumer Group Shadowing:: [#consumer-group-shadowing-task] The **Consumer Group Shadowing task** replicates consumer group offsets and membership information from the source cluster. This ensures that consumer applications can resume processing from the correct position after failover. -The task is controlled by the `consumer_offset_sync_options` configuration section, which includes: +The task is controlled by the `consumer_offset_sync_options` section in the configuration file. It includes: * **Group filters**: Determines which consumer groups have their offsets replicated * **Sync interval**: How frequently to synchronize consumer group offsets @@ -102,7 +102,7 @@ Security Migrator:: [#security-migrator-task] The **Security Migrator task** replicates security policies, primarily ACLs (access control lists), from the source cluster to maintain consistent authorization across both environments. -The task is controlled by the `security_sync_options` configuration section, which includes: +The task is controlled by the `security_sync_options` section in the configuration file. It includes: * **ACL filters**: Determines which security policies replicate * **Sync interval**: How frequently to synchronize security settings diff --git a/modules/manage/pages/disaster-recovery/shadowing/setup.adoc b/modules/manage/pages/disaster-recovery/shadowing/setup.adoc index 9006763fe..b854736b9 100644 --- a/modules/manage/pages/disaster-recovery/shadowing/setup.adoc +++ b/modules/manage/pages/disaster-recovery/shadowing/setup.adoc @@ -102,7 +102,7 @@ endif::[] == Set up Shadowing -To set up Shadowing, you need to create a shadow link and configure filters to select which topics, consumer groups, and ACLs to replicate. +To set up Shadowing, you need to create a shadow link and configure filters to select which topics, consumer groups, ACLs, and Schema Registry data to replicate. ifdef::env-cloud[] If using the Cloud API to set up Shadowing, you must link:/api/doc/cloud-controlplane/authentication[authenticate] to the API by including an access token in your requests. @@ -111,11 +111,11 @@ endif::[] === Create a shadow link ifndef::env-cloud[] -Any cluster can create a shadow link to a source cluster. When you create a shadow link with `rpk` or the API, you need to provide a YAML configuration that defines connection settings, filters, and synchronization options. +Any cluster can create a shadow link to a source cluster. endif::[] ifdef::env-cloud[] -Any BYOC or Dedicated cluster can create a shadow link to a source cluster. When you create a shadow link with `rpk` or the API, you need to provide a YAML configuration that defines connection settings, filters, and synchronization options. +Any BYOC or Dedicated cluster can create a shadow link to a source cluster. endif::[] [TIP] @@ -137,7 +137,13 @@ rpk shadow config generate --for-cloud -o shadow-config.yaml endif::[] ---- +ifndef::env-cloud[] This creates a complete YAML configuration file that you can customize for your environment. The template includes all available fields with comments explaining their purpose. For detailed command options, see xref:reference:rpk/rpk-shadow/rpk-shadow-config-generate.adoc[`rpk shadow config generate`]. +endif::[] + +ifdef::env-cloud[] +This creates a complete YAML configuration file that you can customize for your environment. The template includes all available fields with comments explaining their purpose. For detailed command options, see xref:reference:rpk/rpk-shadow/rpk-shadow-config-generate.adoc[`rpk shadow config generate --for-cloud`]. +endif::[] ==== .Explore the configuration file @@ -146,19 +152,47 @@ This creates a complete YAML configuration file that you can customize for your [,yaml] ---- # Sample ShadowLinkConfig YAML with all fields -name: # Unique name for this shadow link, example: "production-dr" + +name: # Unique name for this shadow link, example: "production-dr" + ifdef::env-cloud[] cloud_options: - source_redpanda_id: # Source cluster ID in Redpanda Cloud - shadow_redpanda_id: # Shadow cluster ID in Redpanda Cloud +# Use either source_redpanda_id or bootstrap_servers: only one is required. + source_redpanda_id: # Optional: 20 character lowercase ID of the cluster + # Example: m7xtv2qq5njbhwruk88f + shadow_redpanda_id: # 20 character lowercase ID of the cluster + # Example: m7xtv2qq5njbhwruk88f endif::[] client_options: - bootstrap_servers: # Source cluster brokers to connect to - - : # Example: "prod-kafka-1.example.com:9092" - - : # Example: "prod-kafka-2.example.com:9092" - - : # Example: "prod-kafka-3.example.com:9092" - source_cluster_id: # Optional: Expected source cluster ID for validation, example: "prod-cluster-123" - + bootstrap_servers: # Source cluster brokers to connect to + - : # Example: "prod-kafka-1.example.com:9092" + - : # Example: "prod-kafka-2.example.com:9092" + - : # Example: "prod-kafka-3.example.com:9092" + source_cluster_id: # Optional: UUID assigned by Redpanda + # Example: a882bc98-7aca-40f6-a657-36a0b4daf1fd +ifndef::env-cloud[] +# To get source_cluster_id, run `rpk cluster config get cluster_id`. +endif::[] +ifdef::env-cloud[] +# This UUID is not available in Redpanda Cloud. +endif::[] + +ifdef::env-cloud[] + # TLS settings using PEM strings + tls_settings: + enabled: true + tls_pem_settings: + ca: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- + key: ${secrets.KEY_FROM_SHADOW_CLUSTER_SECRET_STORE} + cert: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +endif::[] +ifndef::env-cloud[] # TLS settings using file paths tls_settings: enabled: true # Enable TLS @@ -167,10 +201,11 @@ client_options: key_path: # Optional: Path to client private key, example: "/etc/ssl/private/client.key" cert_path: # Optional: Path to client certificate, example: "/etc/ssl/certs/client.crt" do_not_set_sni_hostname: false # Optional: Skip SNI hostname when using TLS (default: false) - - # Create SASL credentials in the source cluster. - # Then, with this configuration, ensure the shadow cluster uses the credentials - # to authenticate to the source cluster. +endif::[] + +# Create SASL credentials in the source cluster. +# Then, with this configuration, ensure the shadow cluster uses the credentials +# to authenticate to the source cluster. authentication_configuration: # SASL/SCRAM authentication scram_configuration: @@ -269,18 +304,53 @@ endif::[] // Use tabs and display the tagged content above inside the rpk tab ifdef::env-cloud[] -. In the shadow cluster, create a secret to store the authentication credential that the cluster will use (`"scram_configuration": "password"` in the example configuration in the next step). Your secret must be scoped to "Redpanda Cluster". +Because the shadow cluster pulls from the source cluster, the shadow cluster requires credentials to connect to the source cluster. And because you cannot store plaintext passwords in Redpanda Cloud, you must create a secret to hold the password for the user on the source cluster. If using mTLS, you must also create a secret to hold the key of the client certificate for the client to authenticate. Reference that secret in `client_options.tls_settings.key_file` in the configuration file. + +. In the shadow cluster, create the secret: ++ +[tabs] +====== +Cloud UI:: ++ +-- +In the shadow cluster, go to the *Secrets Store* page and create a secret for the source cluster user, scoped to Redpanda Cluster. + +If necessary, first create the user with all ACLs enabled in the source cluster. +-- + +rpk:: + -Use one of the following to create a secret: +-- +In the shadow cluster, create a secret to store the authentication credential that the cluster will use (`"scram_configuration": "password"` in the example configuration in the next step). Your secret must be scoped to "Redpanda Cluster". + +Use xref:reference:rpk/rpk-security/rpk-security-secret-create.adoc[`rpk security secret create`] to create the secret from the command line. +-- + +Data Plane API:: + -- In the Redpanda Cloud UI, the *Secrets Store* page of your cluster -- xref:reference:rpk/rpk-security/rpk-security-secret-create.adoc[`rpk security secret create`] -- The xref:manage:api/cloud-dataplane-api.adoc[Data Plane API] +-- +In the shadow cluster, create a secret to store the authentication credential that the cluster will use (`"scram_configuration": "password"` in the example configuration in the next step). Your secret must be scoped to "Redpanda Cluster". + +Use the xref:manage:api/cloud-dataplane-api.adoc[Data Plane API] to programmatically create the secret. +-- +====== . In the shadow cluster, create a shadow link to the source cluster. + [tabs] ====== +Cloud UI:: ++ +-- +. At the organization level of the Cloud UI, navigate to *Shadow Link*. +. Click *Create shadow link*. +. Enter a unique name for the shadow link. The name must start and end with lowercase alphanumeric characters, hyphens allowed. +. Select the source cluster from which data will be replicated. You can select an existing Redpanda Cloud cluster, or you can enter a bootstrap server URL to connect to any Kafka-compatible cluster. For an existing Redpanda Cloud cluster, you select the specific cluster on the next page. +. Enter the authorization and authentication details from the source cluster, including the user and the name of the secret containing the password created in the previous step. +. Optionally, expand *Advanced options* to configure client connection properties. +. Click *Save* to apply changes. +-- + rpk:: + -- @@ -599,6 +669,7 @@ The shadow cluster uses these addresses to discover all brokers in the source cl For production deployments, secure the network connection between clusters: +ifndef::env-cloud[] .TLS encryption: [,yaml] ---- @@ -611,6 +682,28 @@ client_options: cert_path: # Optional: Path to client certificate, example: "/etc/ssl/certs/client.crt" do_not_set_sni_hostname: false # Optional: Skip SNI hostname when using TLS (default: false) ---- +endif::[] + +ifdef::env-cloud[] +.TLS encryption: +[,yaml] +---- +client_options: + tls_settings: + enabled: true # Enable TLS + tls_pem_settings: + ca: |- # CA certificate in PEM format + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- + key: ${secrets.KEY_FROM_SHADOW_CLUSTER_SECRET_STORE} # Client private key (can use secrets reference) + cert: |- # Optional: Client certificate in PEM format for mutual TLS + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- + do_not_set_sni_hostname: false # Optional: Skip SNI hostname when using TLS (default: false) +---- +endif::[] .Authentication: [,yaml] @@ -634,7 +727,7 @@ ifndef::env-cloud[] # SASL/PLAIN authentication plain_configuration: username: # SASL/PLAIN username, example: "shadow-replication-user" - password: # SASL/PLAIN password, example: "secure-password-123" + password: # SASL/PLAIN password, example: ${secrets.SASL_SECRET} endif::[] ---- @@ -678,6 +771,15 @@ endif::[] ifdef::env-cloud[] [tabs] ====== +Cloud UI:: ++ +-- +. At the organization level of the Cloud UI, navigate to *Shadow Link*. +. Select the shadow link you want to modify, and click *Edit*. +. Edit the shadow link settings or the shadowing behavior by specifying which content from the source cluster to shadow (topics, ACLs, consumer groups, Schema Registry). You can also enable additional topic properties to be shadowed or disable optional topic properties from being included in the shadowing. +. Click *Save* to apply changes. +-- + rpk:: + -- @@ -706,5 +808,5 @@ For the full API reference, see link:/api/doc/cloud-controlplane/v1/operation/op ====== endif::[] // end::single-source[] - + See also: link:/api/doc/admin/v2/[Admin API v2 reference^] \ No newline at end of file