Merge pull request #75768 from xenolinux/hcp-ha-dr-content-add

stevsmit · web-flow · commit 3851c07dc471 · 2024-05-21T11:23:51.000-04:00
OSDOCS#10475: Restructure backup, restore, disaster recovery for HCP
diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml
@@ -2375,8 +2375,17 @@ Topics:
   File: hcp-updating
 - Name: Hosted control planes Observability
   File: hcp-observability
-- Name: Backup, restore, and disaster recovery for hosted control planes
-  File: hcp-backup-restore-dr
+- Name: High availability for hosted control planes
+  Dir: hcp_high_availability
+  Topics:
+  - Name: Recovering a failing etcd cluster
+    File: hcp-recovering-etcd-cluster
+  - Name: Backing up and restoring etcd in an on-premise environment
+    File: hcp-backup-restore-on-premise
+  - Name: Backing up and restoring etcd on AWS
+    File: hcp-backup-restore-aws
+  - Name: Disaster recovery for a hosted cluster in AWS
+    File: hcp-disaster-recovery-aws
 - Name: Troubleshooting hosted control planes
   File: hcp-troubleshooting
 ---
diff --git a/backup_and_restore/control_plane_backup_and_restore/backing-up-etcd.adoc b/backup_and_restore/control_plane_backup_and_restore/backing-up-etcd.adoc
@@ -25,7 +25,7 @@ include::modules/backup-etcd.adoc[leveloffset=+1]
 [role="_additional-resources"]
 [id="additional-resources_backup-etcd"]
 == Additional resources
-* xref:../../hosted_control_planes/hcp-backup-restore-dr.adoc#hcp-backup-restore[Backing up and restoring etcd on a hosted cluster]
+* xref:../../hosted_control_planes/hcp_high_availability/hcp-recovering-etcd-cluster.adoc#hcp-recovering-etcd-cluster[Recovering an unhealthy etcd cluster]
 
 // Creating automated etcd backups
 include::modules/etcd-creating-automated-backups.adoc[leveloffset=+1]
diff --git a/hosted_control_planes/hcp_high_availability/_attributes b/hosted_control_planes/hcp_high_availability/_attributes
@@ -0,0 +1 @@
+../../_attributes/
diff --git a/hosted_control_planes/hcp_high_availability/hcp-backup-restore-aws.adoc b/hosted_control_planes/hcp_high_availability/hcp-backup-restore-aws.adoc
@@ -0,0 +1,16 @@
+:_mod-docs-content-type: ASSEMBLY
+[id="hcp-backup-restore-aws"]
+= Backing up and restoring etcd on AWS
+include::_attributes/common-attributes.adoc[]
+:context: hcp-backup-restore-aws
+
+toc::[]
+
+You can back up and restore etcd on a hosted cluster on {aws-first} to fix failures.
+
+:FeatureName: Hosted control planes on the {aws-short} platform
+include::snippets/technology-preview.adoc[]
+
+include::modules/backup-etcd-hosted-cluster.adoc[leveloffset=+1]
+
+include::modules/restoring-etcd-snapshot-hosted-cluster.adoc[leveloffset=+1]
diff --git a/hosted_control_planes/hcp_high_availability/hcp-backup-restore-on-premise.adoc b/hosted_control_planes/hcp_high_availability/hcp-backup-restore-on-premise.adoc
@@ -0,0 +1,11 @@
+:_mod-docs-content-type: ASSEMBLY
+[id="hcp-backup-restore-on-premise"]
+= Backing up and restoring etcd in an on-premise environment
+include::_attributes/common-attributes.adoc[]
+:context: hcp-backup-restore-on-premise
+
+toc::[]
+
+You can back up and restore etcd on a hosted cluster in an on-premise environment to fix failures.
+
+include::modules/hosted-cluster-etcd-backup-restore-on-premise.adoc[leveloffset=+1]
diff --git a/hosted_control_planes/hcp_high_availability/hcp-disaster-recovery-aws.adoc b/hosted_control_planes/hcp_high_availability/hcp-disaster-recovery-aws.adoc
@@ -0,0 +1,43 @@
+:_mod-docs-content-type: ASSEMBLY
+[id="hcp-disaster-recovery-aws"]
+= Disaster recovery for a hosted cluster in AWS 
+include::_attributes/common-attributes.adoc[]
+:context: hcp-disaster-recovery-aws
+
+toc::[]
+
+You can recover a hosted cluster to the same region within {aws-first}. For example, you need disaster recovery when the upgrade of a management cluster fails and the hosted cluster is in a read-only state.
+
+:FeatureName: Hosted control planes
+include::snippets/technology-preview.adoc[]
+
+The disaster recovery process involves the following steps:
+
+. Backing up the hosted cluster on the source management cluster
+. Restoring the hosted cluster on a destination management cluster
+. Deleting the hosted cluster from the source management cluster
+
+Your workloads remain running during the process. The Cluster API might be unavailable for a period, but that does not affect the services that are running on the worker nodes.
+
+[IMPORTANT]
+====
+Both the source management cluster and the destination management cluster must have the `--external-dns` flags to maintain the API server URL. That way, the server URL ends with `https://api-sample-hosted.sample-hosted.aws.openshift.com`. See the following example:
+
+.Example: External DNS flags
+[source,terminal]
+----
+--external-dns-provider=aws \
+--external-dns-credentials=<path_to_aws_credentials_file> \
+--external-dns-domain-filter=<basedomain>
+----
+
+If you do not include the `--external-dns` flags to maintain the API server URL, you cannot migrate the hosted cluster.
+====
+
+include::modules/dr-backup-restore-aws-overview.adoc[leveloffset=+1]
+
+include::modules/dr-hosted-cluster-within-aws-region-backup.adoc[leveloffset=+1]
+
+include::modules/dr-hosted-cluster-within-aws-region-restore.adoc[leveloffset=+1]
+
+include::modules/dr-hosted-cluster-within-aws-region-delete.adoc[leveloffset=+1]
diff --git a/hosted_control_planes/hcp_high_availability/hcp-recovering-etcd-cluster.adoc b/hosted_control_planes/hcp_high_availability/hcp-recovering-etcd-cluster.adoc
@@ -0,0 +1,13 @@
+:_mod-docs-content-type: ASSEMBLY
+[id="hcp-recovering-etcd-cluster"]
+= Recovering an unhealthy etcd cluster
+include::_attributes/common-attributes.adoc[]
+:context: hcp-recovering-etcd-cluster
+
+toc::[]
+
+In a highly available control plane, three etcd pods run as a part of a stateful set in an etcd cluster. To recover an etcd cluster, identify unhealthy etcd pods by checking the etcd cluster health.
+
+include::modules/hosted-cluster-etcd-status.adoc[leveloffset=+1]
+
+include::modules/hcp-recover-failing-etcd-pods.adoc[leveloffset=+1]
diff --git a/hosted_control_planes/hcp_high_availability/images b/hosted_control_planes/hcp_high_availability/images
@@ -0,0 +1 @@
+../../images
diff --git a/hosted_control_planes/hcp_high_availability/modules b/hosted_control_planes/hcp_high_availability/modules
@@ -0,0 +1 @@
+../../modules
diff --git a/hosted_control_planes/hcp_high_availability/snippets b/hosted_control_planes/hcp_high_availability/snippets
@@ -0,0 +1 @@
+../../snippets
diff --git a/modules/backup-etcd-hosted-cluster.adoc b/modules/backup-etcd-hosted-cluster.adoc
@@ -1,12 +1,12 @@
 // Module included in the following assembly:
 //
-// * hcp-backup-restore-dr.adoc
+// * hosted_control_planes/hcp_high_availability/hcp-backup-restore-aws.adoc
 
 :_mod-docs-content-type: PROCEDURE
 [id="backup-etcd-hosted-cluster_{context}"]
-= Taking a snapshot of etcd on a hosted cluster
+= Taking a snapshot of etcd for a hosted cluster
 
-As part of the process to back up etcd for a hosted cluster, you take a snapshot of etcd. After you take the snapshot, you can restore it, for example, as part of a disaster recovery operation.
+To back up etcd for a hosted cluster, you must take a snapshot of etcd. Later, you can restore etcd by using the snapshot.
 
 [IMPORTANT]
 ====
@@ -15,21 +15,21 @@ This procedure requires API downtime.
 
 .Procedure
 
-. Pause reconciliation of the hosted cluster by entering this command:
+. Pause reconciliation of the hosted cluster by entering the following command:
 +
 [source,terminal]
 ----
 $ oc patch -n clusters hostedclusters/<hosted_cluster_name> -p '{"spec":{"pausedUntil":"true"}}' --type=merge
 ----
 
-. Stop all etcd-writer deployments by entering this command:
+. Stop all etcd-writer deployments by entering the following command:
 +
 [source,terminal]
 ----
 $ oc scale deployment -n <hosted_cluster_namespace> --replicas=0 kube-apiserver openshift-apiserver openshift-oauth-apiserver
 ----
 
-. To take an etcd snapshot, use the `exec` command in each etcd container by running the following command:
+. To take an etcd snapshot, use the `exec` command in each etcd container by entering the following command:
 +
 [source,terminal]
 ----
@@ -43,11 +43,11 @@ $ oc exec -it <etcd_pod_name> -n <hosted_cluster_namespace> -- env ETCDCTL_API=3
 $ oc exec -it <etcd_pod_name> -n <hosted_cluster_namespace> -- env ETCDCTL_API=3 /usr/bin/etcdctl -w table snapshot status /var/lib/data/snapshot.db
 ----
 
-. Copy the snapshot data to a location where you can retrieve it later, such as an S3 bucket, as shown in the following example.
+. Copy the snapshot data to a location where you can retrieve it later, such as an S3 bucket. See the following example.
 +
 [NOTE]
 ====
-The following example uses signature version 2. If you are in a region that supports signature version 4, such as the us-east-2 region, use signature version 4. Otherwise, if you use signature version 2 to copy the snapshot to an S3 bucket, the upload fails and signature version 2 is deprecated.
+The following example uses signature version 2. If you are in a region that supports signature version 4, such as the `us-east-2` region, use signature version 4. Otherwise, when copying the snapshot to an S3 bucket, the upload fails.
 ====
 +
 .Example
@@ -70,17 +70,24 @@ oc exec -it etcd-0 -n ${HOSTED_CLUSTER_NAMESPACE} -- curl -X PUT -T "/var/lib/da
   https://${BUCKET_NAME}.s3.amazonaws.com/${CLUSTER_NAME}-snapshot.db
 ----
 
-. If you want to be able to restore the snapshot on a new cluster later, save the encryption secret that the hosted cluster references, as shown in this example:
+. To restore the snapshot on a new cluster later, save the encryption secret that the hosted cluster references.
+
+.. Get the secret encryption key by entering the following command:
 +
-.Example
 [source,terminal]
 ----
-oc get hostedcluster $CLUSTER_NAME -o=jsonpath='{.spec.secretEncryption.aescbc}'
-{"activeKey":{"name":"CLUSTER_NAME-etcd-encryption-key"}}
+$ oc get hostedcluster <hosted_cluster_name> -o=jsonpath='{.spec.secretEncryption.aescbc}'
+{"activeKey":{"name":"<hosted_cluster_name>-etcd-encryption-key"}}
+----
 
-# Save this secret, or the key it contains so the etcd data can later be decrypted
-oc get secret ${CLUSTER_NAME}-etcd-encryption-key -o=jsonpath='{.data.key}'
+.. Save the secret encryption key by entering the following command:
++
+[source,terminal]
+----
+$ oc get secret <hosted_cluster_name>-etcd-encryption-key -o=jsonpath='{.data.key}'
 ----
++
+You can decrypt this key when restoring a snapshot on a new cluster.
 
 .Next steps
 
diff --git a/modules/dr-backup-restore-aws-overview.adoc b/modules/dr-backup-restore-aws-overview.adoc
@@ -0,0 +1,29 @@
+// Module included in the following assemblies:
+//
+// * hosted_control_planes/hcp_high_availability/hcp-backup-restore-aws.adoc
+
+:_mod-docs-content-type: CONCEPT
+[id=" hcp-backup-restore-aws-overview_{context}"]
+= Overview of the backup and restore process
+
+The backup and restore process works as follows:
+
+. On management cluster 1, which you can think of as the source management cluster, the control plane and workers interact by using the external DNS API. The external DNS API is accessible, and a load balancer sits between the management clusters.
++
+image::298_OpenShift_Backup_Restore_0123_00.png[Diagram that shows the workers accessing the external DNS API and the external DNS API pointing to the control plane through a load balancer]
+
+. You take a snapshot of the hosted cluster, which includes etcd, the control plane, and the worker nodes. During this process, the worker nodes continue to try to access the external DNS API even if it is not accessible, the workloads are running, the control plane is saved in a local manifest file, and etcd is backed up to an S3 bucket. The data plane is active and the control plane is paused.
++
+image::298_OpenShift_Backup_Restore_0123_01.png[]
+
+. On management cluster 2, which you can think of as the destination management cluster, you restore etcd from the S3 bucket and restore the control plane from the local manifest file. During this process, the external DNS API is stopped, the hosted cluster API becomes inaccessible, and any workers that use the API are unable to update their manifest files, but the workloads are still running.
++
+image::298_OpenShift_Backup_Restore_0123_02.png[]
+
+. The external DNS API is accessible again, and the worker nodes use it to move to management cluster 2. The external DNS API can access the load balancer that points to the control plane.
++
+image::298_OpenShift_Backup_Restore_0123_03.png[]
+
+. On management cluster 2, the control plane and worker nodes interact by using the external DNS API. The resources are deleted from management cluster 1, except for the S3 backup of etcd. If you try to set up the hosted cluster again on mangagement cluster 1, it will not work.
++
+image::298_OpenShift_Backup_Restore_0123_04.png[]
diff --git a/modules/dr-hosted-cluster-within-aws-region-backup.adoc b/modules/dr-hosted-cluster-within-aws-region-backup.adoc
@@ -1,6 +1,6 @@
 // Module included in the following assembly:
 //
-// * hcp-backup-restore-dr.adoc
+// * hosted_control_planes/hcp_high_availability/hcp-backup-restore-aws.adoc
 
 :_mod-docs-content-type: PROCEDURE
 [id="dr-hosted-cluster-within-aws-region-backup_{context}"]
diff --git a/modules/dr-hosted-cluster-within-aws-region-delete.adoc b/modules/dr-hosted-cluster-within-aws-region-delete.adoc
@@ -1,6 +1,6 @@
 // Module included in the following assembly:
 //
-// * hcp-backup-restore-dr.adoc
+// * hosted_control_planes/hcp_high_availability/hcp-backup-restore-aws.adoc
 
 :_mod-docs-content-type: PROCEDURE
 [id="dr-hosted-cluster-within-aws-region-delete_{context}"]
diff --git a/modules/dr-hosted-cluster-within-aws-region-restore.adoc b/modules/dr-hosted-cluster-within-aws-region-restore.adoc
@@ -1,6 +1,6 @@
 // Module included in the following assembly:
 //
-// * hcp-backup-restore-dr.adoc
+// * hosted_control_planes/hcp_high_availability/hcp-backup-restore-aws.adoc
 
 :_mod-docs-content-type: PROCEDURE
 [id="dr-hosted-cluster-within-aws-region-restore_{context}"]
@@ -153,5 +153,3 @@ done
 .Next steps
 
 Shut down and delete your cluster.
-
-
diff --git a/modules/dr-hosted-cluster-within-aws-region-script.adoc b/modules/dr-hosted-cluster-within-aws-region-script.adoc
diff --git a/modules/hcp-recover-failing-etcd-pods.adoc b/modules/hcp-recover-failing-etcd-pods.adoc
@@ -0,0 +1,54 @@
+// Module included in the following assembly:
+//
+// * hosted_control_planes/hcp_high_availability/hcp-recovering-etcd-cluster.adoc
+
+:_mod-docs-content-type: PROCEDURE
+[id="hcp-recover-failing-etcd-pods_{context}"]
+= Recovering a failing etcd pod
+
+Each etcd pod of a 3-node cluster has its own persistent volume claim (PVC) to store its data. An etcd pod might fail because of corrupted or missing data. You can recover a failing etcd pod and its PVC.
+
+.Procedure
+
+. To confirm that the etcd pod is failing, enter the following command:
++
+[source,terminal]
+----
+$ oc get pods -l app=etcd -n <hosted_control_plane_namespace>
+----
++
+.Example output
+[source,terminal]
+----
+NAME     READY   STATUS             RESTARTS     AGE
+etcd-0   2/2     Running            0            64m
+etcd-1   2/2     Running            0            45m
+etcd-2   1/2     CrashLoopBackOff   1 (5s ago)   64m
+----
++
+The failing etcd pod might have the `CrashLoopBackOff` or `Error` status.
+
+. Delete the failing pod and its PVC by entering the following command:
++
+[source,terminal]
+----
+$ oc delete pvc/<etcd_pvc_name> pod/<etcd_pod_name> --wait=false
+----
+
+.Verification
+
+* Verify that a new etcd pod is up and running by entering the following command:
++
+[source,terminal]
+----
+$ oc get pods -l app=etcd -n <hosted_control_plane_namespace>
+----
++
+.Example output
+[source,terminal]
+----
+NAME     READY   STATUS    RESTARTS   AGE
+etcd-0   2/2     Running   0          67m
+etcd-1   2/2     Running   0          48m
+etcd-2   2/2     Running   0          2m2s
+----
diff --git a/modules/hosted-cluster-etcd-backup-restore-on-premise.adoc b/modules/hosted-cluster-etcd-backup-restore-on-premise.adoc
diff --git a/modules/hosted-cluster-etcd-status.adoc b/modules/hosted-cluster-etcd-status.adoc
diff --git a/modules/hosted-cluster-single-node-recovery.adoc b/modules/hosted-cluster-single-node-recovery.adoc
diff --git a/modules/restoring-etcd-snapshot-hosted-cluster.adoc b/modules/restoring-etcd-snapshot-hosted-cluster.adoc
