Skip to content

Commit 07e4585

Browse files
bluzarragabluzarraga
authored
[OADP Auto] enable multi-instance BR via multiple runs with different properties files (#2629)
* limit restore resources to only include tenant namespaces from properties file Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * update wait for deploy utilities function Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * fix unset var Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * debugging Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * debugging Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * cleanup output by reducing file manipulation attempts Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * no olm problem Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * restore cluster scoped crds Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * get around crd problem Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * get around crd problem Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * typo Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * ensure nss crd is included Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * ensure cluster scoped resources are restored for no olm Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * simplify the crd restore Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * restore all crds across all instances Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * debugging Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * update file names to differentiate between restores Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * update retore names in wait functions Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * left out csdb data Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * cleanup Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * cleanup Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * change to restore cluster for clarity Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> * include webhooks Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com> --------- Signed-off-by: Ben Luzarraga <luzarragaben@gmail.com>
1 parent d73952e commit 07e4585

File tree

2 files changed

+161
-34
lines changed

2 files changed

+161
-34
lines changed

‎br-testing-automation/auto-br-oadp.sh‎

Lines changed: 104 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -278,20 +278,64 @@ function restore_cpfs(){
278278
mkdir templates
279279
info "Copying template files..."
280280
cp -r ../velero/restore ${BASE_DIR}/templates/
281+
282+
local all_namespaces=()
283+
local namespaces=("$OPERATOR_NS")
284+
local tethered_array=()
285+
local singleton_namespaces=()
286+
local extra_namespaces=("openshift-marketplace" "openshift-config" "kube-public")
287+
288+
if [[ $SERVICES_NS != "$OPERATOR_NS" ]]; then
289+
namespaces+=("$SERVICES_NS")
290+
fi
291+
292+
if [[ $TETHERED_NS != "" ]]; then
293+
local space_delimited="${TETHERED_NS//,/ }"
294+
tethered_array=($space_delimited)
295+
fi
296+
297+
if [[ $ENABLE_CERT_MANAGER == "true" ]]; then
298+
singleton_namespaces+=("$CERT_MANAGER_NAMESPACE")
299+
fi
300+
if [[ $ENABLE_LICENSING == "true" ]]; then
301+
singleton_namespaces+=("$LICENSING_NAMESPACE")
302+
fi
303+
if [[ $ENABLE_LSR == "true" ]]; then
304+
singleton_namespaces+=("$LSR_NAMESPACE")
305+
fi
306+
307+
all_namespaces=("${namespaces[@]}" "${tethered_array[@]}" "${extra_namespaces[@]}" "${singleton_namespaces[@]}")
308+
info "All namespaces in scope ${all_namespaces[*]}"
281309

282310
for file in "${BASE_DIR}/templates/restore"/*; do
283-
sed -i -E "s/__BACKUP_NAME__/$BACKUP_NAME/" $file
284-
if [[ $OADP_NS != "velero" ]]; then
285-
set_oadp_namespace $file
311+
if [[ "${file}" == *.yaml ]]; then
312+
sed -i -E "s/__BACKUP_NAME__/$BACKUP_NAME/" $file
313+
if [[ $OADP_NS != "velero" ]]; then
314+
set_oadp_namespace $file
315+
fi
316+
update_restore_name $file
317+
if [[ "${file}" != *restore-crd.yaml ]] && [[ "${file}" != *restore-cluster-auto.yaml ]]; then
318+
update_restore_namespaces $file "${all_namespaces[@]}"
319+
fi
320+
else
321+
info "File $file does not end in \".yaml\", skipping..."
286322
fi
287323
done
288324
#start no olm specific
289325
if [[ $NO_OLM == "true" ]]; then
290326
#update values in no-olm directory for no olm specific restore resources
291327
for file in "${BASE_DIR}/templates/restore/no-olm"/*; do
292-
sed -i -E "s/__BACKUP_NAME__/$BACKUP_NAME/" $file
293-
if [[ $OADP_NS != "velero" ]]; then
294-
set_oadp_namespace $file
328+
if [[ "${file}" == *.yaml ]]; then
329+
sed -i -E "s/__BACKUP_NAME__/$BACKUP_NAME/" $file
330+
if [[ $OADP_NS != "velero" ]]; then
331+
set_oadp_namespace $file
332+
fi
333+
update_restore_name $file
334+
if [[ "${file}" != *restore-crd.yaml ]] && [[ "${file}" != *restore-cluster-auto.yaml ]]; then
335+
update_restore_namespaces $file "${all_namespaces[@]}"
336+
fi
337+
else
338+
info "File $file does not end in \".yaml\", skipping..."
295339
fi
296340
done
297341
fi
@@ -303,33 +347,33 @@ function restore_cpfs(){
303347
info "Restoring namespaces and entitlement keys..."
304348
${OC} apply -f ${BASE_DIR}/templates/restore/restore-namespace.yaml -f ${BASE_DIR}/templates/restore/restore-pull-secret.yaml -f ${BASE_DIR}/templates/restore/restore-entitlementkey.yaml
305349
${OC} get restores.velero.io -n $OADP_NS $custom_columns_str
306-
wait_for_restore restore-namespace
307-
wait_for_restore restore-entitlementkey
350+
wait_for_restore restore-namespace-$OPERATOR_NS
351+
wait_for_restore restore-entitlementkey-$OPERATOR_NS
308352

309353
#start olm specific
310354
if [[ $NO_OLM == "false" ]]; then
311355
info "Cleanup existing pull secret..."
312356
${OC} delete secret pull-secret -n openshift-config --ignore-not-found
313357
info "Restoring pull secret..."
314358
${OC} apply -f ${BASE_DIR}/templates/restore/restore-pull-secret.yaml
315-
wait_for_restore restore-pull-secret
359+
wait_for_restore restore-pull-secret-$OPERATOR_NS
316360
${OC} get restores.velero.io -n $OADP_NS $custom_columns_str
317361
info "Restoring catalog sources..."
318362
${OC} apply -f ${BASE_DIR}/templates/restore/restore-catalog.yaml
319-
wait_for_restore restore-catalog
363+
wait_for_restore restore-catalog-$OPERATOR_NS
320364
info "Restore operator groups..."
321365
${OC} apply -f ${BASE_DIR}/templates/restore/restore-operatorgroup.yaml
322366
${OC} get restores.velero.io -n $OADP_NS $custom_columns_str
323-
wait_for_restore restore-operatorgroup
324-
wait_for_restore restore-crd
367+
wait_for_restore restore-operatorgroup-$OPERATOR_NS
325368
fi
326369
#end olm specific
327370
info "Restore CRDs..."
328-
${OC} apply -f ${BASE_DIR}/templates/restore/restore-crd.yaml
329-
wait_for_restore restore-crd
371+
${OC} apply -f ${BASE_DIR}/templates/restore/restore-crd.yaml && ${OC} apply -f ${BASE_DIR}/templates/restore/restore-cluster-auto.yaml
372+
wait_for_restore restore-crd-$OPERATOR_NS
373+
wait_for_restore restore-cluster-auto-$OPERATOR_NS
330374
info "Restore configmaps..."
331375
${OC} apply -f ${BASE_DIR}/templates/restore/restore-configmap.yaml
332-
wait_for_restore restore-configmap
376+
wait_for_restore restore-configmap-$OPERATOR_NS
333377

334378
#Singleton subscriptions (Cert manager, licensing, LSR)
335379
if [[ $RESTORE_SINGLETONS == "true" ]]; then
@@ -338,22 +382,22 @@ function restore_cpfs(){
338382
info "Restoring licensing configmaps..."
339383
#this will restore the licensing chart in no olm
340384
${OC} apply -f ${BASE_DIR}/templates/restore/restore-licensing.yaml
341-
wait_for_restore restore-licensing
385+
wait_for_restore restore-licensing-$OPERATOR_NS
342386
fi
343387
# same principle for lsr here as for licensing above
344388
if [[ $ENABLE_LSR == "true" ]]; then
345389
info "Restoring License Service Reporter instance..."
346390
#this will restore the LSR chart in no olm
347391
${OC} apply -f ${BASE_DIR}/templates/restore/restore-lsr.yaml
348-
wait_for_restore restore-lsr
392+
wait_for_restore restore-lsr-$OPERATOR_NS
349393
fi
350394

351395
#start olm specific
352396
if [[ $NO_OLM == "false" ]]; then
353397
#this step restores the cert manager and licensing subs
354398
info "Restoring Singleton subscriptions..."
355399
${OC} apply -f ${BASE_DIR}/templates/restore/restore-singleton-subscriptions.yaml
356-
wait_for_restore restore-singleton-subscription
400+
wait_for_restore restore-singleton-subscription-$OPERATOR_NS
357401
fi
358402
#end olm specific
359403

@@ -362,43 +406,43 @@ function restore_cpfs(){
362406
#restore cert manager chart
363407
info "Restoring Cert Manager Operator Chart..."
364408
${OC} apply -f ${BASE_DIR}/templates/restore/no-olm/restore-ibm-cm-chart.yaml
365-
wait_for_restore restore-ibm-cm-chart
409+
wait_for_restore restore-ibm-cm-chart-$OPERATOR_NS
366410
fi
367411
#end no olm specific
368412

369413
if [[ $ENABLE_LSR == "true" ]]; then
370414
info "Restoring License Service Reporter data..."
371415
wait_for_deployment $LSR_NAMESPACE "ibm-license-service-reporter-instance" 30
372416
${OC} apply -f ${BASE_DIR}/templates/restore/restore-lsr-data.yaml
373-
wait_for_restore restore-lsr-data
417+
wait_for_restore restore-lsr-data-$OPERATOR_NS
374418
fi
375419
fi
376420
${OC} get restores.velero.io -n $OADP_NS $custom_columns_str
377421

378422
wait_for_cert_manager $CERT_MANAGER_NAMESPACE $SERVICES_NS
379423
info "Restoring cert manager resources (secrets, certificates, issuers, etc.)..."
380424
${OC} apply -f ${BASE_DIR}/templates/restore/restore-cert-manager.yaml
381-
wait_for_restore restore-cert-manager
425+
wait_for_restore restore-cert-manager-$OPERATOR_NS
382426

383427
#Restore the common service CR and the tenant scope via nss
384428
info "Restoring common service CR..."
385429
${OC} apply -f ${BASE_DIR}/templates/restore/restore-commonservice.yaml
386-
wait_for_restore restore-commonservice
430+
wait_for_restore restore-commonservice-$OPERATOR_NS
387431

388432
#start olm specific
389433
if [[ $NO_OLM == "false" ]]; then
390434
if [[ $NSS_ENABLED == "true" ]]; then
391435
info "Restoring Namespace Scope resources..."
392436
#this will restore nss cluster and chart resources as well in no olm
393437
${OC} apply -f ${BASE_DIR}/templates/restore/restore-nss.yaml
394-
wait_for_restore restore-nss
438+
wait_for_restore restore-nss-$OPERATOR_NS
395439
${OC} get restores.velero.io -n $OADP_NS $custom_columns_str
396440
validate_nss $OPERATOR_NS
397441
fi
398442
#restore common service subscription and odlm operator
399443
info "Restore CS and ODLM Operators..."
400444
${OC} apply -f ${BASE_DIR}/templates/restore/restore-subscriptions.yaml
401-
wait_for_restore restore-subscription
445+
wait_for_restore restore-subscription-$OPERATOR_NS
402446
validate_cs_odlm $OPERATOR_NS
403447
fi
404448
#end olm specific
@@ -407,22 +451,22 @@ function restore_cpfs(){
407451
#restore cluster charts no-olm/restore-cluster-scope.yaml
408452
info "Restoring cluster wide operator resources..."
409453
${OC} apply -f ${BASE_DIR}/templates/restore/no-olm/restore-cluster-scope.yaml
410-
wait_for_restore restore-cluster-charts
454+
wait_for_restore restore-cluster-charts-$OPERATOR_NS
411455

412456
#restore namespace scope operator chart
413457
if [[ $NSS_ENABLED == "true" ]]; then
414458
info "Restoring Namespace Scope resources..."
415459
#this will restore nss chart resources as well in no olm
416460
${OC} apply -f ${BASE_DIR}/templates/restore/restore-nss.yaml
417-
wait_for_restore restore-nss
461+
wait_for_restore restore-nss-$OPERATOR_NS
418462
${OC} get restores.velero.io -n $OADP_NS $custom_columns_str
419463
wait_for_deployment $OPERATOR_NS ibm-namespace-scope-operator
420464
fi
421465

422466
#restore cs op/odlm chart no-olm/restore-installer-ns-charts.yaml
423467
info "Restoring CS Operator and ODLM charts..."
424468
${OC} apply -f ${BASE_DIR}/templates/restore/no-olm/restore-installer-ns-charts.yaml
425-
wait_for_restore restore-installer-charts
469+
wait_for_restore restore-installer-charts-$OPERATOR_NS
426470
wait_for_deployment $OPERATOR_NS ibm-common-service-operator
427471
wait_for_deployment $OPERATOR_NS operand-deployment-lifecycle-manager
428472
#restore im ns chart no-olm/restore-im-ns-charts.yaml
@@ -431,7 +475,7 @@ function restore_cpfs(){
431475
if [[ $IM_ENABLED == "true" ]] || [[ $ZEN_ENABLED == "true" ]]; then
432476
info "Restoring IM, Common UI, and EDB charts..."
433477
${OC} apply -f ${BASE_DIR}/templates/restore/no-olm/restore-im-ns-charts.yaml
434-
wait_for_restore restore-im-charts
478+
wait_for_restore restore-im-charts-$OPERATOR_NS
435479
#TODO implement check for im so we don't wait for im and ui deployments in case where zen does not enable im since this is where we need to restore and check edb
436480
wait_for_deployment $OPERATOR_NS ibm-iam-operator
437481
wait_for_deployment $OPERATOR_NS ibm-commonui-operator
@@ -445,20 +489,20 @@ function restore_cpfs(){
445489
if [[ $UMS_ENABLED == "true" ]]; then
446490
info "Restoring UMS resources..."
447491
${OC} apply -f ${BASE_DIR}/templates/restore/restore-ums.yaml
448-
wait_for_restore restore-ums
492+
wait_for_restore restore-ums-$OPERATOR_NS
449493
fi
450494

451495
${OC} get restores.velero.io -n $OADP_NS $custom_columns_str
452496
info "Restoring operands..."
453497
${OC} apply -f ${BASE_DIR}/templates/restore/restore-operands.yaml
454-
wait_for_restore restore-operands
498+
wait_for_restore restore-operands-$OPERATOR_NS
455499

456500
#start no olm specific
457501
if [[ $NO_OLM == "true" ]]; then
458502
#restore zen ns chart no-olm/restore-zen-ns-charts.yaml
459503
info "Restoring Zen chart..."
460504
${OC} apply -f ${BASE_DIR}/templates/restore/no-olm/restore-zen-ns-chart.yaml
461-
wait_for_restore restore-zen-chart
505+
wait_for_restore restore-zen-chart-$OPERATOR_NS
462506
fi
463507
#end no olm specific
464508

@@ -534,7 +578,7 @@ function restore_im() {
534578
wait_for_deployment $SERVICES_NS "account-iam-ui-account-deployment"
535579
fi
536580
${OC} apply -f ${BASE_DIR}/templates/restore/restore-cs-db.yaml
537-
wait_for_restore restore-cs-db-data
581+
wait_for_restore restore-cs-db-data-$OPERATOR_NS
538582
success "IM data restored successfully."
539583
}
540584

@@ -549,11 +593,11 @@ function wait_for_im() {
549593
function restore_zen() {
550594
info "Restoring zenservice..."
551595
${OC} apply -f ${BASE_DIR}/templates/restore/restore-zen.yaml
552-
wait_for_restore restore-zen
596+
wait_for_restore restore-zen-$OPERATOR_NS
553597
wait_for_zenservice
554598
info "Restoring zen data..."
555599
${OC} apply -f ${BASE_DIR}/templates/restore/restore-zen5-data.yaml
556-
wait_for_restore restore-zen5-data
600+
wait_for_restore restore-zen5-data-$OPERATOR_NS
557601
success "Zen data restored successfully"
558602
}
559603

@@ -928,6 +972,32 @@ function check_yq() {
928972
fi
929973
}
930974

975+
function update_restore_namespaces() {
976+
local file="$1"
977+
shift
978+
local namespaces=("$@")
979+
info "Updating restore resource in file $file to specify namespaces ${namespaces[*]}..."
980+
981+
# Build namespace array
982+
local json_array="["
983+
for i in "${!namespaces[@]}"; do
984+
[ $i -gt 0 ] && json_array+=","
985+
json_array+="\"${namespaces[$i]}\""
986+
done
987+
json_array+="]"
988+
989+
# Update Restore file
990+
${YQ} eval ".spec.includedNamespaces = $json_array" -i "$file"
991+
992+
}
993+
994+
function update_restore_name() {
995+
local file="$1"
996+
cur_name=$(${YQ} '.metadata.name' $file)
997+
cur_name+="-${OPERATOR_NS}"
998+
${YQ} -i '.metadata.name = "'"$cur_name"'"' $file
999+
}
1000+
9311001
function msg() {
9321002
printf '%b\n' "$1"
9331003
}

‎velero/restore/restore-cluster-auto.yaml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
apiVersion: velero.io/v1
2+
kind: Restore
3+
metadata:
4+
name: restore-cluster-auto
5+
namespace: velero
6+
spec:
7+
backupName: __BACKUP_NAME__
8+
excludedResources:
9+
- nodes
10+
- events
11+
- events.events.k8s.io
12+
- backups.velero.io
13+
- restores.velero.io
14+
- resticrepositories.velero.io
15+
- deployments
16+
- serviceaccounts
17+
- role.rbac.authorization.k8s.io
18+
- rolebindings.rbac.authorization.k8s.io
19+
- roles.authorization.openshift.io
20+
- rolebindings.authorization.openshift.io
21+
- namespacescopes.operator.ibm.com
22+
- zenservices.zen.cpd.ibm.com
23+
- subscription.operators.coreos.com
24+
- secrets
25+
- commonservices.operator.ibm.com
26+
- configmaps
27+
- issuers.cert-manager.io
28+
- certificates.cert-manager.io
29+
- ibmlicensings.operator.ibm.com
30+
- ibmlicenseservicereporters.operator.ibm.com
31+
includeResources:
32+
- customresourcedefinitions.apiextensions.k8s.io
33+
- clusterrole.rbac.authorization.k8s.io
34+
- clusterrolebinding.rbac.authorization.k8s.io
35+
- validatingwebhookconfiguration.admissionregistration.k8s.io
36+
- mutatingwebhookconfiguration.admissionregistration.k8s.io
37+
hooks: {}
38+
includedNamespaces:
39+
- '*'
40+
orLabelSelectors:
41+
- matchExpressions:
42+
- key: foundationservices.cloudpak.ibm.com
43+
operator: In
44+
values:
45+
- crd
46+
- nss-cluster
47+
- nss
48+
- ui-chart
49+
- edb-chart
50+
- iam-chart
51+
- cs-chart
52+
- odlm-chart
53+
- zen-chart
54+
- zen
55+
- ls-cluster
56+
- ls-chart
57+
- licensing

0 commit comments

Comments
 (0)