Skip to content

Commit 590fe01

Browse files
valmiranogueiravalmiranogueira
authored
Merge pull request #2178 from percona/fix-backup-tests
Fix backup tests
2 parents 1edf599 + 8b202df commit 590fe01

File tree

9 files changed

+355
-66
lines changed
  • Jenkinsfile
  • e2e-tests
    • data-at-rest-encryption
    • demand-backup-eks-credentials-irsa
    • demand-backup-eks-credentials
    • demand-backup-sharded
    • demand-backup
    • functions
    • scheduled-backup
    • upgrade-partial-backup

9 files changed

+355
-66
lines changed

‎Jenkinsfile‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -177,22 +177,24 @@ void makeReport() {
177177
}
178178

179179
void clusterRunner(String cluster) {
180-
def clusterCreated=0
180+
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]){
181+
def clusterCreated=0
181182

182-
for (int i=0; i<tests.size(); i++) {
183-
if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
184-
tests[i]["result"] = "failure"
185-
tests[i]["cluster"] = cluster
186-
if (clusterCreated == 0) {
187-
createCluster(cluster)
188-
clusterCreated++
183+
for (int i=0; i<tests.size(); i++) {
184+
if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
185+
tests[i]["result"] = "failure"
186+
tests[i]["cluster"] = cluster
187+
if (clusterCreated == 0) {
188+
createCluster(cluster)
189+
clusterCreated++
190+
}
191+
runTest(i)
189192
}
190-
runTest(i)
191193
}
192-
}
193194

194-
if (clusterCreated >= 1) {
195-
shutdownCluster(cluster)
195+
if (clusterCreated >= 1) {
196+
shutdownCluster(cluster)
197+
}
196198
}
197199
}
198200

@@ -265,6 +267,35 @@ EOF
265267
266268
curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
267269
"""
270+
installAzureCLI()
271+
azureAuth()
272+
}
273+
274+
void azureAuth() {
275+
withCredentials([azureServicePrincipal('PERCONA-OPERATORS-SP')]) {
276+
sh '''
277+
az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" -t "$AZURE_TENANT_ID" --allow-no-subscriptions
278+
az account set -s "$AZURE_SUBSCRIPTION_ID"
279+
'''
280+
}
281+
}
282+
283+
void installAzureCLI() {
284+
sh """
285+
if ! command -v az &>/dev/null; then
286+
echo "Installing Azure CLI for Hetzner instances..."
287+
sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc
288+
cat <<EOF | sudo tee /etc/yum.repos.d/azure-cli.repo
289+
[azure-cli]
290+
name=Azure CLI
291+
baseurl=https://packages.microsoft.com/yumrepos/azure-cli
292+
enabled=1
293+
gpgcheck=1
294+
gpgkey=https://packages.microsoft.com/keys/microsoft.asc
295+
EOF
296+
sudo dnf install azure-cli -y
297+
fi
298+
"""
268299
}
269300

270301
boolean isManualBuild() {

‎e2e-tests/data-at-rest-encryption/run‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace"
6666
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
6767
desc 'check backup and restore -- gcp-cs'
6868
backup_dest_gcp=$(get_backup_dest "$backup_name_gcp")
69-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null
69+
setup_gcs_credentials
70+
check_backup_existence_gcs "$backup_dest_gcp" "/rs0/myApp.test.gz"
7071
run_mongos 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster-mongos.$namespace"
7172
compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "-2nd"
7273
run_restore "$backup_name_gcp"

‎e2e-tests/demand-backup-eks-credentials-irsa/run‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ sleep 5
116116

117117
desc 'check backup and restore -- aws-s3'
118118
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
119-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null
119+
setup_aws_credentials
120+
check_backup_existence_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
120121
run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster-rs0.$namespace"
121122
compare_mongo_cmd "find" "myApp:myPass@$cluster-rs0-0.$cluster-rs0.$namespace" "-2nd"
122123
compare_mongo_cmd "find" "myApp:myPass@$cluster-rs0-1.$cluster-rs0.$namespace" "-2nd"

‎e2e-tests/demand-backup-eks-credentials/run‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ sleep 5
5757

5858
desc 'check backup and restore -- aws-s3'
5959
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
60-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null
60+
setup_aws_credentials
61+
check_backup_existence_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
6162
run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster.$namespace"
6263
compare_mongo_cmd "find" "myApp:myPass@$cluster-0.$cluster.$namespace" "-2nd"
6364
compare_mongo_cmd "find" "myApp:myPass@$cluster-1.$cluster.$namespace" "-2nd"
@@ -70,7 +71,7 @@ compare_mongo_cmd "find" "myApp:myPass@$cluster-2.$cluster.$namespace"
7071

7172
desc 'delete backup and check if it is removed from bucket -- aws-s3'
7273
kubectl_bin delete psmdb-backup --all
73-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
74+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
7475

7576
destroy $namespace
7677

‎e2e-tests/demand-backup-sharded/run‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,10 @@ sleep 5
108108
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
109109
desc 'check backup and restore -- aws-s3'
110110
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
111-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null
112-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs1/myApp1.test.gz" | gunzip >/dev/null
113-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs2/myApp2.test.gz" | gunzip >/dev/null
111+
setup_aws_credentials
112+
check_backup_existence_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
113+
check_backup_existence_aws "$backup_dest_aws" "/rs1/myApp1.test.gz"
114+
check_backup_existence_aws "$backup_dest_aws" "/rs2/myApp2.test.gz"
114115
insert_data_mongos "100501" "myApp" "" "$custom_port"
115116
insert_data_mongos "100501" "myApp1" "" "$custom_port"
116117
insert_data_mongos "100501" "myApp2" "" "$custom_port"
@@ -121,9 +122,10 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
121122

122123
desc 'check backup and restore -- gcp-cs'
123124
backup_dest_gcp=$(get_backup_dest "$backup_name_gcp")
124-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null
125-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs1/myApp1.test.gz" | gunzip >/dev/null
126-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs2/myApp2.test.gz" | gunzip >/dev/null
125+
setup_gcs_credentials
126+
check_backup_existence_gcs "$backup_dest_gcp" "/rs0/myApp.test.gz"
127+
check_backup_existence_gcs "$backup_dest_gcp" "/rs1/myApp1.test.gz"
128+
check_backup_existence_gcs "$backup_dest_gcp" "/rs2/myApp2.test.gz"
127129
insert_data_mongos "100501" "myApp" "" "$custom_port"
128130
insert_data_mongos "100501" "myApp1" "" "$custom_port"
129131
insert_data_mongos "100501" "myApp2" "" "$custom_port"
@@ -134,9 +136,10 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
134136

135137
desc 'check backup and restore -- azure-blob'
136138
backup_dest_azure=$(get_backup_dest "$backup_name_azure")
137-
curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null
138-
curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs1/myApp1.test.gz" | gunzip >/dev/null
139-
curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs2/myApp2.test.gz" | gunzip >/dev/null
139+
setup_azure_credentials
140+
check_backup_existence_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
141+
check_backup_existence_azure "$backup_dest_azure" "/rs1/myApp1.test.gz"
142+
check_backup_existence_azure "$backup_dest_azure" "/rs2/myApp2.test.gz"
140143
insert_data_mongos "100501" "myApp" "" "$custom_port"
141144
insert_data_mongos "100501" "myApp1" "" "$custom_port"
142145
insert_data_mongos "100501" "myApp2" "" "$custom_port"
@@ -169,9 +172,9 @@ if [[ $backup_exists -eq 1 ]]; then
169172
fi
170173

171174
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
172-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
173-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp}" "gcp-cs"
174-
check_backup_deletion "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}" "azure-blob"
175+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
176+
check_backup_deletion_gcs "$backup_dest_gcp" "/rs0/myApp.test.gz"
177+
check_backup_deletion_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
175178
fi
176179

177180
# Temporarily skipping this check
@@ -180,3 +183,4 @@ fi
180183

181184
kubectl_bin delete -f "$conf_dir/container-rc.yaml"
182185
destroy "$namespace"
186+
desc 'test passed'

‎e2e-tests/demand-backup/run‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ wait_backup_agent $cluster-1
188188
wait_backup_agent $cluster-2
189189

190190
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
191-
desc 'Check GCS pfofiles'
191+
desc 'Check GCS profiles'
192192
compare_pbm_profile_setup "some-name" "gcp-cs-s3"
193193
compare_pbm_profile_setup "some-name" "gcp-cs-sa"
194194
fi
@@ -223,22 +223,25 @@ sleep 5
223223
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
224224
desc 'check backup and restore -- aws-s3'
225225
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
226-
check_backup_in_storage ${backup_name_aws} s3 rs0 'myApp.test.gz'
226+
setup_aws_credentials
227+
check_backup_existence_aws ${backup_dest_aws} '/rs0/myApp.test.gz'
227228
run_recovery_check "$backup_name_aws" "$cluster"
228229

229230
desc 'check backup and restore -- gcp-cs-s3'
230231
backup_dest_gcp_s3=$(get_backup_dest "$backup_name_gcp_s3")
231-
check_backup_in_storage ${backup_name_gcp_s3} gcs rs0 'myApp.test.gz'
232+
setup_gcs_credentials
233+
check_backup_existence_gcs ${backup_dest_gcp_s3} '/rs0/myApp.test.gz'
232234
run_recovery_check "$backup_name_gcp_s3" "$cluster"
233235

234236
desc 'check backup and restore -- gcp-cs-sa'
235237
backup_dest_gcp_sa=$(get_backup_dest "$backup_name_gcp_sa")
236-
check_backup_in_storage ${backup_name_gcp_sa} gcs rs0 'myApp.test.gz'
238+
check_backup_existence_gcs ${backup_dest_gcp_sa} '/rs0/myApp.test.gz'
237239
run_recovery_check "$backup_name_gcp_sa" "$cluster"
238240

239241
desc 'check backup and restore -- azure-blob'
240242
backup_dest_azure=$(get_backup_dest "$backup_name_azure")
241-
check_backup_in_storage ${backup_name_azure} azure rs0 'myApp.test.gz'
243+
setup_azure_credentials
244+
check_backup_existence_azure ${backup_dest_azure} '/rs0/myApp.test.gz'
242245
run_recovery_check "$backup_name_azure" "$cluster"
243246
fi
244247

@@ -285,10 +288,10 @@ if [[ $backup_exists -eq 1 ]]; then
285288
fi
286289

287290
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
288-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
289-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_s3}" "gcp-cs-s3"
290-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_sa}" "gcp-cs-sa"
291-
check_backup_deletion "${backup_dest_azure}" "azure-blob"
291+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
292+
check_backup_deletion_gcs "$backup_dest_gcp_s3" "/rs0/myApp.test.gz"
293+
check_backup_deletion_gcs "$backup_dest_gcp_sa" "/rs0/myApp.test.gz"
294+
check_backup_deletion_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
292295
fi
293296

294297
desc 'checking backup deletion without cr'
@@ -327,10 +330,10 @@ if [[ $backup_exists -eq 1 ]]; then
327330
fi
328331

329332
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
330-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
331-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_s3}" "gcp-cs-s3"
332-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_sa}" "gcp-cs-sa"
333-
check_backup_deletion "${backup_dest_azure}" "azure-blob"
333+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
334+
check_backup_deletion_gcs "$backup_dest_gcp_s3" "/rs0/myApp.test.gz"
335+
check_backup_deletion_gcs "$backup_dest_gcp_sa" "/rs0/myApp.test.gz"
336+
check_backup_deletion_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
334337
fi
335338

336339
# Temporarily skipping this check

0 commit comments

Comments
 (0)