Skip to content

Commit 51832eb

Browse files
author
Lucas Gameiro
authored
[WIP][DPE-4751] Slice test_backups.py to speedup testing (#511)
* port some stabilizations from k8s * slice test in 2 groups * refactor test * fix aws test * split other tests in 3rd group * revert extra slicing * Revert "revert extra slicing" This reverts commit 24c82de. * fix last integration test * add unit test and fix integration test * fix condition on charm code * remove noqa * add docstring to helper function * revert change start primary change
1 parent 35849d3 commit 51832eb

File tree

2 files changed

+319
-263
lines changed

2 files changed

+319
-263
lines changed

‎tests/integration/helpers.py

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import asyncio
55
import itertools
66
import json
7+
import logging
78
import os
89
import subprocess
910
import tempfile
@@ -37,6 +38,8 @@
3738
STORAGE_PATH = METADATA["storage"]["pgdata"]["location"]
3839
APPLICATION_NAME = "postgresql-test-app"
3940

41+
logger = logging.getLogger(__name__)
42+
4043

4144
async def build_connection_string(
4245
ops_test: OpsTest,
@@ -1064,3 +1067,222 @@ def wait_for_relation_removed_between(
10641067
break
10651068
except RetryError:
10661069
assert False, "Relation failed to exit after 3 minutes."
1070+
1071+
1072+
async def backup_operations(
1073+
ops_test: OpsTest,
1074+
s3_integrator_app_name: str,
1075+
tls_certificates_app_name: str,
1076+
tls_config,
1077+
tls_channel,
1078+
credentials,
1079+
cloud,
1080+
config,
1081+
charm,
1082+
) -> None:
1083+
"""Basic set of operations for backup testing in different cloud providers."""
1084+
# Deploy S3 Integrator and TLS Certificates Operator.
1085+
await ops_test.model.deploy(s3_integrator_app_name)
1086+
await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel)
1087+
1088+
# Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now
1089+
# as archive_mode is disabled after restoring the backup) and to TLS Certificates Operator
1090+
# (to be able to create backups from replicas).
1091+
database_app_name = f"{DATABASE_APP_NAME}-{cloud.lower()}"
1092+
await ops_test.model.deploy(
1093+
charm,
1094+
application_name=database_app_name,
1095+
num_units=2,
1096+
series=CHARM_SERIES,
1097+
config={"profile": "testing"},
1098+
)
1099+
1100+
await ops_test.model.relate(database_app_name, tls_certificates_app_name)
1101+
async with ops_test.fast_forward(fast_interval="60s"):
1102+
await ops_test.model.wait_for_idle(apps=[database_app_name], status="active", timeout=1000)
1103+
await ops_test.model.relate(database_app_name, s3_integrator_app_name)
1104+
1105+
# Configure and set access and secret keys.
1106+
logger.info(f"configuring S3 integrator for {cloud}")
1107+
await ops_test.model.applications[s3_integrator_app_name].set_config(config)
1108+
action = await ops_test.model.units.get(f"{s3_integrator_app_name}/0").run_action(
1109+
"sync-s3-credentials",
1110+
**credentials,
1111+
)
1112+
await action.wait()
1113+
async with ops_test.fast_forward(fast_interval="60s"):
1114+
await ops_test.model.wait_for_idle(
1115+
apps=[database_app_name, s3_integrator_app_name], status="active", timeout=1500
1116+
)
1117+
1118+
primary = await get_primary(ops_test, f"{database_app_name}/0")
1119+
for unit in ops_test.model.applications[database_app_name].units:
1120+
if unit.name != primary:
1121+
replica = unit.name
1122+
break
1123+
1124+
# Write some data.
1125+
password = await get_password(ops_test, primary)
1126+
address = get_unit_address(ops_test, primary)
1127+
logger.info("creating a table in the database")
1128+
with db_connect(host=address, password=password) as connection:
1129+
connection.autocommit = True
1130+
connection.cursor().execute(
1131+
"CREATE TABLE IF NOT EXISTS backup_table_1 (test_collumn INT );"
1132+
)
1133+
connection.close()
1134+
1135+
# Run the "create backup" action.
1136+
logger.info("creating a backup")
1137+
action = await ops_test.model.units.get(replica).run_action("create-backup")
1138+
await action.wait()
1139+
backup_status = action.results.get("backup-status")
1140+
assert backup_status, "backup hasn't succeeded"
1141+
await ops_test.model.wait_for_idle(
1142+
apps=[database_app_name, s3_integrator_app_name], status="active", timeout=1000
1143+
)
1144+
1145+
# With a stable cluster, Run the "create backup" action
1146+
async with ops_test.fast_forward():
1147+
await ops_test.model.wait_for_idle(status="active", timeout=1000, idle_period=30)
1148+
logger.info("listing the available backups")
1149+
action = await ops_test.model.units.get(replica).run_action("list-backups")
1150+
await action.wait()
1151+
backups = action.results.get("backups")
1152+
# 2 lines for header output, 1 backup line ==> 3 total lines
1153+
assert len(backups.split("\n")) == 3, "full backup is not outputted"
1154+
await ops_test.model.wait_for_idle(status="active", timeout=1000)
1155+
1156+
# Write some data.
1157+
logger.info("creating a second table in the database")
1158+
with db_connect(host=address, password=password) as connection:
1159+
connection.autocommit = True
1160+
connection.cursor().execute("CREATE TABLE backup_table_2 (test_collumn INT );")
1161+
connection.close()
1162+
1163+
# Run the "create backup" action.
1164+
logger.info("creating a backup")
1165+
action = await ops_test.model.units.get(replica).run_action(
1166+
"create-backup", **{"type": "differential"}
1167+
)
1168+
await action.wait()
1169+
backup_status = action.results.get("backup-status")
1170+
assert backup_status, "backup hasn't succeeded"
1171+
async with ops_test.fast_forward():
1172+
await ops_test.model.wait_for_idle(status="active", timeout=1000)
1173+
1174+
# Run the "list backups" action.
1175+
logger.info("listing the available backups")
1176+
action = await ops_test.model.units.get(replica).run_action("list-backups")
1177+
await action.wait()
1178+
backups = action.results.get("backups")
1179+
# 2 lines for header output, 2 backup lines ==> 4 total lines
1180+
assert len(backups.split("\n")) == 4, "differential backup is not outputted"
1181+
await ops_test.model.wait_for_idle(status="active", timeout=1000)
1182+
1183+
# Write some data.
1184+
logger.info("creating a second table in the database")
1185+
with db_connect(host=address, password=password) as connection:
1186+
connection.autocommit = True
1187+
connection.cursor().execute("CREATE TABLE backup_table_3 (test_collumn INT );")
1188+
connection.close()
1189+
# Scale down to be able to restore.
1190+
async with ops_test.fast_forward():
1191+
await ops_test.model.destroy_unit(replica)
1192+
await ops_test.model.block_until(
1193+
lambda: len(ops_test.model.applications[database_app_name].units) == 1
1194+
)
1195+
1196+
for unit in ops_test.model.applications[database_app_name].units:
1197+
remaining_unit = unit
1198+
break
1199+
1200+
# Run the "restore backup" action for differential backup.
1201+
for attempt in Retrying(
1202+
stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30)
1203+
):
1204+
with attempt:
1205+
logger.info("restoring the backup")
1206+
last_diff_backup = backups.split("\n")[-1]
1207+
backup_id = last_diff_backup.split()[0]
1208+
action = await remaining_unit.run_action("restore", **{"backup-id": backup_id})
1209+
await action.wait()
1210+
restore_status = action.results.get("restore-status")
1211+
assert restore_status, "restore hasn't succeeded"
1212+
1213+
# Wait for the restore to complete.
1214+
async with ops_test.fast_forward():
1215+
await ops_test.model.wait_for_idle(status="active", timeout=1000)
1216+
1217+
# Check that the backup was correctly restored by having only the first created table.
1218+
logger.info("checking that the backup was correctly restored")
1219+
primary = await get_primary(ops_test, remaining_unit.name)
1220+
address = get_unit_address(ops_test, primary)
1221+
with db_connect(host=address, password=password) as connection, connection.cursor() as cursor:
1222+
cursor.execute(
1223+
"SELECT EXISTS (SELECT FROM information_schema.tables"
1224+
" WHERE table_schema = 'public' AND table_name = 'backup_table_1');"
1225+
)
1226+
assert cursor.fetchone()[
1227+
0
1228+
], "backup wasn't correctly restored: table 'backup_table_1' doesn't exist"
1229+
cursor.execute(
1230+
"SELECT EXISTS (SELECT FROM information_schema.tables"
1231+
" WHERE table_schema = 'public' AND table_name = 'backup_table_2');"
1232+
)
1233+
assert cursor.fetchone()[
1234+
0
1235+
], "backup wasn't correctly restored: table 'backup_table_2' doesn't exist"
1236+
cursor.execute(
1237+
"SELECT EXISTS (SELECT FROM information_schema.tables"
1238+
" WHERE table_schema = 'public' AND table_name = 'backup_table_3');"
1239+
)
1240+
assert not cursor.fetchone()[
1241+
0
1242+
], "backup wasn't correctly restored: table 'backup_table_3' exists"
1243+
connection.close()
1244+
1245+
# Run the "restore backup" action for full backup.
1246+
for attempt in Retrying(
1247+
stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30)
1248+
):
1249+
with attempt:
1250+
logger.info("restoring the backup")
1251+
last_full_backup = backups.split("\n")[-2]
1252+
backup_id = last_full_backup.split()[0]
1253+
action = await remaining_unit.run_action("restore", **{"backup-id": backup_id})
1254+
await action.wait()
1255+
restore_status = action.results.get("restore-status")
1256+
assert restore_status, "restore hasn't succeeded"
1257+
1258+
# Wait for the restore to complete.
1259+
async with ops_test.fast_forward():
1260+
await ops_test.model.wait_for_idle(status="active", timeout=1000)
1261+
1262+
# Check that the backup was correctly restored by having only the first created table.
1263+
primary = await get_primary(ops_test, remaining_unit.name)
1264+
address = get_unit_address(ops_test, primary)
1265+
logger.info("checking that the backup was correctly restored")
1266+
with db_connect(host=address, password=password) as connection, connection.cursor() as cursor:
1267+
cursor.execute(
1268+
"SELECT EXISTS (SELECT FROM information_schema.tables"
1269+
" WHERE table_schema = 'public' AND table_name = 'backup_table_1');"
1270+
)
1271+
assert cursor.fetchone()[
1272+
0
1273+
], "backup wasn't correctly restored: table 'backup_table_1' doesn't exist"
1274+
cursor.execute(
1275+
"SELECT EXISTS (SELECT FROM information_schema.tables"
1276+
" WHERE table_schema = 'public' AND table_name = 'backup_table_2');"
1277+
)
1278+
assert not cursor.fetchone()[
1279+
0
1280+
], "backup wasn't correctly restored: table 'backup_table_2' exists"
1281+
cursor.execute(
1282+
"SELECT EXISTS (SELECT FROM information_schema.tables"
1283+
" WHERE table_schema = 'public' AND table_name = 'backup_table_3');"
1284+
)
1285+
assert not cursor.fetchone()[
1286+
0
1287+
], "backup wasn't correctly restored: table 'backup_table_3' exists"
1288+
connection.close()

0 commit comments

Comments
 (0)