Skip to content

Commit fe00564

Browse files
gsmolkgsmolk
committed
tests: minor fixes
1 parent c0f0f7e commit fe00564

File tree

3 files changed

+93
-73
lines changed

3 files changed

+93
-73
lines changed

‎tests/archive.py‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import subprocess
88
from sys import exit
99
from time import sleep
10-
from shutil import copyfile
1110

1211

1312
module_name = 'archive'
@@ -248,7 +247,9 @@ def test_pgpro434_3(self):
248247
node.append_conf(
249248
'postgresql.auto.conf', "archive_command = '{0} %p %f'".format(
250249
archive_script_path))
250+
251251
node.slow_start()
252+
252253
try:
253254
self.backup_node(
254255
backup_dir, 'node', node,
@@ -262,6 +263,7 @@ def test_pgpro434_3(self):
262263
"Expecting Error because pg_stop_backup failed to answer.\n "
263264
"Output: {0} \n CMD: {1}".format(
264265
repr(self.output), self.cmd))
266+
265267
except ProbackupException as e:
266268
self.assertTrue(
267269
"ERROR: pg_stop_backup doesn't answer" in e.message and
@@ -327,9 +329,11 @@ def test_arhive_push_file_exists(self):
327329
)
328330
self.assertFalse('pg_probackup archive-push completed successfully' in log_content)
329331

330-
wal_src = os.path.join(node.data_dir, 'pg_wal', '000000010000000000000001')
332+
wal_src = os.path.join(
333+
node.data_dir, 'pg_wal', '000000010000000000000001')
331334
if self.archive_compress:
332-
with open(wal_src, 'rb') as f_in, gzip.open(file, 'wb', compresslevel=1) as f_out:
335+
with open(wal_src, 'rb') as f_in, gzip.open(
336+
file, 'wb', compresslevel=1) as f_out:
333337
shutil.copyfileobj(f_in, f_out)
334338
else:
335339
shutil.copyfile(wal_src, file)
@@ -506,10 +510,6 @@ def test_replica_archive(self):
506510
"postgres",
507511
"CHECKPOINT")
508512

509-
# copyfile(
510-
# os.path.join(backup_dir, 'wal/master/000000010000000000000002'),
511-
# os.path.join(backup_dir, 'wal/replica/000000010000000000000002'))
512-
513513
backup_id = self.backup_node(
514514
backup_dir, 'replica',
515515
replica, backup_type='page',
@@ -604,10 +604,9 @@ def test_master_and_replica_parallel_archiving(self):
604604
"md5(repeat(i::text,10))::tsvector as tsvector "
605605
"from generate_series(0, 60000) i")
606606

607-
# TAKE FULL ARCHIVE BACKUP FROM REPLICA
608-
copyfile(
609-
os.path.join(backup_dir, 'wal/master/000000010000000000000001'),
610-
os.path.join(backup_dir, 'wal/replica/000000010000000000000001'))
607+
master.psql(
608+
"postgres",
609+
"CHECKPOINT")
611610

612611
backup_id = self.backup_node(
613612
backup_dir, 'replica', replica,

‎tests/helpers/ptrack_helpers.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -874,8 +874,8 @@ def get_recovery_conf(self, node):
874874
return out_dict
875875

876876
def set_archiving(
877-
self, backup_dir, instance, node, replica=False, overwrite=False, compress=False,
878-
old_binary=False):
877+
self, backup_dir, instance, node, replica=False,
878+
overwrite=False, compress=False, old_binary=False):
879879

880880
if replica:
881881
archive_mode = 'always'

‎tests/replica.py‎

Lines changed: 81 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import subprocess
66
from sys import exit
77
import time
8-
from shutil import copyfile
98

109

1110
module_name = 'replica'
@@ -27,8 +26,9 @@ def test_replica_stream_ptrack_backup(self):
2726
set_replication=True,
2827
initdb_params=['--data-checksums'],
2928
pg_options={
30-
'wal_level': 'replica', 'max_wal_senders': '2',
31-
'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
29+
'wal_level': 'replica',
30+
'max_wal_senders': '2',
31+
'ptrack_enable': 'on'}
3232
)
3333
master.start()
3434
self.init_pb(backup_dir)
@@ -144,7 +144,6 @@ def test_replica_archive_page_backup(self):
144144
pg_options={
145145
'wal_level': 'replica',
146146
'max_wal_senders': '2',
147-
'checkpoint_timeout': '30s',
148147
'archive_timeout': '10s'}
149148
)
150149
self.init_pb(backup_dir)
@@ -171,7 +170,8 @@ def test_replica_archive_page_backup(self):
171170
self.restore_node(backup_dir, 'master', replica)
172171

173172
# Settings for Replica
174-
self.set_replica(master, replica)
173+
self.add_instance(backup_dir, 'replica', replica)
174+
self.set_replica(master, replica, synchronous=True)
175175
self.set_archiving(backup_dir, 'replica', replica, replica=True)
176176

177177
replica.slow_start(replica=True)
@@ -187,31 +187,23 @@ def test_replica_archive_page_backup(self):
187187
"postgres",
188188
"insert into t_heap as select i as id, md5(i::text) as text, "
189189
"md5(repeat(i::text,10))::tsvector as tsvector "
190-
"from generate_series(256,5120) i")
190+
"from generate_series(256,25120) i")
191191

192192
before = master.safe_psql("postgres", "SELECT * FROM t_heap")
193-
self.add_instance(backup_dir, 'replica', replica)
194193

195-
copyfile(
196-
os.path.join(backup_dir, 'wal/master/000000010000000000000003'),
197-
os.path.join(backup_dir, 'wal/replica/000000010000000000000003'))
198-
199-
copyfile(
200-
os.path.join(backup_dir, 'wal/master/000000010000000000000004'),
201-
os.path.join(backup_dir, 'wal/replica/000000010000000000000004'))
194+
master.psql(
195+
"postgres",
196+
"CHECKPOINT")
202197

203-
copyfile(
204-
os.path.join(backup_dir, 'wal/master/000000010000000000000005'),
205-
os.path.join(backup_dir, 'wal/replica/000000010000000000000005'))
198+
self.wait_until_replica_catch_with_master(master, replica)
206199

207200
backup_id = self.backup_node(
208201
backup_dir, 'replica', replica,
209202
options=[
210-
'--archive-timeout=30',
203+
'--archive-timeout=60',
211204
'--master-host=localhost',
212205
'--master-db=postgres',
213-
'--master-port={0}'.format(master.port),
214-
'--stream'])
206+
'--master-port={0}'.format(master.port)])
215207

216208
self.validate_pb(backup_dir, 'replica')
217209
self.assertEqual(
@@ -222,8 +214,13 @@ def test_replica_archive_page_backup(self):
222214
base_dir="{0}/{1}/node".format(module_name, fname))
223215
node.cleanup()
224216
self.restore_node(backup_dir, 'replica', data_dir=node.data_dir)
217+
225218
node.append_conf(
226219
'postgresql.auto.conf', 'port = {0}'.format(node.port))
220+
221+
node.append_conf(
222+
'postgresql.auto.conf', 'archive_mode = off'.format(node.port))
223+
227224
node.slow_start()
228225

229226
# CHECK DATA CORRECTNESS
@@ -234,41 +231,53 @@ def test_replica_archive_page_backup(self):
234231
# Change data on master, make PAGE backup from replica,
235232
# restore taken backup and check that restored data equal
236233
# to original data
237-
master.psql(
238-
"postgres",
239-
"insert into t_heap as select i as id, md5(i::text) as text, "
240-
"md5(repeat(i::text,10))::tsvector as tsvector "
241-
"from generate_series(512,22680) i")
234+
master.pgbench_init(scale=5)
242235

243-
before = master.safe_psql("postgres", "SELECT * FROM t_heap")
236+
pgbench = master.pgbench(
237+
options=['-T', '30', '-c', '2', '--no-vacuum'])
238+
239+
# master.psql(
240+
# "postgres",
241+
# "insert into t_heap as select i as id, md5(i::text) as text, "
242+
# "md5(repeat(i::text,10))::tsvector as tsvector "
243+
# "from generate_series(512,25120) i")
244244

245245
backup_id = self.backup_node(
246246
backup_dir, 'replica',
247247
replica, backup_type='page',
248248
options=[
249-
'--archive-timeout=30',
249+
'--archive-timeout=60',
250250
'--master-host=localhost',
251251
'--master-db=postgres',
252-
'--master-port={0}'.format(master.port),
253-
'--stream'])
252+
'--master-port={0}'.format(master.port)])
253+
254+
pgbench.wait()
255+
256+
self.switch_wal_segment(master)
257+
258+
before = master.safe_psql("postgres", "SELECT * FROM pgbench_accounts")
254259

255260
self.validate_pb(backup_dir, 'replica')
256261
self.assertEqual(
257262
'OK', self.show_pb(backup_dir, 'replica', backup_id)['status'])
258263

259264
# RESTORE PAGE BACKUP TAKEN FROM replica
260265
self.restore_node(
261-
backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id)
266+
backup_dir, 'replica', data_dir=node.data_dir,
267+
backup_id=backup_id)
262268

263269
node.append_conf(
264270
'postgresql.auto.conf', 'port = {0}'.format(node.port))
271+
265272
node.append_conf(
266273
'postgresql.auto.conf', 'archive_mode = off')
274+
267275
node.slow_start()
268276

269277
# CHECK DATA CORRECTNESS
270-
after = node.safe_psql("postgres", "SELECT * FROM t_heap")
271-
self.assertEqual(before, after)
278+
after = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts")
279+
self.assertEqual(
280+
before, after, 'Restored data is not equal to original')
272281

273282
self.add_instance(backup_dir, 'node', node)
274283
self.backup_node(
@@ -290,8 +299,9 @@ def test_make_replica_via_restore(self):
290299
set_replication=True,
291300
initdb_params=['--data-checksums'],
292301
pg_options={
293-
'wal_level': 'replica', 'max_wal_senders': '2',
294-
'checkpoint_timeout': '30s'}
302+
'wal_level': 'replica',
303+
'max_wal_senders': '2',
304+
'archive_timeout': '10s'}
295305
)
296306
self.init_pb(backup_dir)
297307
self.add_instance(backup_dir, 'master', master)
@@ -310,7 +320,7 @@ def test_make_replica_via_restore(self):
310320
"postgres",
311321
"create table t_heap as select i as id, md5(i::text) as text, "
312322
"md5(repeat(i::text,10))::tsvector as tsvector "
313-
"from generate_series(0,256) i")
323+
"from generate_series(0,8192) i")
314324

315325
before = master.safe_psql("postgres", "SELECT * FROM t_heap")
316326

@@ -320,6 +330,7 @@ def test_make_replica_via_restore(self):
320330
backup_dir, 'master', replica, options=['-R'])
321331

322332
# Settings for Replica
333+
self.add_instance(backup_dir, 'replica', replica)
323334
self.set_archiving(backup_dir, 'replica', replica, replica=True)
324335
replica.append_conf(
325336
'postgresql.auto.conf', 'port = {0}'.format(replica.port))
@@ -328,13 +339,9 @@ def test_make_replica_via_restore(self):
328339

329340
replica.slow_start(replica=True)
330341

331-
self.add_instance(backup_dir, 'replica', replica)
332-
333-
copyfile(
334-
os.path.join(backup_dir, 'wal/master/000000010000000000000003'),
335-
os.path.join(backup_dir, 'wal/replica/000000010000000000000003'))
336-
337-
self.backup_node(backup_dir, 'replica', replica)
342+
self.backup_node(
343+
backup_dir, 'replica', replica,
344+
options=['--archive-timeout=30s', '--stream'])
338345

339346
# Clean after yourself
340347
self.del_test_dir(module_name, fname)
@@ -353,14 +360,13 @@ def test_take_backup_from_delayed_replica(self):
353360
set_replication=True,
354361
initdb_params=['--data-checksums'],
355362
pg_options={
356-
'wal_level': 'replica', 'max_wal_senders': '2',
357-
'checkpoint_timeout': '30s'}
363+
'wal_level': 'replica',
364+
'max_wal_senders': '2',
365+
'archive_timeout': '10s'}
358366
)
359367
self.init_pb(backup_dir)
360368
self.add_instance(backup_dir, 'master', master)
361369
self.set_archiving(backup_dir, 'master', master)
362-
# force more frequent wal switch
363-
#master.append_conf('postgresql.auto.conf', 'archive_timeout = 10')
364370
master.slow_start()
365371

366372
replica = self.make_simple_node(
@@ -369,43 +375,58 @@ def test_take_backup_from_delayed_replica(self):
369375

370376
self.backup_node(backup_dir, 'master', master)
371377

378+
master.psql(
379+
"postgres",
380+
"create table t_heap as select i as id, md5(i::text) as text, "
381+
"md5(repeat(i::text,10))::tsvector as tsvector "
382+
"from generate_series(0,165000) i")
383+
384+
master.psql(
385+
"postgres",
386+
"CHECKPOINT")
387+
388+
master.psql(
389+
"postgres",
390+
"create table t_heap_1 as select i as id, md5(i::text) as text, "
391+
"md5(repeat(i::text,10))::tsvector as tsvector "
392+
"from generate_series(0,165000) i")
393+
372394
self.restore_node(
373395
backup_dir, 'master', replica, options=['-R'])
374396

375397
# Settings for Replica
376398
self.add_instance(backup_dir, 'replica', replica)
377399
self.set_archiving(backup_dir, 'replica', replica, replica=True)
378400

379-
# stupid hack
380-
copyfile(
381-
os.path.join(backup_dir, 'wal/master/000000010000000000000001'),
382-
os.path.join(backup_dir, 'wal/replica/000000010000000000000001'))
383-
384401
replica.append_conf(
385402
'postgresql.auto.conf', 'port = {0}'.format(replica.port))
386403

387-
replica.append_conf(
388-
'postgresql.auto.conf', 'hot_standby = on')
404+
replica.slow_start(replica=True)
405+
406+
self.wait_until_replica_catch_with_master(master, replica)
389407

390408
replica.append_conf(
391409
'recovery.conf', "recovery_min_apply_delay = '300s'")
392410

393-
replica.slow_start(replica=True)
411+
replica.restart()
394412

395413
master.pgbench_init(scale=10)
396414

397415
pgbench = master.pgbench(
398-
options=['-T', '30', '-c', '2', '--no-vacuum'])
416+
options=['-T', '60', '-c', '2', '--no-vacuum'])
399417

400418
self.backup_node(
401-
backup_dir, 'replica', replica)
419+
backup_dir, 'replica',
420+
replica, options=['--archive-timeout=60s'])
402421

403422
self.backup_node(
404423
backup_dir, 'replica', replica,
405-
data_dir=replica.data_dir, backup_type='page')
424+
data_dir=replica.data_dir,
425+
backup_type='page', options=['--archive-timeout=60s'])
406426

407427
self.backup_node(
408-
backup_dir, 'replica', replica, backup_type='delta')
428+
backup_dir, 'replica', replica,
429+
backup_type='delta', options=['--archive-timeout=60s'])
409430

410431
pgbench.wait()
411432

@@ -442,8 +463,8 @@ def test_make_block_from_future(self):
442463
set_replication=True,
443464
initdb_params=['--data-checksums'],
444465
pg_options={
445-
'wal_level': 'replica', 'max_wal_senders': '2',
446-
'checkpoint_timeout': '30s'}
466+
'wal_level': 'replica',
467+
'max_wal_senders': '2'}
447468
)
448469
self.init_pb(backup_dir)
449470
self.add_instance(backup_dir, 'master', master)

0 commit comments

Comments
 (0)