Skip to content

Commit dcd85e9

Browse files
gsmolkgsmolk
committed
Merge branch 'repprotocol_v1'
2 parents 8eda9d4 + dc792f5 commit dcd85e9

File tree

3 files changed

+95
-49
lines changed

3 files changed

+95
-49
lines changed

‎src/backup.c

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -596,11 +596,11 @@ do_backup_instance(void)
596596
{
597597
XLogRecPtr ptrack_lsn = get_last_ptrack_lsn();
598598

599-
if (ptrack_lsn > prev_backup->stop_lsn)
599+
if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr)
600600
{
601-
elog(ERROR, "LSN from ptrack_control %lx differs from LSN of previous backup %lx.\n"
601+
elog(ERROR, "LSN from ptrack_control %lx differs from STOP LSN of previous backup %lx.\n"
602602
"Create new full backup before an incremental one.",
603-
ptrack_lsn, prev_backup->start_lsn);
603+
ptrack_lsn, prev_backup->stop_lsn);
604604
}
605605
parray_qsort(backup_files_list, pgFileComparePath);
606606
make_pagemap_from_ptrack(backup_files_list);
@@ -782,14 +782,17 @@ do_backup(void)
782782
current.stream = stream_wal;
783783

784784
is_ptrack_support = pg_ptrack_support();
785+
if (is_ptrack_support)
786+
{
787+
is_ptrack_enable = pg_ptrack_enable();
788+
}
785789

786790
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
787791
{
788792
if (!is_ptrack_support)
789793
elog(ERROR, "This PostgreSQL instance does not support ptrack");
790794
else
791795
{
792-
is_ptrack_enable = pg_ptrack_enable();
793796
if(!is_ptrack_enable)
794797
elog(ERROR, "Ptrack is disabled");
795798
}

‎tests/helpers/ptrack_helpers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -599,7 +599,7 @@ def del_test_dir(self, module_name, fname):
599599
def pgdata_content(self, directory):
600600
""" return dict with directory content. TAKE IT AFTER CHECKPOINT or BACKUP"""
601601
dirs_to_ignore = ['pg_xlog', 'pg_wal', 'pg_log', 'pg_stat_tmp', 'pg_subtrans', 'pg_notify']
602-
files_to_ignore = ['postmaster.pid', 'postmaster.opts', 'pg_internal.init']
602+
files_to_ignore = ['postmaster.pid', 'postmaster.opts', 'pg_internal.init', 'postgresql.auto.conf']
603603
suffixes_to_ignore = ('_ptrack', 'ptrack_control', 'pg_control', 'ptrack_init')
604604
directory_dict = {}
605605
directory_dict['pgdata'] = directory

‎tests/ptrack.py

Lines changed: 87 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
class PtrackBackupTest(ProbackupTest, unittest.TestCase):
1414

15-
#@unittest.skip("skip")
15+
# @unittest.skip("skip")
1616
# @unittest.expectedFailure
1717
def test_ptrack_enable(self):
1818
"""make ptrack without full backup, should result in error"""
@@ -42,7 +42,54 @@ def test_ptrack_enable(self):
4242
# Clean after yourself
4343
self.del_test_dir(module_name, fname)
4444

45-
#@unittest.skip("skip")
45+
# @unittest.skip("skip")
46+
# @unittest.expectedFailure
47+
def test_ptrack_disable(self):
48+
"""Take full backup, disable ptrack restart postgresql, enable ptrack, restart postgresql, take ptrack backup which should fail"""
49+
self.maxDiff = None
50+
fname = self.id().split('.')[3]
51+
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
52+
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
53+
set_replication=True,
54+
initdb_params=['--data-checksums'],
55+
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
56+
)
57+
self.init_pb(backup_dir)
58+
self.add_instance(backup_dir, 'node', node)
59+
self.set_archiving(backup_dir, 'node', node)
60+
node.start()
61+
62+
# FULL BACKUP
63+
# print('START')
64+
# print(node.safe_psql('postgres', "select pg_ptrack_control_lsn()"))
65+
self.backup_node(backup_dir, 'node', node, options=['--stream'])
66+
# print('AFTER FULL')
67+
# print(node.safe_psql('postgres', "select pg_ptrack_control_lsn()"))
68+
# DISABLE PTRACK
69+
node.safe_psql('postgres', "alter system set ptrack_enable to off")
70+
node.restart()
71+
# print('DISABLED')
72+
# print(node.safe_psql('postgres', "select pg_ptrack_control_lsn()"))
73+
# ENABLE PTRACK
74+
node.safe_psql('postgres', "alter system set ptrack_enable to on")
75+
node.restart()
76+
# print('ENABLED')
77+
# print(node.safe_psql('postgres', "select pg_ptrack_control_lsn()"))
78+
79+
# PTRACK BACKUP
80+
try:
81+
self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
82+
# we should die here because exception is what we expect to happen
83+
self.assertEqual(1, 0, "Expecting Error because ptrack_enable was set to OFF at some point after previous backup.\n Output: {0} \n CMD: {1}".format(
84+
repr(self.output), self.cmd))
85+
except ProbackupException as e:
86+
self.assertIn('ERROR: LSN from ptrack_control', e.message,
87+
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
88+
89+
# Clean after yourself
90+
self.del_test_dir(module_name, fname)
91+
92+
# @unittest.skip("skip")
4693
def test_ptrack_stream(self):
4794
"""make node, make full and ptrack stream backups, restore them and check data correctness"""
4895
self.maxDiff = None
@@ -99,7 +146,7 @@ def test_ptrack_stream(self):
99146
# Clean after yourself
100147
self.del_test_dir(module_name, fname)
101148

102-
#@unittest.skip("skip")
149+
# @unittest.skip("skip")
103150
def test_ptrack_archive(self):
104151
"""make archive node, make full and ptrack backups, check data correctness in restored instance"""
105152
self.maxDiff = None
@@ -157,7 +204,7 @@ def test_ptrack_archive(self):
157204
# Clean after yourself
158205
self.del_test_dir(module_name, fname)
159206

160-
#@unittest.skip("skip")
207+
# @unittest.skip("skip")
161208
def test_ptrack_pgpro417(self):
162209
"""Make node, take full backup, take ptrack backup, delete ptrack backup. Try to take ptrack backup, which should fail"""
163210
self.maxDiff = None
@@ -202,14 +249,13 @@ def test_ptrack_pgpro417(self):
202249
self.assertEqual(1, 0, "Expecting Error because of LSN mismatch from ptrack_control and previous backup start_lsn.\n Output: {0} \n CMD: {1}".format(
203250
repr(self.output), self.cmd))
204251
except ProbackupException as e:
205-
self.assertTrue('ERROR: LSN from ptrack_control' in e.message
206-
and 'differs from LSN of previous backup' in e.message,
252+
self.assertTrue('ERROR: LSN from ptrack_control' in e.message,
207253
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
208254

209255
# Clean after yourself
210256
self.del_test_dir(module_name, fname)
211257

212-
#@unittest.skip("skip")
258+
# @unittest.skip("skip")
213259
def test_page_pgpro417(self):
214260
"""Make archive node, take full backup, take page backup, delete page backup. Try to take ptrack backup, which should fail"""
215261
self.maxDiff = None
@@ -241,6 +287,7 @@ def test_page_pgpro417(self):
241287
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
242288

243289
self.delete_pb(backup_dir, 'node', backup_id)
290+
# sys.exit(1)
244291

245292
# PTRACK BACKUP
246293
node.safe_psql(
@@ -253,14 +300,13 @@ def test_page_pgpro417(self):
253300
self.assertEqual(1, 0, "Expecting Error because of LSN mismatch from ptrack_control and previous backup start_lsn.\n Output: {0} \n CMD: {1}".format(
254301
repr(self.output), self.cmd))
255302
except ProbackupException as e:
256-
self.assertTrue('ERROR: LSN from ptrack_control' in e.message
257-
and 'differs from LSN of previous backup' in e.message,
303+
self.assertTrue('ERROR: LSN from ptrack_control' in e.message,
258304
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
259305

260306
# Clean after yourself
261307
# self.del_test_dir(module_name, fname)
262308

263-
#@unittest.skip("skip")
309+
# @unittest.skip("skip")
264310
def test_full_pgpro417(self):
265311
"""Make node, take two full backups, delete full second backup. Try to take ptrack backup, which should fail"""
266312
self.maxDiff = None
@@ -303,14 +349,13 @@ def test_full_pgpro417(self):
303349
repr(self.output), self.cmd))
304350
except ProbackupException as e:
305351
self.assertTrue('ERROR: LSN from ptrack_control' in e.message
306-
and 'differs from LSN of previous backup' in e.message
307352
and 'Create new full backup before an incremental one' in e.message,
308353
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
309354

310355
# Clean after yourself
311356
self.del_test_dir(module_name, fname)
312357

313-
#@unittest.skip("skip")
358+
# @unittest.skip("skip")
314359
def test_create_db(self):
315360
"""Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense"""
316361
self.maxDiff = None
@@ -319,11 +364,12 @@ def test_create_db(self):
319364
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
320365
set_replication=True,
321366
initdb_params=['--data-checksums'],
322-
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
367+
pg_options={'wal_level': 'replica', 'max_wal_size': '10GB', 'max_wal_senders': '2', 'checkpoint_timeout': '5min', 'ptrack_enable': 'on', 'autovacuum': 'off'}
323368
)
324369

325370
self.init_pb(backup_dir)
326371
self.add_instance(backup_dir, 'node', node)
372+
self.set_archiving(backup_dir, 'node', node)
327373
node.start()
328374

329375
# FULL BACKUP
@@ -338,41 +384,39 @@ def test_create_db(self):
338384

339385
# PTRACK BACKUP
340386
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
341-
pgdata_content = self.pgdata_content(node.data_dir)
387+
pgdata = self.pgdata_content(node.data_dir)
342388

343389
# RESTORE
344390
node_restored = self.make_simple_node(base_dir="{0}/{1}/node_restored".format(module_name, fname))
345391
node_restored.cleanup()
346392
# COMPARE PHYSICAL CONTENT
347393
self.restore_node(backup_dir, 'node', node_restored, backup_id=backup_id, options=["-j", "4"])
348-
pgdata_content_new = self.pgdata_content(node_restored.data_dir)
349-
self.compare_pgdata(pgdata_content, pgdata_content_new)
394+
pgdata_new = self.pgdata_content(node_restored.data_dir)
395+
self.compare_pgdata(pgdata, pgdata_new)
350396

351397
# START RESTORED NODE
352398
node_restored.append_conf("postgresql.auto.conf", "port = {0}".format(node_restored.port))
353399
node_restored.start()
354-
result_new = node_restored.safe_psql("postgres", "select * from pg_class")
355-
356400

357401
# DROP DATABASE DB1
358402
node.safe_psql(
359403
"postgres", "drop database db1")
360404
# SECOND PTRACK BACKUP
361405
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
362-
pgdata_content = self.pgdata_content(node.data_dir)
406+
pgdata = self.pgdata_content(node.data_dir)
363407

364408
# RESTORE SECOND PTRACK BACKUP
365409
node_restored.cleanup()
366410
self.restore_node(backup_dir, 'node', node_restored, backup_id=backup_id, options=["-j", "4"])
367-
# COMPARE PHYSICAL CONTENT
368-
pgdata_content_new = self.pgdata_content(node_restored.data_dir)
369-
self.compare_pgdata(pgdata_content, pgdata_content_new)
370-
371411

372412
# START RESTORED NODE
373413
node_restored.append_conf("postgresql.auto.conf", "port = {0}".format(node_restored.port))
374414
node_restored.start()
375415

416+
# COMPARE PHYSICAL CONTENT
417+
pgdata_new = self.pgdata_content(node_restored.data_dir)
418+
self.compare_pgdata(pgdata, pgdata_new)
419+
376420
try:
377421
node_restored.safe_psql('db1', 'select 1')
378422
# we should die here because exception is what we expect to happen
@@ -385,7 +429,7 @@ def test_create_db(self):
385429
# Clean after yourself
386430
self.del_test_dir(module_name, fname)
387431

388-
#@unittest.skip("skip")
432+
# @unittest.skip("skip")
389433
def test_alter_table_set_tablespace_ptrack(self):
390434
"""Make node, create tablespace with table, take full backup, alter tablespace location, take ptrack backup, restore database."""
391435
self.maxDiff = None
@@ -416,7 +460,6 @@ def test_alter_table_set_tablespace_ptrack(self):
416460

417461
# PTRACK BACKUP
418462
result = node.safe_psql("postgres", "select * from t_heap")
419-
node.safe_psql("postgres", "select * from pg_class; checkpoint")
420463
self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
421464
pgdata_content = self.pgdata_content(node.data_dir)
422465
node.stop()
@@ -442,7 +485,7 @@ def test_alter_table_set_tablespace_ptrack(self):
442485
# Clean after yourself
443486
self.del_test_dir(module_name, fname)
444487

445-
#@unittest.skip("skip")
488+
# @unittest.skip("skip")
446489
def test_alter_database_set_tablespace_ptrack(self):
447490
"""Make node, create tablespace with database, take full backup, alter tablespace location, take ptrack backup, restore database."""
448491
self.maxDiff = None
@@ -470,7 +513,7 @@ def test_alter_database_set_tablespace_ptrack(self):
470513

471514
# PTRACK BACKUP
472515
self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
473-
pgdata_content = self.pgdata_content(node.data_dir)
516+
pgdata= self.pgdata_content(node.data_dir)
474517
node.stop()
475518

476519
# RESTORE
@@ -480,16 +523,16 @@ def test_alter_database_set_tablespace_ptrack(self):
480523
"-T", "{0}={1}".format(self.get_tblspace_path(node,'somedata'), self.get_tblspace_path(node_restored,'somedata'))])
481524

482525
# GET PHYSICAL CONTENT
483-
pgdata_content_new = self.pgdata_content(node_restored.data_dir)
526+
pgdata_new = self.pgdata_content(node_restored.data_dir)
484527
# COMPARE PHYSICAL CONTENT
485-
self.compare_pgdata(pgdata_content, pgdata_content_new)
528+
self.compare_pgdata(pgdata, pgdata_new)
486529
# START RESTORED NODE
487530
node_restored.start()
488531

489532
# Clean after yourself
490533
self.del_test_dir(module_name, fname)
491534

492-
#@unittest.skip("skip")
535+
# @unittest.skip("skip")
493536
def test_drop_tablespace(self):
494537
"""Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup"""
495538
self.maxDiff = None
@@ -549,7 +592,7 @@ def test_drop_tablespace(self):
549592
# Clean after yourself
550593
self.del_test_dir(module_name, fname)
551594

552-
#@unittest.skip("skip")
595+
# @unittest.skip("skip")
553596
def test_alter_tablespace(self):
554597
"""Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup"""
555598
self.maxDiff = None
@@ -606,17 +649,17 @@ def test_alter_tablespace(self):
606649
node.safe_psql(
607650
"postgres", "alter table t_heap set tablespace pg_default")
608651
# SECOND PTRACK BACKUP
609-
node.safe_psql("template1", "checkpoint")
610652
self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
611-
pgdata_content = self.pgdata_content(node.data_dir)
612653

613654
# Restore second ptrack backup and check table consistency
614655
self.restore_node(backup_dir, 'node', restored_node, options=[
615656
"-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)])
616-
pgdata_content_new = self.pgdata_content(restored_node.data_dir)
617-
self.compare_pgdata(pgdata_content, pgdata_content_new)
618657
restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port))
619658
restored_node.start()
659+
# COMPARE PHYSICAL CONTENT
660+
#pgdata = self.pgdata_content(node.data_dir)
661+
#pgdata_new = self.pgdata_content(restored_node.data_dir)
662+
#self.compare_pgdata(pgdata, pgdata_new)
620663
result_new = restored_node.safe_psql("postgres", "select * from t_heap")
621664
self.assertEqual(result, result_new)
622665

@@ -639,11 +682,10 @@ def test_relation_with_multiple_segments(self):
639682
self.add_instance(backup_dir, 'node', node)
640683
node.start()
641684

642-
# self.create_tblspace_in_node(node, 'somedata')
685+
self.create_tblspace_in_node(node, 'somedata')
643686

644687
# CREATE TABLE
645-
# node.pgbench_init(scale=300, options=['--tablespace=somedata'])
646-
pgbench = node.pgbench_init(scale=30)
688+
node.pgbench_init(scale=300, options=['--tablespace=somedata'])
647689
# FULL BACKUP
648690
self.backup_node(backup_dir, 'node', node, options=["--stream"])
649691

@@ -657,24 +699,25 @@ def test_relation_with_multiple_segments(self):
657699
pgdata = self.pgdata_content(node.data_dir)
658700

659701
# RESTORE NODE
660-
# self.restore_node(backup_dir, 'node', restored_node, options=[
661-
# "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)])
662702
restored_node = self.make_simple_node(base_dir="{0}/{1}/restored_node".format(module_name, fname))
663703
restored_node.cleanup()
664-
# tblspc_path = self.get_tblspace_path(node, 'somedata')
665-
# tblspc_path_new = self.get_tblspace_path(restored_node, 'somedata_restored')
704+
tblspc_path = self.get_tblspace_path(node, 'somedata')
705+
tblspc_path_new = self.get_tblspace_path(restored_node, 'somedata_restored')
666706

667707
self.restore_node(backup_dir, 'node', restored_node, options=[
668-
"-j", "4"])
708+
"-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)])
709+
# self.restore_node(backup_dir, 'node', restored_node, options=[
710+
# "-j", "4"])
669711
# GET PHYSICAL CONTENT FROM NODE_RESTORED
670712
pgdata_restored = self.pgdata_content(restored_node.data_dir)
671-
self.compare_pgdata(pgdata, pgdata_restored)
672713

714+
# START RESTORED NODE
673715
restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port))
674716
restored_node.start()
675717

676718
result_new = restored_node.safe_psql("postgres", "select * from pgbench_accounts")
677719
self.assertEqual(result, result_new)
720+
self.compare_pgdata(pgdata, pgdata_restored)
678721

679722
# Clean after yourself
680723
self.del_test_dir(module_name, fname)

0 commit comments

Comments
 (0)