Skip to content

Commit fa1ba6c

Browse files
gsmolkgsmolk
committed
pgpro561 added
1 parent 442cef1 commit fa1ba6c

20 files changed

+161
-111
lines changed

‎tests/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@
55
retention_test, ptrack_clean, ptrack_cluster, \
66
ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \
77
ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \
8-
ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro668
8+
ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro561, pgpro668
99

1010

1111
def load_tests(loader, tests, pattern):
1212
suite = unittest.TestSuite()
13+
suite.addTests(loader.loadTestsFromModule(pgpro561))
1314
suite.addTests(loader.loadTestsFromModule(pgpro668))
1415
suite.addTests(loader.loadTestsFromModule(init_test))
1516
suite.addTests(loader.loadTestsFromModule(option_test))

‎tests/backup_test.py

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ def __init__(self, *args, **kwargs):
1717
def test_backup_modes_archive(self):
1818
"""standart backup modes with ARCHIVE WAL method"""
1919
fname = self.id().split('.')[3]
20-
print '{0} started'.format(fname)
2120
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
2221
set_archiving=True,
2322
initdb_params=['--data-checksums'],
@@ -74,7 +73,6 @@ def test_backup_modes_archive(self):
7473
def test_smooth_checkpoint(self):
7574
"""full backup with smooth checkpoint"""
7675
fname = self.id().split('.')[3]
77-
print '{0} started'.format(fname)
7876
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
7977
set_archiving=True,
8078
initdb_params=['--data-checksums'],
@@ -94,7 +92,6 @@ def test_smooth_checkpoint(self):
9492
def test_page_backup_without_full(self):
9593
"""page-level backup without validated full backup"""
9694
fname = self.id().split('.')[3]
97-
print '{0} started'.format(fname)
9895
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
9996
set_archiving=True,
10097
initdb_params=['--data-checksums'],
@@ -115,7 +112,6 @@ def test_page_backup_without_full(self):
115112
def test_ptrack_threads(self):
116113
"""ptrack multi thread backup mode"""
117114
fname = self.id().split('.')[3]
118-
print '{0} started'.format(fname)
119115
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
120116
set_archiving=True,
121117
initdb_params=['--data-checksums'],
@@ -140,7 +136,6 @@ def test_ptrack_threads(self):
140136
def test_ptrack_threads_stream(self):
141137
"""ptrack multi thread backup mode and stream"""
142138
fname = self.id().split('.')[3]
143-
print '{0} started'.format(fname)
144139
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
145140
set_replication=True,
146141
initdb_params=['--data-checksums'],

‎tests/delete_test.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ def __init__(self, *args, **kwargs):
1818
def test_delete_full_backups(self):
1919
"""delete full backups"""
2020
fname = self.id().split('.')[3]
21-
print '{0} started'.format(fname)
2221
node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname),
2322
set_archiving=True,
2423
initdb_params=['--data-checksums'],
@@ -60,7 +59,6 @@ def test_delete_full_backups(self):
6059
def test_delete_increment(self):
6160
"""delete increment and all after him"""
6261
fname = self.id().split('.')[3]
63-
print '{0} started'.format(fname)
6462
node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname),
6563
set_archiving=True,
6664
initdb_params=['--data-checksums'],

‎tests/init_test.py

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ def __init__(self, *args, **kwargs):
1515
def test_success_1(self):
1616
"""Success normal init"""
1717
fname = self.id().split(".")[3]
18-
print '{0} started'.format(fname)
1918
node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname))
2019
self.assertEqual(self.init_pb(node), six.b(""))
2120
self.assertEqual(
@@ -26,7 +25,6 @@ def test_success_1(self):
2625
def test_already_exist_2(self):
2726
"""Failure with backup catalog already existed"""
2827
fname = self.id().split(".")[3]
29-
print '{0} started'.format(fname)
3028
node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname))
3129
self.init_pb(node)
3230
try:
@@ -42,7 +40,6 @@ def test_already_exist_2(self):
4240
def test_abs_path_3(self):
4341
"""failure with backup catalog should be given as absolute path"""
4442
fname = self.id().split(".")[3]
45-
print '{0} started'.format(fname)
4643
node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname))
4744
try:
4845
self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)])

‎tests/option_test.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ def tearDownClass(cls):
1717
def test_help_1(self):
1818
"""help options"""
1919
fname = self.id().split(".")[3]
20-
print '{0} started'.format(fname)
2120
with open(path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out:
2221
self.assertEqual(
2322
self.run_pb(["--help"]),
@@ -27,7 +26,6 @@ def test_help_1(self):
2726
def test_version_2(self):
2827
"""help options"""
2928
fname = self.id().split(".")[3]
30-
print '{0} started'.format(fname)
3129
with open(path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out:
3230
self.assertEqual(
3331
self.run_pb(["--version"]),
@@ -37,7 +35,6 @@ def test_version_2(self):
3735
def test_without_backup_path_3(self):
3836
"""backup command failure without backup mode option"""
3937
fname = self.id().split(".")[3]
40-
print '{0} started'.format(fname)
4138
try:
4239
self.run_pb(["backup", "-b", "full"])
4340
# we should die here because exception is what we expect to happen
@@ -51,7 +48,6 @@ def test_without_backup_path_3(self):
5148
def test_options_4(self):
5249
"""check options test"""
5350
fname = self.id().split(".")[3]
54-
print '{0} started'.format(fname)
5551
node = self.make_simple_node(base_dir="tmp_dirs/option/{0}".format(fname))
5652
try:
5753
node.stop()

‎tests/pgpro561.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import unittest
2+
import os
3+
import six
4+
from .ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
5+
from datetime import datetime, timedelta
6+
from testgres import stop_all
7+
import subprocess
8+
from sys import exit
9+
10+
11+
class SomeTest(ProbackupTest, unittest.TestCase):
12+
13+
def __init__(self, *args, **kwargs):
14+
super(SomeTest, self).__init__(*args, **kwargs)
15+
16+
# @classmethod
17+
# def tearDownClass(cls):
18+
# stop_all()
19+
20+
def test_pgpro561(self):
21+
"""
22+
make node with archiving, make stream backup,
23+
get Recovery Time, try to make pitr to Recovery Time
24+
"""
25+
fname = self.id().split('.')[3]
26+
master = self.make_simple_node(base_dir="tmp_dirs/pgpro561/{0}/master".format(fname),
27+
set_archiving=True,
28+
set_replication=True,
29+
initdb_params=['--data-checksums'],
30+
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
31+
)
32+
master.start()
33+
34+
node1 = self.make_simple_node(base_dir="tmp_dirs/pgpro561/{0}/node1".format(fname))
35+
node2 = self.make_simple_node(base_dir="tmp_dirs/pgpro561/{0}/node2".format(fname))
36+
node1.cleanup()
37+
node2.cleanup()
38+
39+
self.assertEqual(self.init_pb(master), six.b(""))
40+
self.backup_pb(master, backup_type='full')
41+
42+
master.psql(
43+
"postgres",
44+
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
45+
# for i in idx_ptrack:
46+
# if idx_ptrack[i]['type'] == 'heap':
47+
# continue
48+
# master.psql("postgres", "create index {0} on {1} using {2}({3})".format(
49+
# i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
50+
#before = master.execute("postgres", "SELECT * FROM t_heap")
51+
52+
id = self.backup_pb(master, backup_type='page')
53+
self.restore_pb(backup_dir=self.backup_dir(master), data_dir=node1.data_dir)
54+
node1.append_conf('postgresql.auto.conf', 'port = {0}'.format(node1.port))
55+
56+
self.restore_pb(backup_dir=self.backup_dir(master), data_dir=node2.data_dir)
57+
node2.append_conf('postgresql.auto.conf', 'port = {0}'.format(node2.port))
58+
59+
node1.start({"-t": "600"})
60+
node2.start({"-t": "600"})
61+
62+
timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"]
63+
timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"]
64+
self.assertEqual(timeline_node1, timeline_node2,
65+
"Node1 and Node2 timelines are different.\nWhich means that Node2 applied wals archived by Master AND Node1.\nWhich means that Master and Node1 have common archive.\nTHIS IS BAD\nCheck archive directory in {0}".format(os.path.join(self.backup_dir(master), "wal")))
66+
# node1.pgbench_init(scale=5)
67+
# node2.pgbench_init(scale=5)

‎tests/pgpro668.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self):
2323
get Recovery Time, try to make pitr to Recovery Time
2424
"""
2525
fname = self.id().split('.')[3]
26-
print '{0} started'.format(fname)
27-
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
26+
node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname),
2827
set_archiving=True,
2928
set_replication=True,
3029
initdb_params=['--data-checksums'],
@@ -50,8 +49,7 @@ def test_validate_to_recovery_time(self):
5049
Should fail. Waiting PGPRO-688
5150
"""
5251
fname = self.id().split('.')[3]
53-
print '{0} started'.format(fname)
54-
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
52+
node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname),
5553
set_archiving=True,
5654
set_replication=True,
5755
initdb_params=['--data-checksums'],
@@ -84,8 +82,7 @@ def test_archive_node_backup_stream_additional_commit_pitr(self):
8482
try to make pitr to Recovery Time, check that t_heap do not exists
8583
"""
8684
fname = self.id().split('.')[3]
87-
print '{0} started'.format(fname)
88-
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
85+
node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname),
8986
set_archiving=True,
9087
set_replication=True,
9188
initdb_params=['--data-checksums'],

‎tests/ptrack_cluster.py

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ def teardown(self):
1515
# @unittest.skip("123")
1616
def test_ptrack_cluster_btree(self):
1717
fname = self.id().split('.')[3]
18-
print '{0} started'.format(fname)
1918
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
2019
set_replication=True,
2120
initdb_params=['--data-checksums', '-A trust'],
@@ -71,10 +70,8 @@ def test_ptrack_cluster_btree(self):
7170
self.clean_pb(node)
7271
node.stop()
7372

74-
@unittest.skip("123")
7573
def test_ptrack_cluster_spgist(self):
7674
fname = self.id().split('.')[3]
77-
print '{0} started'.format(fname)
7875
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
7976
set_replication=True,
8077
initdb_params=['--data-checksums', '-A trust'],
@@ -130,10 +127,8 @@ def test_ptrack_cluster_spgist(self):
130127
self.clean_pb(node)
131128
node.stop()
132129

133-
@unittest.skip("123")
134130
def test_ptrack_cluster_brin(self):
135131
fname = self.id().split('.')[3]
136-
print '{0} started'.format(fname)
137132
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
138133
set_replication=True,
139134
initdb_params=['--data-checksums', '-A trust'],
@@ -189,10 +184,8 @@ def test_ptrack_cluster_brin(self):
189184
self.clean_pb(node)
190185
node.stop()
191186

192-
@unittest.skip("123")
193187
def test_ptrack_cluster_gist(self):
194188
fname = self.id().split('.')[3]
195-
print '{0} started'.format(fname)
196189
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
197190
set_replication=True,
198191
initdb_params=['--data-checksums', '-A trust'],
@@ -248,10 +241,8 @@ def test_ptrack_cluster_gist(self):
248241
self.clean_pb(node)
249242
node.stop()
250243

251-
@unittest.skip("123")
252244
def test_ptrack_cluster_gin(self):
253245
fname = self.id().split('.')[3]
254-
print '{0} started'.format(fname)
255246
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
256247
set_replication=True,
257248
initdb_params=['--data-checksums', '-A trust'],

‎tests/ptrack_helpers.py

Lines changed: 15 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -152,29 +152,6 @@ def arcwal_dir(self, node):
152152
def backup_dir(self, node):
153153
return os.path.abspath("%s/backup" % node.base_dir)
154154

155-
# def make_bnode(self, base_dir=None, allows_streaming=False, options={}):
156-
# real_base_dir = os.path.join(self.dir_path, base_dir)
157-
# shutil.rmtree(real_base_dir, ignore_errors=True)
158-
#
159-
# node = get_new_node('test', base_dir=real_base_dir)
160-
# node.init(allows_streaming=allows_streaming)
161-
#
162-
# if not allows_streaming:
163-
# node.append_conf("postgresql.auto.conf", "wal_level = hot_standby")
164-
# node.append_conf("postgresql.auto.conf", "archive_mode = on")
165-
# node.append_conf(
166-
# "postgresql.auto.conf",
167-
# """archive_command = 'cp "%%p" "%s/%%f"'""" % os.path.abspath(self.arcwal_dir(node))
168-
# )
169-
#
170-
# for key, value in six.iteritems(options):
171-
# node.append_conf("postgresql.conf", "%s = %s" % (key, value))
172-
#
173-
# return node
174-
175-
# def print_started(self, fname):
176-
# print
177-
178155
def make_simple_node(self, base_dir=None, set_replication=False,
179156
set_archiving=False, initdb_params=[], pg_options={}):
180157
real_base_dir = os.path.join(self.dir_path, base_dir)
@@ -199,7 +176,6 @@ def make_simple_node(self, base_dir=None, set_replication=False,
199176
self.set_archiving_conf(node, self.arcwal_dir(node))
200177
return node
201178

202-
203179
def create_tblspace_in_node(self, node, tblspc_name, cfs=False):
204180
res = node.execute(
205181
"postgres", "select exists (select 1 from pg_tablespace where spcname = '{0}')".format(
@@ -346,11 +322,16 @@ def init_pb(self, node):
346322
def clean_pb(self, node):
347323
shutil.rmtree(self.backup_dir(node), ignore_errors=True)
348324

349-
def backup_pb(self, node, backup_type="full", options=[]):
325+
def backup_pb(self, node=None, data_dir=None, backup_dir=None, backup_type="full", options=[]):
326+
if data_dir is None:
327+
data_dir = node.data_dir
328+
if backup_dir is None:
329+
backup_dir = self.backup_dir(node)
330+
350331
cmd_list = [
351332
"backup",
352-
"-D", node.data_dir,
353-
"-B", self.backup_dir(node),
333+
"-B", backup_dir,
334+
"-D", data_dir,
354335
"-p", "%i" % node.port,
355336
"-d", "postgres"
356337
]
@@ -359,37 +340,20 @@ def backup_pb(self, node, backup_type="full", options=[]):
359340

360341
return self.run_pb(cmd_list + options)
361342

362-
def backup_pb_proc(self, node, backup_type="full",
363-
stdout=None, stderr=None, options=[]):
364-
cmd_list = [
365-
self.probackup_path,
366-
"backup",
367-
"-D", node.data_dir,
368-
"-B", self.backup_dir(node),
369-
"-p", "%i" % (node.port),
370-
"-d", "postgres"
371-
]
372-
if backup_type:
373-
cmd_list += ["-b", backup_type]
374-
375-
proc = subprocess.Popen(
376-
cmd_list + options,
377-
stdout=stdout,
378-
stderr=stderr
379-
)
380-
381-
return proc
343+
def restore_pb(self, node=None, backup_dir=None, data_dir=None, id=None, options=[]):
344+
if data_dir is None:
345+
data_dir = node.data_dir
346+
if backup_dir is None:
347+
backup_dir = self.backup_dir(node)
382348

383-
def restore_pb(self, node, id=None, options=[]):
384349
cmd_list = [
385350
"restore",
386-
"-D", node.data_dir,
387-
"-B", self.backup_dir(node)
351+
"-B", backup_dir,
352+
"-D", data_dir
388353
]
389354
if id:
390355
cmd_list += ["-i", id]
391356

392-
# print(cmd_list)
393357
return self.run_pb(cmd_list + options)
394358

395359
def show_pb(self, node, id=None, options=[], as_text=False):

‎tests/ptrack_move_to_tablespace.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ def teardown(self):
1717

1818
def test_ptrack_recovery(self):
1919
fname = self.id().split(".")[3]
20-
print '{0} started'.format(fname)
2120
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
2221
set_replication=True,
2322
initdb_params=['--data-checksums', '-A trust'],

‎tests/ptrack_recovery.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ def teardown(self):
1717

1818
def test_ptrack_recovery(self):
1919
fname = self.id().split(".")[3]
20-
print '{0} started'.format(fname)
2120
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
2221
set_replication=True,
2322
initdb_params=['--data-checksums', '-A trust'],

0 commit comments

Comments
 (0)