Skip to content

Commit 40118ca

Browse files
gsmolkgsmolk
committed
cfs tests small temporal chages
1 parent 9a1e6ee commit 40118ca

File tree

4 files changed

+141
-78
lines changed

4 files changed

+141
-78
lines changed

‎tests/__init__.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \
77
ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \
88
ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \
9-
false_positive, replica, compression, page, ptrack, archive
9+
false_positive, replica, compression, page, ptrack, archive, \
10+
cfs_backup, cfs_restore, cfs_validate_backup
1011

1112

1213
def load_tests(loader, tests, pattern):
@@ -36,6 +37,9 @@ def load_tests(loader, tests, pattern):
3637
suite.addTests(loader.loadTestsFromModule(compression))
3738
suite.addTests(loader.loadTestsFromModule(page))
3839
suite.addTests(loader.loadTestsFromModule(archive))
40+
suite.addTests(loader.loadTestsFromModule(cfs_backup))
41+
suite.addTests(loader.loadTestsFromModule(cfs_restore))
42+
suite.addTests(loader.loadTestsFromModule(cfs_validate_backup))
3943

4044
return suite
4145

‎tests/cfs_backup.py

Lines changed: 109 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import os
22
import unittest
33
import random
4+
import shutil
45

56
from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file
67
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
@@ -21,9 +22,10 @@ def setUp(self):
2122
initdb_params=['--data-checksums'],
2223
pg_options={
2324
'wal_level': 'replica',
24-
'ptrack_enable': 'on',
25+
# 'ptrack_enable': 'on',
2526
'cfs_encryption': 'off',
26-
'max_wal_senders': '2'
27+
'max_wal_senders': '2',
28+
'shared_buffers': '200MB'
2729
}
2830
)
2931

@@ -34,7 +36,7 @@ def setUp(self):
3436
self.node.start()
3537

3638
self.create_tblspace_in_node(self.node, tblspace_name, True)
37-
39+
3840
tblspace = self.node.safe_psql(
3941
"postgres",
4042
"SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name)
@@ -95,6 +97,7 @@ def test_fullbackup_empty_tablespace_stream(self):
9597
repr(e.message)
9698
)
9799
)
100+
98101
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
99102
self.assertEqual(
100103
"OK",
@@ -105,10 +108,6 @@ def test_fullbackup_empty_tablespace_stream(self):
105108
find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']),
106109
"ERROR: File pg_compression not found in backup dir"
107110
)
108-
self.assertFalse(
109-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']),
110-
"ERROR: _ptrack files found in backup dir"
111-
)
112111

113112
# @unittest.expectedFailure
114113
# @unittest.skip("skip")
@@ -131,11 +130,12 @@ def test_fullbackup_after_create_table(self):
131130
backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full')
132131
except ProbackupException as e:
133132
self.fail(
134-
"ERROR: Full backup failed.\n {0} \n {0}".format(
135-
repr(self.cmd),
136-
repr(e.message)
133+
"\n ERROR: {0}\n CMD: {1}".format(
134+
repr(e.message),
135+
repr(self.cmd)
137136
)
138137
)
138+
return False
139139
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
140140
self.assertEqual(
141141
"OK",
@@ -147,15 +147,11 @@ def test_fullbackup_after_create_table(self):
147147
"ERROR: File pg_compression not found in {0}".format(os.path.join(self.backup_dir, 'node', backup_id))
148148
)
149149
self.assertTrue(
150-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']),
150+
find_by_extensions([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['.cfm']),
151151
"ERROR: .cfm files not found in backup dir"
152152
)
153-
self.assertFalse(
154-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']),
155-
"ERROR: _ptrack files was found in backup dir"
156-
)
157153

158-
# @unittest.expectedFailure
154+
@unittest.expectedFailure
159155
# @unittest.skip("skip")
160156
# PGPRO-1018 invalid file size
161157
def test_fullbackup_after_create_table_stream(self):
@@ -195,13 +191,9 @@ def test_fullbackup_after_create_table_stream(self):
195191
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']),
196192
"ERROR: .cfm files not found in backup dir"
197193
)
198-
self.assertFalse(
199-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']),
200-
"ERROR: _ptrack files was found in backup dir"
201-
)
202194

203195
# --- Section: Incremental from empty tablespace --- #
204-
# @unittest.expectedFailure
196+
@unittest.expectedFailure
205197
# @unittest.skip("skip")
206198
def test_fullbackup_empty_tablespace_ptrack_after_create_table(self):
207199
"""
@@ -251,12 +243,8 @@ def test_fullbackup_empty_tablespace_ptrack_after_create_table(self):
251243
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']),
252244
"ERROR: .cfm files not found in backup dir"
253245
)
254-
self.assertFalse(
255-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']),
256-
"ERROR: _ptrack files was found in backup dir"
257-
)
258246

259-
# @unittest.expectedFailure
247+
@unittest.expectedFailure
260248
# @unittest.skip("skip")
261249
def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self):
262250
"""
@@ -307,7 +295,7 @@ def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self):
307295
"ERROR: .cfm files not found in backup dir"
308296
)
309297
self.assertFalse(
310-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']),
298+
find_by_extensions([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['_ptrack']),
311299
"ERROR: _ptrack files was found in backup dir"
312300
)
313301

@@ -358,13 +346,9 @@ def test_fullbackup_empty_tablespace_page_after_create_table(self):
358346
"ERROR: File pg_compression not found"
359347
)
360348
self.assertTrue(
361-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']),
349+
find_by_extensions([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['.cfm']),
362350
"ERROR: .cfm files not found in backup dir"
363351
)
364-
self.assertFalse(
365-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']),
366-
"ERROR: _ptrack files was found in backup dir"
367-
)
368352

369353
# @unittest.expectedFailure
370354
# @unittest.skip("skip")
@@ -413,16 +397,16 @@ def test_fullbackup_empty_tablespace_page_after_create_table_stream(self):
413397
"ERROR: File pg_compression not found"
414398
)
415399
self.assertTrue(
416-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']),
400+
find_by_extensions([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['.cfm']),
417401
"ERROR: .cfm files not found in backup dir"
418402
)
419403
self.assertFalse(
420-
find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']),
404+
find_by_extensions([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['_ptrack']),
421405
"ERROR: _ptrack files was found in backup dir"
422406
)
423407

424408
# --- Section: Incremental from fill tablespace --- #
425-
# @unittest.expectedFailure
409+
@unittest.expectedFailure
426410
# @unittest.skip("skip")
427411
def test_fullbackup_after_create_table_ptrack_after_create_table(self):
428412
"""
@@ -480,7 +464,7 @@ def test_fullbackup_after_create_table_ptrack_after_create_table(self):
480464
)
481465
)
482466

483-
# @unittest.expectedFailure
467+
@unittest.expectedFailure
484468
# @unittest.skip("skip")
485469
def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self):
486470
"""
@@ -596,6 +580,83 @@ def test_fullbackup_after_create_table_page_after_create_table(self):
596580
)
597581
)
598582

583+
# @unittest.expectedFailure
584+
# @unittest.skip("skip")
585+
def test_multiple_segments(self):
586+
"""
587+
Case: Make full backup before created table in the tablespace.
588+
Make ptrack backup after create table.
589+
Check: incremental backup will not greater as full
590+
"""
591+
592+
self.node.safe_psql(
593+
"postgres",
594+
"CREATE TABLE {0} TABLESPACE {1} \
595+
AS SELECT i AS id, MD5(i::text) AS text, \
596+
MD5(repeat(i::text,10))::tsvector AS tsvector \
597+
FROM generate_series(0,10050000) i".format('t_heap', tblspace_name)
598+
)
599+
600+
full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap")
601+
602+
try:
603+
backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full')
604+
except ProbackupException as e:
605+
self.fail(
606+
"ERROR: Full backup failed.\n {0} \n {1}".format(
607+
repr(self.cmd),
608+
repr(e.message)
609+
)
610+
)
611+
612+
self.node.safe_psql(
613+
"postgres",
614+
'INSERT INTO {0} \
615+
SELECT i AS id, MD5(i::text) AS text, \
616+
MD5(repeat(i::text,10))::tsvector AS tsvector \
617+
FROM generate_series(0,10050000) i'.format('t_heap')
618+
)
619+
620+
page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap")
621+
622+
try:
623+
backup_id_page = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page')
624+
except ProbackupException as e:
625+
self.fail(
626+
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
627+
repr(self.cmd),
628+
repr(e.message)
629+
)
630+
)
631+
632+
show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full)
633+
show_backup_page = self.show_pb(self.backup_dir, 'node', backup_id_page)
634+
self.assertGreater(
635+
show_backup_page["data-bytes"],
636+
show_backup_full["data-bytes"],
637+
"ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format(
638+
show_backup_page["data-bytes"],
639+
show_backup_full["data-bytes"]
640+
)
641+
)
642+
643+
# CHECK FULL BACKUP
644+
self.node.stop()
645+
self.node.cleanup()
646+
shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name), ignore_errors=True)
647+
self.restore_node(self.backup_dir, 'node', self.node, backup_id=backup_id_full, options=["-j", "4"])
648+
self.node.start()
649+
self.assertEqual(full_result, self.node.safe_psql("postgres", "SELECT * FROM t_heap"), 'Lost data after restore')
650+
651+
# CHECK PAGE BACKUP
652+
self.node.stop()
653+
self.node.cleanup()
654+
shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name), ignore_errors=True)
655+
self.restore_node(self.backup_dir, 'node', self.node, backup_id=backup_id_page, options=["-j", "4"])
656+
self.node.start()
657+
self.assertEqual(page_result, self.node.safe_psql("postgres", "SELECT * FROM t_heap"), 'Lost data after restore')
658+
659+
599660
# @unittest.expectedFailure
600661
# @unittest.skip("skip")
601662
def test_fullbackup_after_create_table_page_after_create_table_stream(self):
@@ -655,7 +716,7 @@ def test_fullbackup_after_create_table_page_after_create_table_stream(self):
655716
)
656717

657718
# --- Make backup with not valid data(broken .cfm) --- #
658-
# @unittest.expectedFailure
719+
@unittest.expectedFailure
659720
# @unittest.skip("skip")
660721
def test_delete_random_cfm_file_from_tablespace_dir(self):
661722
self.node.safe_psql(
@@ -679,7 +740,7 @@ def test_delete_random_cfm_file_from_tablespace_dir(self):
679740
self.backup_node,self.backup_dir, 'node', self.node, backup_type='full'
680741
)
681742

682-
# @unittest.expectedFailure
743+
@unittest.expectedFailure
683744
# @unittest.skip("skip")
684745
def test_delete_file_pg_compression_from_tablespace_dir(self):
685746
os.remove(find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression'])[0])
@@ -689,7 +750,7 @@ def test_delete_file_pg_compression_from_tablespace_dir(self):
689750
self.backup_node,self.backup_dir, 'node', self.node, backup_type='full'
690751
)
691752

692-
# @unittest.expectedFailure
753+
@unittest.expectedFailure
693754
# @unittest.skip("skip")
694755
def test_delete_random_data_file_from_tablespace_dir(self):
695756
self.node.safe_psql(
@@ -713,7 +774,7 @@ def test_delete_random_data_file_from_tablespace_dir(self):
713774
self.backup_node,self.backup_dir, 'node', self.node, backup_type='full'
714775
)
715776

716-
# @unittest.expectedFailure
777+
@unittest.expectedFailure
717778
# @unittest.skip("skip")
718779
def test_broken_random_cfm_file_into_tablespace_dir(self):
719780
self.node.safe_psql(
@@ -724,7 +785,7 @@ def test_broken_random_cfm_file_into_tablespace_dir(self):
724785
FROM generate_series(0,256) i'.format('t1', tblspace_name)
725786
)
726787

727-
list_cmf = find_by_extensions([self.get_tblspace_path(self.node,tblspace_name)],['.cfm'])
788+
list_cmf = find_by_extensions([self.get_tblspace_path(self.node, tblspace_name)],['.cfm'])
728789
self.assertTrue(
729790
list_cmf,
730791
"ERROR: .cfm-files not found into tablespace dir"
@@ -737,7 +798,7 @@ def test_broken_random_cfm_file_into_tablespace_dir(self):
737798
self.backup_node,self.backup_dir, 'node', self.node, backup_type='full'
738799
)
739800

740-
# @unittest.expectedFailure
801+
@unittest.expectedFailure
741802
# @unittest.skip("skip")
742803
def test_broken_random_data_file_into_tablespace_dir(self):
743804
self.node.safe_psql(
@@ -761,7 +822,7 @@ def test_broken_random_data_file_into_tablespace_dir(self):
761822
self.backup_node,self.backup_dir, 'node', self.node, backup_type='full'
762823
)
763824

764-
# @unittest.expectedFailure
825+
@unittest.expectedFailure
765826
# @unittest.skip("skip")
766827
def test_broken_file_pg_compression_into_tablespace_dir(self):
767828

@@ -783,8 +844,8 @@ def tearDown(self):
783844
self.del_test_dir(module_name, self.fname)
784845

785846

786-
class CfsBackupEncTest(CfsBackupNoEncTest):
787-
# --- Begin --- #
788-
def setUp(self):
789-
os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key"
790-
super(CfsBackupEncTest, self).setUp()
847+
#class CfsBackupEncTest(CfsBackupNoEncTest):
848+
# # --- Begin --- #
849+
# def setUp(self):
850+
# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key"
851+
# super(CfsBackupEncTest, self).setUp()

0 commit comments

Comments
 (0)