Skip to content

Commit c9fc20b

Browse files
funny-falconfunny-falcon
committed
PBCKP-604: Allow partial incremental restore only with a flag --destroy-all-other-dbs
1 parent 0690f8d commit c9fc20b

File tree

7 files changed

+170
-9
lines changed

7 files changed

+170
-9
lines changed

‎src/help.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ help_pg_probackup(void)
175175
printf(_(" [-X WALDIR | --waldir=WALDIR]\n"));
176176
printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n"));
177177
printf(_(" [--db-include | --db-exclude]\n"));
178+
printf(_(" [--destroy-all-other-dbs]\n"));
178179
printf(_(" [--remote-proto] [--remote-host]\n"));
179180
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
180181
printf(_(" [--ssh-options]\n"));
@@ -450,6 +451,7 @@ help_restore(void)
450451
printf(_(" [-X WALDIR | --waldir=WALDIR]\n"));
451452
printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n"));
452453
printf(_(" [--db-include dbname | --db-exclude dbname]\n"));
454+
printf(_(" [--destroy-all-other-dbs]\n"));
453455
printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n"));
454456
printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n"));
455457
printf(_(" [--recovery-target-timeline=timeline]\n"));
@@ -497,6 +499,9 @@ help_restore(void)
497499
printf(_("\n Partial restore options:\n"));
498500
printf(_(" --db-include dbname restore only specified databases\n"));
499501
printf(_(" --db-exclude dbname do not restore specified databases\n"));
502+
printf(_(" --destroy-all-other-dbs\n"));
503+
printf(_(" allows to do partial restore that is prohibited by default,\n"));
504+
printf(_(" because it might remove all other databases.\n"));
500505

501506
printf(_("\n Recovery options:\n"));
502507
printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n"));

‎src/pg_probackup.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ static parray *datname_include_list = NULL;
124124
static parray *exclude_absolute_paths_list = NULL;
125125
static parray *exclude_relative_paths_list = NULL;
126126
static char* gl_waldir_path = NULL;
127+
static bool allow_partial_incremental = false;
127128

128129
/* checkdb options */
129130
bool need_amcheck = false;
@@ -242,6 +243,7 @@ static ConfigOption cmd_options[] =
242243
{ 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT },
243244
{ 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT },
244245
{ 's', 'X', "waldir", &gl_waldir_path, SOURCE_CMD_STRICT },
246+
{ 'b', 242, "destroy-all-other-dbs", &allow_partial_incremental, SOURCE_CMD_STRICT },
245247
/* checkdb options */
246248
{ 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT },
247249
{ 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT },
@@ -764,6 +766,7 @@ main(int argc, char *argv[])
764766
restore_params->partial_restore_type = NONE;
765767
restore_params->primary_conninfo = primary_conninfo;
766768
restore_params->incremental_mode = incremental_mode;
769+
restore_params->allow_partial_incremental = allow_partial_incremental;
767770

768771
/* handle partial restore parameters */
769772
if (datname_exclude_list && datname_include_list)

‎src/pg_probackup.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ typedef enum DestDirIncrCompatibility
179179
POSTMASTER_IS_RUNNING,
180180
SYSTEM_ID_MISMATCH,
181181
BACKUP_LABEL_EXISTS,
182+
PARTIAL_INCREMENTAL_FORBIDDEN,
182183
DEST_IS_NOT_OK,
183184
DEST_OK
184185
} DestDirIncrCompatibility;
@@ -585,7 +586,8 @@ typedef struct pgRestoreParams
585586
/* options for partial restore */
586587
PartialRestoreType partial_restore_type;
587588
parray *partial_db_list;
588-
589+
bool allow_partial_incremental;
590+
589591
char* waldir;
590592
} pgRestoreParams;
591593

@@ -903,7 +905,9 @@ extern parray *get_backup_filelist(pgBackup *backup, bool strict);
903905
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict);
904906
extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli);
905907
extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
906-
IncrRestoreMode incremental_mode);
908+
IncrRestoreMode incremental_mode,
909+
parray *partial_db_list,
910+
bool allow_partial_incremental);
907911

908912
/* in remote.c */
909913
extern void check_remote_agent_compatibility(int agent_version,

‎src/restore.c

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,19 +150,23 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
150150
if (params->incremental_mode != INCR_NONE)
151151
{
152152
DestDirIncrCompatibility rc;
153+
const char *message = NULL;
153154
bool ok_to_go = true;
154155

155156
elog(INFO, "Running incremental restore into nonempty directory: \"%s\"",
156157
instance_config.pgdata);
157158

158159
rc = check_incremental_compatibility(instance_config.pgdata,
159160
instance_config.system_identifier,
160-
params->incremental_mode);
161+
params->incremental_mode,
162+
params->partial_db_list,
163+
params->allow_partial_incremental);
161164
if (rc == POSTMASTER_IS_RUNNING)
162165
{
163166
/* Even with force flag it is unwise to run
164167
* incremental restore over running instance
165168
*/
169+
message = "Postmaster is running.";
166170
ok_to_go = false;
167171
}
168172
else if (rc == SYSTEM_ID_MISMATCH)
@@ -174,7 +178,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
174178
if (params->incremental_mode != INCR_NONE && params->force)
175179
cleanup_pgdata = true;
176180
else
181+
{
182+
message = "System ID mismatch.";
177183
ok_to_go = false;
184+
}
178185
}
179186
else if (rc == BACKUP_LABEL_EXISTS)
180187
{
@@ -187,7 +194,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
187194
* to calculate switchpoint.
188195
*/
189196
if (params->incremental_mode == INCR_LSN)
197+
{
198+
message = "Backup label exists. Cannot use incremental restore in LSN mode.";
190199
ok_to_go = false;
200+
}
191201
}
192202
else if (rc == DEST_IS_NOT_OK)
193203
{
@@ -196,11 +206,16 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
196206
* so we cannot be sure that postmaster is running or not.
197207
* It is better to just error out.
198208
*/
209+
message = "We cannot be sure about the database state.";
210+
ok_to_go = false;
211+
} else if (rc == PARTIAL_INCREMENTAL_FORBIDDEN)
212+
{
213+
message = "Partial incremental restore into non-empty PGDATA is forbidden.";
199214
ok_to_go = false;
200215
}
201216

202217
if (!ok_to_go)
203-
elog(ERROR, "Incremental restore is not allowed");
218+
elog(ERROR, "Incremental restore is not allowed: %s", message);
204219
}
205220
else
206221
elog(ERROR, "Restore destination is not empty: \"%s\"",
@@ -2142,7 +2157,9 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
21422157
*/
21432158
DestDirIncrCompatibility
21442159
check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
2145-
IncrRestoreMode incremental_mode)
2160+
IncrRestoreMode incremental_mode,
2161+
parray *partial_db_list,
2162+
bool allow_partial_incremental)
21462163
{
21472164
uint64 system_id_pgdata;
21482165
bool system_id_match = false;
@@ -2226,6 +2243,8 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
22262243
if (backup_label_exists)
22272244
return BACKUP_LABEL_EXISTS;
22282245

2246+
if (partial_db_list && !allow_partial_incremental)
2247+
return PARTIAL_INCREMENTAL_FORBIDDEN;
22292248
/* some other error condition */
22302249
if (!success)
22312250
return DEST_IS_NOT_OK;

‎tests/expected/option_help.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
9292
[-X WALDIR | --waldir=WALDIR]
9393
[-I | --incremental-mode=none|checksum|lsn]
9494
[--db-include | --db-exclude]
95+
[--destroy-all-other-dbs]
9596
[--remote-proto] [--remote-host]
9697
[--remote-port] [--remote-path] [--remote-user]
9798
[--ssh-options]

‎tests/expected/option_help_ru.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ pg_probackup - утилита для управления резервным к
9292
[-X WALDIR | --waldir=WALDIR]
9393
[-I | --incremental-mode=none|checksum|lsn]
9494
[--db-include | --db-exclude]
95+
[--destroy-all-other-dbs]
9596
[--remote-proto] [--remote-host]
9697
[--remote-port] [--remote-path] [--remote-user]
9798
[--ssh-options]

‎tests/incr_restore_test.py

Lines changed: 132 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1962,7 +1962,9 @@ def test_incremental_partial_restore_exclude_checksum(self):
19621962
node2, options=[
19631963
"--db-exclude=db1",
19641964
"--db-exclude=db5",
1965-
"-I", "checksum"])
1965+
"-I", "checksum",
1966+
"--destroy-all-other-dbs",
1967+
])
19661968

19671969
pgdata2 = self.pgdata_content(node2.data_dir)
19681970

@@ -2068,7 +2070,9 @@ def test_incremental_partial_restore_exclude_lsn(self):
20682070
node2, options=[
20692071
"--db-exclude=db1",
20702072
"--db-exclude=db5",
2071-
"-I", "lsn"])
2073+
"-I", "lsn",
2074+
"--destroy-all-other-dbs",
2075+
])
20722076

20732077
pgdata2 = self.pgdata_content(node2.data_dir)
20742078

@@ -2188,7 +2192,8 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self):
21882192
"--db-exclude=db1",
21892193
"--db-exclude=db5",
21902194
"-T", "{0}={1}".format(
2191-
node_tablespace, node2_tablespace)])
2195+
node_tablespace, node2_tablespace),
2196+
"--destroy-all-other-dbs"])
21922197
# we should die here because exception is what we expect to happen
21932198
self.assertEqual(
21942199
1, 0,
@@ -2209,7 +2214,9 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self):
22092214
"--db-exclude=db1",
22102215
"--db-exclude=db5",
22112216
"-T", "{0}={1}".format(
2212-
node_tablespace, node2_tablespace)])
2217+
node_tablespace, node2_tablespace),
2218+
"--destroy-all-other-dbs",
2219+
])
22132220

22142221
pgdata2 = self.pgdata_content(node2.data_dir)
22152222

@@ -2241,6 +2248,127 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self):
22412248

22422249
self.assertNotIn('PANIC', output)
22432250

2251+
def test_incremental_partial_restore_deny(self):
2252+
"""
2253+
Do now allow partial incremental restore into non-empty PGDATA
2254+
becase we can't limit WAL replay to a single database.
2255+
"""
2256+
backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup')
2257+
node = self.make_simple_node(
2258+
base_dir=os.path.join(self.module_name, self.fname, 'node'),
2259+
initdb_params=['--data-checksums'])
2260+
2261+
self.init_pb(backup_dir)
2262+
self.add_instance(backup_dir, 'node', node)
2263+
self.set_archiving(backup_dir, 'node', node)
2264+
node.slow_start()
2265+
2266+
for i in range(1, 3):
2267+
node.safe_psql('postgres', f'CREATE database db{i}')
2268+
2269+
# FULL backup
2270+
backup_id = self.backup_node(backup_dir, 'node', node)
2271+
pgdata = self.pgdata_content(node.data_dir)
2272+
2273+
try:
2274+
self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN'])
2275+
self.fail("incremental partial restore is not allowed")
2276+
except ProbackupException as e:
2277+
self.assertIn("Incremental restore is not allowed: Postmaster is running.", e.message)
2278+
2279+
node.safe_psql('db2', 'create table x (id int)')
2280+
node.safe_psql('db2', 'insert into x values (42)')
2281+
2282+
node.stop()
2283+
2284+
try:
2285+
self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN'])
2286+
self.fail("because incremental partial restore is not allowed")
2287+
except ProbackupException as e:
2288+
self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message)
2289+
2290+
node.slow_start()
2291+
value = node.execute('db2', 'select * from x')[0][0]
2292+
self.assertEqual(42, value)
2293+
2294+
def test_deny_incremental_partial_restore_exclude_tablespace_checksum(self):
2295+
"""
2296+
Do now allow partial incremental restore into non-empty PGDATA
2297+
becase we can't limit WAL replay to a single database.
2298+
(case of tablespaces)
2299+
"""
2300+
backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup')
2301+
node = self.make_simple_node(
2302+
base_dir=os.path.join(self.module_name, self.fname, 'node'),
2303+
initdb_params=['--data-checksums'])
2304+
2305+
self.init_pb(backup_dir)
2306+
self.add_instance(backup_dir, 'node', node)
2307+
self.set_archiving(backup_dir, 'node', node)
2308+
node.slow_start()
2309+
2310+
self.create_tblspace_in_node(node, 'somedata')
2311+
2312+
node_tablespace = self.get_tblspace_path(node, 'somedata')
2313+
2314+
tbl_oid = node.safe_psql(
2315+
'postgres',
2316+
"SELECT oid "
2317+
"FROM pg_tablespace "
2318+
"WHERE spcname = 'somedata'").rstrip()
2319+
2320+
for i in range(1, 10, 1):
2321+
node.safe_psql(
2322+
'postgres',
2323+
'CREATE database db{0} tablespace somedata'.format(i))
2324+
2325+
db_list_raw = node.safe_psql(
2326+
'postgres',
2327+
'SELECT to_json(a) '
2328+
'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
2329+
2330+
db_list_splitted = db_list_raw.splitlines()
2331+
2332+
db_list = {}
2333+
for line in db_list_splitted:
2334+
line = json.loads(line)
2335+
db_list[line['datname']] = line['oid']
2336+
2337+
# FULL backup
2338+
backup_id = self.backup_node(backup_dir, 'node', node)
2339+
2340+
# node2
2341+
node2 = self.make_simple_node('node2')
2342+
node2.cleanup()
2343+
node2_tablespace = self.get_tblspace_path(node2, 'somedata')
2344+
2345+
# in node2 restore full backup
2346+
self.restore_node(
2347+
backup_dir, 'node',
2348+
node2, options=[
2349+
"-T", f"{node_tablespace}={node2_tablespace}"])
2350+
2351+
# partial incremental restore into node2
2352+
try:
2353+
self.restore_node(backup_dir, 'node', node2,
2354+
options=["-I", "checksum",
2355+
"--db-exclude=db1",
2356+
"--db-exclude=db5",
2357+
"-T", f"{node_tablespace}={node2_tablespace}"])
2358+
self.fail("remapped tablespace contain old data")
2359+
except ProbackupException as e:
2360+
pass
2361+
2362+
try:
2363+
self.restore_node(backup_dir, 'node', node2,
2364+
options=[
2365+
"-I", "checksum", "--force",
2366+
"--db-exclude=db1", "--db-exclude=db5",
2367+
"-T", f"{node_tablespace}={node2_tablespace}"])
2368+
self.fail("incremental partial restore is not allowed")
2369+
except ProbackupException as e:
2370+
self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message)
2371+
22442372
def test_incremental_pg_filenode_map(self):
22452373
"""
22462374
https://github.com/postgrespro/pg_probackup/issues/320

0 commit comments

Comments
 (0)