Skip to content

Commit 4e94454

Browse files
alubennikovaalubennikova
committed
1. Implement backup of multiple instances. Add option --instance, commands add-instance and del-instance.
2. Implement pg_probackup specific commands for archiving: archive-push for archive_command and archive-get for restore_command
1 parent 5390887 commit 4e94454

File tree

16 files changed

+521
-75
lines changed

16 files changed

+521
-75
lines changed

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ OBJS = backup.o \
1919
xlogreader.o \
2020
streamutil.o \
2121
receivelog.o \
22+
archive.o \
2223
utils/parray.o \
2324
utils/pgut.o \
2425
utils/logger.o

‎archive.c

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*-------------------------------------------------------------------------
2+
*
3+
* archive.c: - pg_probackup specific archive commands for archive backups.
4+
*
5+
*
6+
* Portions Copyright (c) 2017, Postgres Professional
7+
*
8+
*-------------------------------------------------------------------------
9+
*/
10+
#include "pg_probackup.h"
11+
12+
#include <unistd.h>
13+
#include <sys/stat.h>
14+
15+
/*
16+
* pg_probackup specific archive command for archive backups
17+
* set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup
18+
* --wal-file-path %p --wal-file-name %f', to move backups into arclog_path.
19+
* Where archlog_path is $BACKUP_PATH/wal/system_id.
20+
* Currently it just copies wal files to the new location.
21+
* TODO Planned options: compress, list the arclog content,
22+
* compute and validate checksums.
23+
*/
24+
int
25+
do_archive_push(char *wal_file_path, char *wal_file_name)
26+
{
27+
char backup_wal_file_path[MAXPGPATH];
28+
char absolute_wal_file_path[MAXPGPATH];
29+
char current_dir[MAXPGPATH];
30+
int64 system_id;
31+
pgBackupConfig *config;
32+
33+
if (!getcwd(current_dir, sizeof(current_dir)))
34+
elog(ERROR, "getcwd() error");
35+
36+
/* verify that archive-push --instance parameter is valid */
37+
config = readBackupCatalogConfigFile();
38+
system_id = get_system_identifier(current_dir);
39+
40+
if (config->pgdata == NULL)
41+
elog(ERROR, "cannot read pg_probackup.conf for this instance");
42+
43+
if(system_id != config->system_identifier)
44+
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
45+
"Instance '%s' should have SYSTEM_ID = %ld instead of %ld",
46+
wal_file_name, instance_name, config->system_identifier, system_id);
47+
48+
if (strcmp(current_dir, config->pgdata) != 0)
49+
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
50+
"Instance '%s' should have PGDATA = %s instead of %s",
51+
wal_file_name, instance_name, config->pgdata, current_dir);
52+
53+
/* Create 'archlog_path' directory. Do nothing if it already exists. */
54+
dir_create_dir(arclog_path, DIR_PERMISSION);
55+
56+
join_path_components(absolute_wal_file_path, current_dir, wal_file_path);
57+
join_path_components(backup_wal_file_path, arclog_path, wal_file_name);
58+
59+
elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path);
60+
copy_wal_file(absolute_wal_file_path, backup_wal_file_path);
61+
elog(INFO, "pg_probackup archive-push completed successfully");
62+
63+
return 0;
64+
}
65+
66+
/*
67+
* pg_probackup specific restore command.
68+
* Move files from arclog_path to pgdata/wal_file_path.
69+
*/
70+
int
71+
do_archive_get(char *wal_file_path, char *wal_file_name)
72+
{
73+
char backup_wal_file_path[MAXPGPATH];
74+
char absolute_wal_file_path[MAXPGPATH];
75+
char current_dir[MAXPGPATH];
76+
77+
if (!getcwd(current_dir, sizeof(current_dir)))
78+
elog(ERROR, "getcwd() error");
79+
80+
join_path_components(absolute_wal_file_path, current_dir, wal_file_path);
81+
join_path_components(backup_wal_file_path, arclog_path, wal_file_name);
82+
83+
elog(INFO, "pg_probackup archive-get from %s to %s", backup_wal_file_path, absolute_wal_file_path);
84+
copy_wal_file(backup_wal_file_path, absolute_wal_file_path);
85+
elog(INFO, "pg_probackup archive-get completed successfully");
86+
87+
return 0;
88+
}

‎backup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -512,7 +512,7 @@ check_system_identifiers(void)
512512
uint64 system_id_pgdata;
513513
char *val;
514514

515-
system_id_pgdata = get_system_identifier();
515+
system_id_pgdata = get_system_identifier(pgdata);
516516

517517
res = pgut_execute(backup_conn,
518518
"SELECT system_identifier FROM pg_control_system()",

‎catalog.c

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ catalog_lock(void)
5050
pid_t my_pid,
5151
my_p_pid;
5252

53-
join_path_components(lock_file, backup_path, BACKUP_CATALOG_PID);
53+
join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID);
5454

5555
/*
5656
* If the PID in the lockfile is our own PID or our parent's or
@@ -84,7 +84,7 @@ catalog_lock(void)
8484

8585
/*
8686
* We need a loop here because of race conditions. But don't loop forever
87-
* (for example, a non-writable $backup_path directory might cause a failure
87+
* (for example, a non-writable $backup_instance_path directory might cause a failure
8888
* that won't go away). 100 tries seems like plenty.
8989
*/
9090
for (ntries = 0;; ntries++)
@@ -253,16 +253,14 @@ catalog_get_backup_list(time_t requested_backup_id)
253253
{
254254
DIR *date_dir = NULL;
255255
struct dirent *date_ent = NULL;
256-
char backups_path[MAXPGPATH];
257256
parray *backups = NULL;
258257
pgBackup *backup = NULL;
259258

260-
/* open backup root directory */
261-
join_path_components(backups_path, backup_path, BACKUPS_DIR);
262-
date_dir = opendir(backups_path);
259+
/* open backup instance backups directory */
260+
date_dir = opendir(backup_instance_path);
263261
if (date_dir == NULL)
264262
{
265-
elog(WARNING, "cannot open directory \"%s\": %s", backups_path,
263+
elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path,
266264
strerror(errno));
267265
goto err_proc;
268266
}
@@ -275,12 +273,12 @@ catalog_get_backup_list(time_t requested_backup_id)
275273
char date_path[MAXPGPATH];
276274

277275
/* skip not-directory entries and hidden entries */
278-
if (!IsDir(backups_path, date_ent->d_name)
276+
if (!IsDir(backup_instance_path, date_ent->d_name)
279277
|| date_ent->d_name[0] == '.')
280278
continue;
281279

282280
/* open subdirectory of specific backup */
283-
join_path_components(date_path, backups_path, date_ent->d_name);
281+
join_path_components(date_path, backup_instance_path, date_ent->d_name);
284282

285283
/* read backup information from BACKUP_CONTROL_FILE */
286284
snprintf(backup_conf_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONTROL_FILE);
@@ -309,7 +307,7 @@ catalog_get_backup_list(time_t requested_backup_id)
309307
if (errno)
310308
{
311309
elog(WARNING, "cannot read backup root directory \"%s\": %s",
312-
backups_path, strerror(errno));
310+
backup_instance_path, strerror(errno));
313311
goto err_proc;
314312
}
315313

@@ -620,9 +618,9 @@ pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subd
620618

621619
datetime = base36enc(backup->start_time);
622620
if (subdir)
623-
snprintf(path, len, "%s/%s/%s/%s", backup_path, BACKUPS_DIR, datetime, subdir);
621+
snprintf(path, len, "%s/%s/%s", backup_instance_path, datetime, subdir);
624622
else
625-
snprintf(path, len, "%s/%s/%s", backup_path, BACKUPS_DIR, datetime);
623+
snprintf(path, len, "%s/%s", backup_instance_path, datetime);
626624
free(datetime);
627625

628626
make_native_path(path);

‎configure.c

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,7 @@ writeBackupCatalogConfigFile(pgBackupConfig *config)
121121
char path[MAXPGPATH];
122122
FILE *fp;
123123

124-
join_path_components(path, backup_path, BACKUPS_DIR);
125-
join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE);
124+
join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
126125
fp = fopen(path, "wt");
127126
if (fp == NULL)
128127
elog(ERROR, "cannot create %s: %s",
@@ -164,8 +163,7 @@ readBackupCatalogConfigFile(void)
164163

165164
cur_config = config;
166165

167-
join_path_components(path, backup_path, BACKUPS_DIR);
168-
join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE);
166+
join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
169167

170168
pgBackupConfigInit(config);
171169
pgut_readopt(path, options, ERROR);

‎data.c

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,105 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
679679
return true;
680680
}
681681

682+
/* Almost like copy file, except the fact we don't calculate checksum */
683+
void
684+
copy_wal_file(const char *from_path, const char *to_path)
685+
{
686+
FILE *in;
687+
FILE *out;
688+
size_t read_len = 0;
689+
int errno_tmp;
690+
char buf[XLOG_BLCKSZ];
691+
struct stat st;
692+
693+
/* open file for read */
694+
in = fopen(from_path, "r");
695+
if (in == NULL)
696+
{
697+
/* maybe deleted, it's not error */
698+
if (errno == ENOENT)
699+
elog(ERROR, "cannot open source WAL file \"%s\": %s", from_path,
700+
strerror(errno));
701+
}
702+
703+
/* open backup file for write */
704+
out = fopen(to_path, "w");
705+
if (out == NULL)
706+
{
707+
int errno_tmp = errno;
708+
fclose(in);
709+
elog(ERROR, "cannot open destination file \"%s\": %s",
710+
to_path, strerror(errno_tmp));
711+
}
712+
713+
/* stat source file to change mode of destination file */
714+
if (fstat(fileno(in), &st) == -1)
715+
{
716+
fclose(in);
717+
fclose(out);
718+
elog(ERROR, "cannot stat \"%s\": %s", from_path,
719+
strerror(errno));
720+
}
721+
722+
if (st.st_size > XLOG_SEG_SIZE)
723+
elog(ERROR, "Unexpected wal file size %s : %ld", from_path, st.st_size);
724+
725+
/* copy content */
726+
for (;;)
727+
{
728+
if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf))
729+
break;
730+
731+
if (fwrite(buf, 1, read_len, out) != read_len)
732+
{
733+
errno_tmp = errno;
734+
/* oops */
735+
fclose(in);
736+
fclose(out);
737+
elog(ERROR, "cannot write to \"%s\": %s", to_path,
738+
strerror(errno_tmp));
739+
}
740+
}
741+
742+
errno_tmp = errno;
743+
if (!feof(in))
744+
{
745+
fclose(in);
746+
fclose(out);
747+
elog(ERROR, "cannot read backup mode file \"%s\": %s",
748+
from_path, strerror(errno_tmp));
749+
}
750+
751+
/* copy odd part */
752+
if (read_len > 0)
753+
{
754+
if (fwrite(buf, 1, read_len, out) != read_len)
755+
{
756+
errno_tmp = errno;
757+
/* oops */
758+
fclose(in);
759+
fclose(out);
760+
elog(ERROR, "cannot write to \"%s\": %s", to_path,
761+
strerror(errno_tmp));
762+
}
763+
}
764+
765+
766+
/* update file permission. */
767+
if (chmod(to_path, st.st_mode) == -1)
768+
{
769+
errno_tmp = errno;
770+
fclose(in);
771+
fclose(out);
772+
elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
773+
strerror(errno_tmp));
774+
}
775+
776+
fclose(in);
777+
fclose(out);
778+
779+
}
780+
682781
/*
683782
* Save part of the file into backup.
684783
* skip_size - size of the file in previous backup. We can skip it

‎delete.c

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,3 +384,48 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli)
384384
elog(WARNING, "could not open archive location \"%s\": %s",
385385
arclog_path, strerror(errno));
386386
}
387+
388+
389+
/* Delete all backup files and wal files of given instance. */
390+
int
391+
do_delete_instance(void)
392+
{
393+
parray *backup_list;
394+
int i;
395+
char instance_config_path[MAXPGPATH];
396+
397+
/* Delete all backups. */
398+
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
399+
400+
for (i = 0; i < parray_num(backup_list); i++)
401+
{
402+
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
403+
pgBackupDeleteFiles(backup);
404+
}
405+
406+
/* Cleanup */
407+
parray_walk(backup_list, pgBackupFree);
408+
parray_free(backup_list);
409+
410+
/* Delete all wal files. */
411+
delete_walfiles(InvalidXLogRecPtr, 0);
412+
413+
/* Delete backup instance config file */
414+
join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
415+
if (remove(instance_config_path))
416+
{
417+
elog(ERROR, "can't remove \"%s\": %s", instance_config_path,
418+
strerror(errno));
419+
}
420+
421+
/* Delete instance root directories */
422+
if (rmdir(backup_instance_path) != 0)
423+
elog(ERROR, "can't remove \"%s\": %s", backup_instance_path,
424+
strerror(errno));
425+
if (rmdir(arclog_path) != 0)
426+
elog(ERROR, "can't remove \"%s\": %s", backup_instance_path,
427+
strerror(errno));
428+
429+
elog(INFO, "Instance '%s' deleted successfully", instance_name);
430+
return 0;
431+
}

‎dir.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
307307
parray *black_list = NULL;
308308
char path[MAXPGPATH];
309309

310-
join_path_components(path, backup_path, PG_BLACK_LIST);
310+
join_path_components(path, backup_instance_path, PG_BLACK_LIST);
311311
/* List files with black list */
312312
if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path))
313313
{

0 commit comments

Comments
 (0)