Skip to content

Commit f2c523b

Browse files
author
Michael Paquier
committed
Remove un-needed WAL segments from archive at backup deletion
During the execution of the "delete" command, the oldest start LSN is saved and used to determine what is the oldest WAL segment that needs to be kept for the existing backups. Like pg_archivecleanup, the implemented logic ignores the timeline part of the WAL segment name, ensuring that a segment from a parent timeline will not be removed prematurely. This logic could be made more complicated regarding this matter, but in order to take backups from a node on a given timeline this is far enough.
1 parent a5afc87 commit f2c523b

File tree

5 files changed

+117
-18
lines changed

5 files changed

+117
-18
lines changed

‎delete.c

Lines changed: 87 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,20 @@
99

1010
#include "pg_arman.h"
1111

12+
#include <dirent.h>
13+
#include <unistd.h>
14+
1215
static int pgBackupDeleteFiles(pgBackup *backup);
1316

1417
int
1518
do_delete(pgBackupRange *range)
1619
{
17-
int i;
18-
int ret;
19-
parray *backup_list;
20-
bool do_delete = false;
20+
int i;
21+
int ret;
22+
parray *backup_list;
23+
bool do_delete = false;
24+
XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
25+
TimeLineID oldest_tli;
2126

2227
/* DATE are always required */
2328
if (!pgBackupRangeIsValid(range))
@@ -56,7 +61,11 @@ do_delete(pgBackupRange *range)
5661
if (backup->backup_mode >= BACKUP_MODE_FULL &&
5762
backup->status == BACKUP_STATUS_OK &&
5863
backup->start_time <= range->begin)
64+
{
5965
do_delete = true;
66+
oldest_lsn = backup->start_lsn;
67+
oldest_tli = backup->tli;
68+
}
6069
}
6170

6271
/* release catalog lock */
@@ -65,6 +74,80 @@ do_delete(pgBackupRange *range)
6574
/* cleanup */
6675
parray_walk(backup_list, pgBackupFree);
6776
parray_free(backup_list);
77+
78+
/*
79+
* Delete in archive WAL segments that are not needed anymore. The oldest
80+
* segment to be kept is the first segment that the oldest full backup
81+
* found around needs to keep.
82+
*/
83+
if (!XLogRecPtrIsInvalid(oldest_lsn))
84+
{
85+
XLogSegNo targetSegNo;
86+
char oldestSegmentNeeded[MAXFNAMELEN];
87+
DIR *arcdir;
88+
struct dirent *arcde;
89+
char wal_file[MAXPGPATH];
90+
int rc;
91+
92+
XLByteToSeg(oldest_lsn, targetSegNo);
93+
XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo);
94+
elog(LOG, "Removing segments older than %s", oldestSegmentNeeded);
95+
96+
/*
97+
* Now is time to do the actual work and to remove all the segments
98+
* not needed anymore.
99+
*/
100+
if ((arcdir = opendir(arclog_path)) != NULL)
101+
{
102+
while (errno = 0, (arcde = readdir(arcdir)) != NULL)
103+
{
104+
/*
105+
* We ignore the timeline part of the XLOG segment identifiers in
106+
* deciding whether a segment is still needed. This ensures that
107+
* we won't prematurely remove a segment from a parent timeline.
108+
* We could probably be a little more proactive about removing
109+
* segments of non-parent timelines, but that would be a whole lot
110+
* more complicated.
111+
*
112+
* We use the alphanumeric sorting property of the filenames to
113+
* decide which ones are earlier than the exclusiveCleanupFileName
114+
* file. Note that this means files are not removed in the order
115+
* they were originally written, in case this worries you.
116+
*/
117+
if ((IsXLogFileName(arcde->d_name) ||
118+
IsPartialXLogFileName(arcde->d_name)) &&
119+
strcmp(arcde->d_name + 8, oldestSegmentNeeded + 8) < 0)
120+
{
121+
/*
122+
* Use the original file name again now, including any
123+
* extension that might have been chopped off before testing
124+
* the sequence.
125+
*/
126+
snprintf(wal_file, MAXPGPATH, "%s/%s",
127+
arclog_path, arcde->d_name);
128+
129+
rc = unlink(wal_file);
130+
if (rc != 0)
131+
{
132+
elog(WARNING, "could not remove file \"%s\": %s",
133+
wal_file, strerror(errno));
134+
break;
135+
}
136+
elog(LOG, "removed WAL segment \"%s\"", wal_file);
137+
}
138+
}
139+
if (errno)
140+
elog(WARNING, "could not read archive location \"%s\": %s",
141+
arclog_path, strerror(errno));
142+
if (closedir(arcdir))
143+
elog(WARNING, "could not close archive location \"%s\": %s",
144+
arclog_path, strerror(errno));
145+
}
146+
else
147+
elog(WARNING, "could not open archive location \"%s\": %s",
148+
arclog_path, strerror(errno));
149+
}
150+
68151
return 0;
69152
}
70153

‎doc/pg_arman.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,9 @@ You can check the "RECOVERY_XID" and "RECOVERY_TIME" which are used for
178178
restore option "--recovery-target-xid", "--recovery-target-time".
179179

180180
The delete command deletes backup files not required by recovery after
181-
the specified date.
181+
the specified date. This command also cleans up in the WAL archive the
182+
WAL segments that are no longer needed to restore from the remaining
183+
backups.
182184

183185
== OPTIONS ==
184186

‎expected/option.out

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,37 +72,42 @@ ERROR: invalid backup-mode "bad"
7272
12
7373

7474
###### COMMAND OPTION TEST-0006 ######
75+
###### delete failure without archive path ######
76+
ERROR: delete command needs ARCLOG_PATH (-A, --arclog-path) to be set
77+
12
78+
79+
###### COMMAND OPTION TEST-0007 ######
7580
###### delete failure without DATE ######
7681
ERROR: required delete range option not specified: delete DATE
7782
12
7883

79-
###### COMMAND OPTION TEST-0007 ######
84+
###### COMMAND OPTION TEST-0008 ######
8085
###### syntax error in pg_arman.ini ######
8186
WARNING: syntax error in " = INFINITE"
8287
ERROR: Required parameter not specified: BACKUP_MODE (-b, --backup-mode)
8388
12
8489

85-
###### COMMAND OPTION TEST-0008 ######
90+
###### COMMAND OPTION TEST-0009 ######
8691
###### invalid value in pg_arman.ini ######
8792
ERROR: invalid backup-mode ""
8893
12
8994

90-
###### COMMAND OPTION TEST-0009 ######
95+
###### COMMAND OPTION TEST-0010 ######
9196
###### invalid value in pg_arman.ini ######
9297
ERROR: option --keep-data-generations should be a 32bit signed integer: 'TRUE'
9398
12
9499

95-
###### COMMAND OPTION TEST-0010 ######
100+
###### COMMAND OPTION TEST-0011 ######
96101
###### invalid value in pg_arman.ini ######
97102
ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'
98103
12
99104

100-
###### COMMAND OPTION TEST-0011 ######
105+
###### COMMAND OPTION TEST-0012 ######
101106
###### invalid option in pg_arman.ini ######
102107
ERROR: invalid option "TIMELINEID"
103108
12
104109

105-
###### COMMAND OPTION TEST-0012 ######
110+
###### COMMAND OPTION TEST-0013 ######
106111
###### check priority of several pg_arman.ini files ######
107112
ERROR: invalid backup-mode "ENV_PATH"
108113
12

‎pg_arman.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@ main(int argc, char *argv[])
148148
if (arclog_path != NULL && !is_absolute_path(arclog_path))
149149
elog(ERROR_ARGS, "-A, --arclog-path must be an absolute path");
150150

151+
/* Sanity checks with commands */
152+
if (pg_strcasecmp(cmd, "delete") == 0 && arclog_path == NULL)
153+
elog(ERROR_ARGS, "delete command needs ARCLOG_PATH (-A, --arclog-path) to be set");
154+
151155
/* setup exclusion list for file search */
152156
for (i = 0; pgdata_exclude[i]; i++) /* find first empty slot */
153157
;

‎sql/option.sh

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,47 +35,52 @@ pg_arman backup -B ${BACKUP_PATH} -A ${ARCLOG_PATH} -b bad -p ${TEST_PGPORT};ech
3535
echo ''
3636

3737
echo '###### COMMAND OPTION TEST-0006 ######'
38-
echo '###### delete failure without DATE ######'
38+
echo '###### delete failure without archive path ######'
3939
pg_arman delete -B ${BACKUP_PATH};echo $?
4040
echo ''
4141

42+
echo '###### COMMAND OPTION TEST-0007 ######'
43+
echo '###### delete failure without DATE ######'
44+
pg_arman delete -B ${BACKUP_PATH} -A ${ARCLOG_PATH};echo $?
45+
echo ''
46+
4247
init_backup
4348

44-
echo '###### COMMAND OPTION TEST-0007 ######'
49+
echo '###### COMMAND OPTION TEST-0008 ######'
4550
echo '###### syntax error in pg_arman.ini ######'
4651
echo " = INFINITE" >> ${BACKUP_PATH}/pg_arman.ini
4752
pg_arman backup -B ${BACKUP_PATH} -A ${ARCLOG_PATH} -p ${TEST_PGPORT};echo $?
4853
echo ''
4954

50-
echo '###### COMMAND OPTION TEST-0008 ######'
55+
echo '###### COMMAND OPTION TEST-0009 ######'
5156
echo '###### invalid value in pg_arman.ini ######'
5257
init_catalog
5358
echo "BACKUP_MODE=" >> ${BACKUP_PATH}/pg_arman.ini
5459
pg_arman backup -B ${BACKUP_PATH} -A ${ARCLOG_PATH} -p ${TEST_PGPORT};echo $?
5560
echo ''
5661

57-
echo '###### COMMAND OPTION TEST-0009 ######'
62+
echo '###### COMMAND OPTION TEST-0010 ######'
5863
echo '###### invalid value in pg_arman.ini ######'
5964
init_catalog
6065
echo "KEEP_DATA_GENERATIONS=TRUE" >> ${BACKUP_PATH}/pg_arman.ini
6166
pg_arman backup -B ${BACKUP_PATH} -A ${ARCLOG_PATH} -b full -p ${TEST_PGPORT};echo $?
6267
echo ''
6368

64-
echo '###### COMMAND OPTION TEST-0010 ######'
69+
echo '###### COMMAND OPTION TEST-0011 ######'
6570
echo '###### invalid value in pg_arman.ini ######'
6671
init_catalog
6772
echo "SMOOTH_CHECKPOINT=FOO" >> ${BACKUP_PATH}/pg_arman.ini
6873
pg_arman backup -B ${BACKUP_PATH} -A ${ARCLOG_PATH} -b full -p ${TEST_PGPORT};echo $?
6974
echo ''
7075

71-
echo '###### COMMAND OPTION TEST-0011 ######'
76+
echo '###### COMMAND OPTION TEST-0012 ######'
7277
echo '###### invalid option in pg_arman.ini ######'
7378
init_catalog
7479
echo "TIMELINEID=1" >> ${BACKUP_PATH}/pg_arman.ini
7580
pg_arman backup -B ${BACKUP_PATH} -A ${ARCLOG_PATH} -b full -p ${TEST_PGPORT};echo $?
7681
echo ''
7782

78-
echo '###### COMMAND OPTION TEST-0012 ######'
83+
echo '###### COMMAND OPTION TEST-0013 ######'
7984
echo '###### check priority of several pg_arman.ini files ######'
8085
init_catalog
8186
mkdir -p ${BACKUP_PATH}/conf_path_a

0 commit comments

Comments
 (0)