Skip to content

Commit 9c475ec

Browse files
stalkergstalkerg
committed
First version of ptrack support.
1 parent 7cbbdf0 commit 9c475ec

File tree

13 files changed

+201
-30
lines changed

13 files changed

+201
-30
lines changed

‎backup.c

Lines changed: 96 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
#include "libpq/pqsignal.h"
2121
#include "pgut/pgut-port.h"
22+
#include "storage/bufpage.h"
23+
#include "datapagemap.h"
2224

2325
/* wait 10 sec until WAL archive complete */
2426
#define TIMEOUT_ARCHIVE 10
@@ -44,6 +46,7 @@ static void pg_stop_backup(pgBackup *backup);
4446
static bool pg_is_standby(void);
4547
static void get_lsn(PGresult *res, XLogRecPtr *lsn);
4648
static void get_xid(PGresult *res, uint32 *xid);
49+
static void pg_ptrack_clear(void);
4750

4851
static void add_files(parray *files, const char *root, bool add_root, bool is_pgdata);
4952
static void create_file_list(parray *files,
@@ -52,6 +55,7 @@ static void create_file_list(parray *files,
5255
const char *prefix,
5356
bool is_append);
5457
static void wait_for_archive(pgBackup *backup, const char *sql);
58+
static void make_pagemap_from_ptrack(parray *files);
5559

5660
/*
5761
* Take a backup of database and return the list of files backed up.
@@ -96,7 +100,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
96100
* In differential backup mode, check if there is an already-validated
97101
* full backup on current timeline.
98102
*/
99-
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
103+
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
104+
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
100105
{
101106
pgBackup *prev_backup;
102107

@@ -156,7 +161,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
156161
* To take differential backup, the file list of the last completed database
157162
* backup is needed.
158163
*/
159-
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
164+
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
165+
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
160166
{
161167
/* find last completed database backup */
162168
prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
@@ -209,15 +215,24 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
209215
current.start_lsn);
210216
}
211217

218+
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
219+
{
220+
parray_qsort(backup_files_list, pgFileComparePathDesc);
221+
make_pagemap_from_ptrack(backup_files_list);
222+
}
223+
212224
backup_files(pgdata, path, backup_files_list, prev_files, lsn, NULL);
213225

214-
/* notify end of backup */
226+
/* Clear ptrack files after backup */
227+
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
228+
pg_ptrack_clear();
229+
/* Notify end of backup */
215230
pg_stop_backup(&current);
216231

217-
/* create file list */
232+
/* Create file list */
218233
create_file_list(backup_files_list, pgdata, DATABASE_FILE_LIST, NULL, false);
219234

220-
/* print summary of size of backup mode files */
235+
/* Print summary of size of backup mode files */
221236
for (i = 0; i < parray_num(backup_files_list); i++)
222237
{
223238
pgFile *file = (pgFile *) parray_get(backup_files_list, i);
@@ -228,7 +243,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
228243
* amount of data written counts while for an differential
229244
* backup only the data read counts.
230245
*/
231-
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
246+
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
247+
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
232248
current.data_bytes += file->read_size;
233249
else if (current.backup_mode == BACKUP_MODE_FULL)
234250
current.data_bytes += file->size;
@@ -332,7 +348,8 @@ do_backup(pgBackupOption bkupopt)
332348

333349
/* Database data */
334350
if (current.backup_mode == BACKUP_MODE_FULL ||
335-
current.backup_mode == BACKUP_MODE_DIFF_PAGE)
351+
current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
352+
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
336353
total_read += current.data_bytes;
337354

338355
if (total_read == 0)
@@ -435,6 +452,17 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup)
435452
disconnect();
436453
}
437454

455+
static void
456+
pg_ptrack_clear(void)
457+
{
458+
PGresult *res;
459+
460+
reconnect();
461+
res = execute("select pg_ptrack_clear()", 0, NULL);
462+
PQclear(res);
463+
disconnect();
464+
}
465+
438466
static void
439467
wait_for_archive(pgBackup *backup, const char *sql)
440468
{
@@ -806,6 +834,7 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
806834
pgFile *file = (pgFile *) parray_get(list_file, i);
807835
char *relative;
808836
char *fname;
837+
int path_len;
809838

810839
/* data file must be a regular file */
811840
if (!S_ISREG(file->mode))
@@ -819,6 +848,27 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
819848
!path_is_prefix_of_path("pg_tblspc", relative))
820849
continue;
821850

851+
path_len = strlen(file->path);
852+
if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
853+
{
854+
pgFile tmp_file;
855+
pgFile *search_file;
856+
//elog(WARNING, "Remove ptrack file from backup %s", file->path);
857+
tmp_file.path = pg_strdup(file->path);
858+
tmp_file.path[path_len-7] = '\0';
859+
search_file = *(pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
860+
if (search_file != NULL)
861+
{
862+
//elog(WARNING, "Finded main fork for ptrak:%s", search_file->path);
863+
search_file->ptrack_path = pg_strdup(file->path);
864+
}
865+
free(tmp_file.path);
866+
pgFileFree(file);
867+
parray_remove(list_file, i);
868+
i--;
869+
continue;
870+
}
871+
822872
/* name of data file start with digit */
823873
fname = last_dir_separator(relative);
824874
if (fname == NULL)
@@ -927,3 +977,42 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
927977
pg_free(path);
928978
pg_free(rel_path);
929979
}
980+
981+
void make_pagemap_from_ptrack(parray *files)
982+
{
983+
int i;
984+
for (i = 0; i < parray_num(files); i++)
985+
{
986+
pgFile *p = (pgFile *) parray_get(files, i);
987+
if (p->ptrack_path != NULL)
988+
{
989+
DataPage page;
990+
int i;
991+
struct stat st;
992+
FILE *ptrack_file = fopen(p->ptrack_path, "r");
993+
if (ptrack_file == NULL)
994+
{
995+
elog(ERROR, "cannot open ptrack file \"%s\": %s", p->ptrack_path,
996+
strerror(errno));
997+
}
998+
999+
elog(LOG, "Start copy bitmap from ptrack:%s", p->ptrack_path);
1000+
fstat(fileno(ptrack_file), &st);
1001+
p->pagemap.bitmapsize = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
1002+
p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
1003+
while(fread(page.data, BLCKSZ, 1, ptrack_file) == BLCKSZ)
1004+
{
1005+
char *map = PageGetContents(page.data);
1006+
memcpy(p->pagemap.bitmap, map, BLCKSZ-MAXALIGN(SizeOfPageHeaderData));
1007+
}
1008+
fclose(ptrack_file);
1009+
for(i = 0; i < p->pagemap.bitmapsize; i++)
1010+
if (p->pagemap.bitmap[i] != 0)
1011+
goto end_loop;
1012+
1013+
pg_free(p->pagemap.bitmap);
1014+
p->pagemap.bitmapsize = 0;
1015+
}
1016+
end_loop:;
1017+
}
1018+
}

‎catalog.c

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli)
252252
if (backup->status == BACKUP_STATUS_OK &&
253253
backup->tli == tli &&
254254
(backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
255+
backup->backup_mode == BACKUP_MODE_DIFF_PTRACK ||
255256
backup->backup_mode == BACKUP_MODE_FULL))
256257
return backup;
257258
}
@@ -286,7 +287,7 @@ pgBackupCreateDir(pgBackup *backup)
286287
void
287288
pgBackupWriteConfigSection(FILE *out, pgBackup *backup)
288289
{
289-
static const char *modes[] = { "", "PAGE", "FULL"};
290+
static const char *modes[] = { "", "PAGE", "PTRACK", "FULL"};
290291

291292
fprintf(out, "# configuration\n");
292293
fprintf(out, "BACKUP_MODE=%s\n", modes[backup->backup_mode]);
@@ -478,6 +479,8 @@ parse_backup_mode(const char *value)
478479
return BACKUP_MODE_FULL;
479480
else if (len > 0 && pg_strncasecmp("page", v, strlen("page")) == 0)
480481
return BACKUP_MODE_DIFF_PAGE;
482+
else if (len > 0 && pg_strncasecmp("ptrack", v, strlen("ptrack")) == 0)
483+
return BACKUP_MODE_DIFF_PTRACK;
481484

482485
/* Backup mode is invalid, so leave with an error */
483486
elog(ERROR, "invalid backup-mode \"%s\"", value);

‎data.c

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,6 @@
1818
#include "storage/block.h"
1919
#include "storage/bufpage.h"
2020

21-
typedef union DataPage
22-
{
23-
PageHeaderData page_data;
24-
char data[BLCKSZ];
25-
} DataPage;
26-
2721
typedef struct BackupPageHeader
2822
{
2923
BlockNumber block; /* block number */
@@ -184,7 +178,6 @@ backup_data_file(const char *from_root, const char *to_root,
184178
else
185179
{
186180
datapagemap_iterator_t *iter;
187-
188181
iter = datapagemap_iterate(&file->pagemap);
189182
while (datapagemap_next(iter, &blknum))
190183
{

‎dir.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ pgFileNew(const char *path, bool omit_symlink)
9191
file->linked = NULL;
9292
file->pagemap.bitmap = NULL;
9393
file->pagemap.bitmapsize = 0;
94+
file->ptrack_path = NULL;
9495
file->path = pgut_malloc(strlen(path) + 1);
9596
strcpy(file->path, path); /* enough buffer size guaranteed */
9697

@@ -172,6 +173,8 @@ pgFileFree(void *file)
172173
return;
173174
free(((pgFile *)file)->linked);
174175
free(((pgFile *)file)->path);
176+
if (((pgFile *)file)->ptrack_path != NULL)
177+
free(((pgFile *)file)->ptrack_path);
175178
free(file);
176179
}
177180

@@ -549,6 +552,7 @@ dir_read_file_list(const char *root, const char *file_txt)
549552

550553
file = (pgFile *) pgut_malloc(sizeof(pgFile));
551554
file->path = pgut_malloc((root ? strlen(root) + 1 : 0) + strlen(path) + 1);
555+
file->ptrack_path = NULL;
552556
file->pagemap.bitmap = NULL;
553557
file->pagemap.bitmapsize = 0;
554558

‎expected/backup.out

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,9 @@ page-level backup without validated full backup
3232
1
3333
0
3434
0
35+
###### BACKUP COMMAND TEST-0006 ######
36+
###### ptrack backup mode ######
37+
0
38+
0
39+
2
40+
6

‎expected/restore.out

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,19 @@ OK: recovery.conf has the given target timeline.
3030
OK: recovery-target-xid options works well.
3131

3232
###### RESTORE COMMAND TEST-0006 ######
33+
###### recovery to latest from full + ptrack backups ######
34+
0
35+
0
36+
0
37+
38+
###### RESTORE COMMAND TEST-0007 ######
39+
###### recovery to latest from full + ptrack + ptrack backups ######
40+
0
41+
0
42+
0
43+
0
44+
45+
###### RESTORE COMMAND TEST-0008 ######
3346
###### recovery with target inclusive false ######
3447
0
3548
0

‎pg_arman.h

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
#include "utils/pg_crc.h"
2222
#include "parray.h"
2323
#include "datapagemap.h"
24+
#include "storage/bufpage.h"
25+
#include "storage/block.h"
2426

2527
/* Query to fetch current transaction ID */
2628
#define TXID_CURRENT_SQL "SELECT txid_current();"
@@ -55,7 +57,8 @@ typedef struct pgFile
5557
pg_crc32 crc; /* CRC value of the file, regular file only */
5658
char *linked; /* path of the linked file */
5759
bool is_datafile; /* true if the file is PostgreSQL data file */
58-
char *path; /* path of the file */
60+
char *path; /* path of the file */
61+
char *ptrack_path;
5962
datapagemap_t pagemap;
6063
} pgFile;
6164

@@ -96,6 +99,7 @@ typedef enum BackupMode
9699
{
97100
BACKUP_MODE_INVALID,
98101
BACKUP_MODE_DIFF_PAGE, /* differential page backup */
102+
BACKUP_MODE_DIFF_PTRACK, /* differential page backup with ptrack system*/
99103
BACKUP_MODE_FULL /* full backup */
100104
} BackupMode;
101105

@@ -161,6 +165,11 @@ typedef struct pgRecoveryTarget
161165
bool recovery_target_inclusive;
162166
} pgRecoveryTarget;
163167

168+
typedef union DataPage
169+
{
170+
PageHeaderData page_data;
171+
char data[BLCKSZ];
172+
} DataPage;
164173

165174
/*
166175
* return pointer that exceeds the length of prefix from character string.

‎restore.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,8 @@ do_restore(const char *target_time,
166166
continue;
167167

168168
/* use database backup only */
169-
if (backup->backup_mode != BACKUP_MODE_DIFF_PAGE)
169+
if (backup->backup_mode != BACKUP_MODE_DIFF_PAGE &&
170+
backup->backup_mode != BACKUP_MODE_DIFF_PTRACK)
170171
continue;
171172

172173
/* is the backup is necessary for restore to target timeline ? */

‎show.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ show_backup_list(FILE *out, parray *backup_list, bool show_all)
176176
for (i = 0; i < parray_num(backup_list); i++)
177177
{
178178
pgBackup *backup;
179-
const char *modes[] = { "", "PAGE", "FULL"};
179+
const char *modes[] = { "", "PAGE", "PTRACK", "FULL"};
180180
TimeLineID parent_tli;
181181
char timestamp[20];
182182
char duration[20] = "----";
@@ -204,7 +204,7 @@ show_backup_list(FILE *out, parray *backup_list, bool show_all)
204204
/* Get parent timeline before printing */
205205
parent_tli = get_parent_tli(backup->tli);
206206

207-
fprintf(out, "%-19s %-4s %10d %10d %5s %6s %s \n",
207+
fprintf(out, "%-19s %-6s %10d %10d %5s %6s %s \n",
208208
timestamp,
209209
modes[backup->backup_mode],
210210
backup->tli,

‎sql/backup.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,17 @@ pg_arman show -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0005.log 2>&1
7474
grep OK ${TEST_BASE}/TEST-0005.log | grep FULL | wc -l | sed 's/^ *//'
7575
grep ERROR ${TEST_BASE}/TEST-0005.log | grep INCR | wc -l | sed 's/^ *//'
7676

77+
echo '###### BACKUP COMMAND TEST-0006 ######'
78+
echo '###### ptrack backup mode ######'
79+
init_catalog
80+
pg_arman backup -B ${BACKUP_PATH} -b full -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0006-run.log 2>&1;echo $?
81+
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.log 2>&1
82+
pg_arman backup -B ${BACKUP_PATH} -b ptrack -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0006-run.log 2>&1;echo $?
83+
pg_arman validate -B ${BACKUP_PATH} >> ${TEST_BASE}/TEST-0006-run.log 2>&1
84+
pg_arman show -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0006.log 2>&1
85+
grep -c OK ${TEST_BASE}/TEST-0006.log
86+
grep OK ${TEST_BASE}/TEST-0006.log | sed -e 's@[^-]@@g' | wc -c | sed 's/^ *//'
87+
7788
# cleanup
7889
## clean up the temporal test data
7990
pg_ctl stop -m immediate -D ${PGDATA_PATH} > /dev/null 2>&1

‎sql/common.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ wal_level = hot_standby
7373
wal_log_hints = on
7474
archive_mode = on
7575
archive_command = 'cp %p ${ARCLOG_PATH}/%f'
76+
ptrack_enable = on
7677
EOF
7778

7879
# start PostgreSQL

0 commit comments

Comments
 (0)