Skip to content

Commit 091fe0a

Browse files
stalkergstalkerg
committed
Fix backup many segments of relation (big relations) with ptrack.
Add fail test for show problem in lost pages in backup process.
1 parent 5698515 commit 091fe0a

File tree

6 files changed

+70
-19
lines changed

6 files changed

+70
-19
lines changed

‎backup.c

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,6 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
110110
*/
111111
current.tli = get_current_timeline(false);
112112

113-
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK)
114-
pg_ptrack_clear();
115-
116113
/*
117114
* In differential backup mode, check if there is an already-validated
118115
* full backup on current timeline.
@@ -148,6 +145,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
148145
elog(ERROR, "backup_label does not exist in PGDATA.");
149146
}
150147

148+
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK)
149+
pg_ptrack_clear();
151150
/*
152151
* List directories and symbolic links with the physical path to make
153152
* mkdirs.sh, then sort them in order of path. Omit $PGDATA.
@@ -915,18 +914,31 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
915914
path_len = strlen(file->path);
916915
if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
917916
{
918-
pgFile tmp_file;
919917
pgFile *search_file;
920-
//elog(WARNING, "Remove ptrack file from backup %s", file->path);
921-
tmp_file.path = pg_strdup(file->path);
922-
tmp_file.path[path_len-7] = '\0';
923-
search_file = *(pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
924-
if (search_file != NULL)
925-
{
926-
//elog(WARNING, "Finded main fork for ptrak:%s", search_file->path);
927-
search_file->ptrack_path = pg_strdup(file->path);
918+
pgFile **pre_search_file;
919+
int segno = 0;
920+
while(true) {
921+
pgFile tmp_file;
922+
tmp_file.path = pg_strdup(file->path);
923+
/* I hope segno not more than 999999 */
924+
if (segno > 0)
925+
sprintf(tmp_file.path+path_len-7, ".%d", segno);
926+
else
927+
tmp_file.path[path_len-7] = '\0';
928+
pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
929+
if (pre_search_file != NULL)
930+
{
931+
search_file = *pre_search_file;
932+
search_file->ptrack_path = pg_strdup(file->path);
933+
search_file->segno = segno;
934+
} else {
935+
pg_free(tmp_file.path);
936+
break;
937+
}
938+
pg_free(tmp_file.path);
939+
segno++;
928940
}
929-
free(tmp_file.path);
941+
930942
pgFileFree(file);
931943
parray_remove(list_file, i);
932944
i--;
@@ -1051,7 +1063,11 @@ void make_pagemap_from_ptrack(parray *files)
10511063
if (p->ptrack_path != NULL)
10521064
{
10531065
DataPage page;
1066+
char *flat_memory, *flat_mamory_cur;
1067+
size_t flat_size = 0;
1068+
size_t start_addr;
10541069
struct stat st;
1070+
10551071
FILE *ptrack_file = fopen(p->ptrack_path, "r");
10561072
if (ptrack_file == NULL)
10571073
{
@@ -1060,16 +1076,21 @@ void make_pagemap_from_ptrack(parray *files)
10601076
}
10611077

10621078
fstat(fileno(ptrack_file), &st);
1063-
p->pagemap.bitmapsize = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
1064-
p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
1079+
flat_size = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
1080+
flat_mamory_cur = flat_memory = pg_malloc(flat_size);
10651081

1066-
elog(LOG, "Start copy bitmap from ptrack:%s size:%i", p->ptrack_path, p->pagemap.bitmapsize);
10671082
while(fread(page.data, BLCKSZ, 1, ptrack_file) == 1)
10681083
{
10691084
char *map = PageGetContents(page.data);
1070-
memcpy(p->pagemap.bitmap, map, BLCKSZ - MAXALIGN(SizeOfPageHeaderData));
1085+
memcpy(flat_memory, map, MAPSIZE);
1086+
flat_mamory_cur += MAPSIZE;
10711087
}
10721088
fclose(ptrack_file);
1089+
start_addr = (RELSEG_SIZE/8)*p->segno;
1090+
p->pagemap.bitmapsize = start_addr+RELSEG_SIZE/8 > flat_size ? flat_size - start_addr : RELSEG_SIZE/8;
1091+
p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
1092+
memcpy(p->pagemap.bitmap, flat_memory+start_addr, p->pagemap.bitmapsize);
1093+
pg_free(flat_memory);
10731094
}
10741095
}
10751096
}

‎data.c

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -213,8 +213,6 @@ backup_data_file(const char *from_root, const char *to_root,
213213
return copy_file(from_root, to_root, file);
214214
}
215215

216-
217-
218216
/* if the page has not been modified since last backup, skip it */
219217
if (lsn && !XLogRecPtrIsInvalid(page_lsn) && page_lsn < *lsn)
220218
continue;

‎dir.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ pgFileNew(const char *path, bool omit_symlink)
8888
file->pagemap.bitmap = NULL;
8989
file->pagemap.bitmapsize = 0;
9090
file->ptrack_path = NULL;
91+
file->segno = 0;
9192
file->path = pgut_malloc(strlen(path) + 1);
9293
strcpy(file->path, path); /* enough buffer size guaranteed */
9394

@@ -549,6 +550,7 @@ dir_read_file_list(const char *root, const char *file_txt)
549550
file = (pgFile *) pgut_malloc(sizeof(pgFile));
550551
file->path = pgut_malloc((root ? strlen(root) + 1 : 0) + strlen(path) + 1);
551552
file->ptrack_path = NULL;
553+
file->segno = 0;
552554
file->pagemap.bitmap = NULL;
553555
file->pagemap.bitmapsize = 0;
554556

‎expected/restore.out

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ OK: recovery-target-xid options works well.
4242
0
4343
0
4444

45+
###### RESTORE COMMAND TEST-0009 ######
46+
###### recovery to latest from full + ptrack backups with loads when full backup do ######
47+
0
48+
0
49+
0
50+
4551
###### RESTORE COMMAND TEST-0008 ######
4652
###### recovery with target inclusive false ######
4753
0

‎pg_arman.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ typedef struct pgFile
5959
bool is_datafile; /* true if the file is PostgreSQL data file */
6060
char *path; /* path of the file */
6161
char *ptrack_path;
62+
int segno; /* Segment number for ptrack */
6263
datapagemap_t pagemap;
6364
} pgFile;
6465

@@ -81,6 +82,9 @@ typedef struct pgBackupRange
8182
(tm.tm_mon >= 0 && tm.tm_mon <= 11) && /* range check for tm_mon (0-23) */ \
8283
(tm.tm_year + 1900 >= 1900)) /* range check for tm_year(70-) */
8384

85+
/* Effective data size */
86+
#define MAPSIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData))
87+
8488
/* Backup status */
8589
/* XXX re-order ? */
8690
typedef enum BackupStatus

‎sql/restore.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,26 @@ psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches
170170
diff ${TEST_BASE}/TEST-0007-before.out ${TEST_BASE}/TEST-0007-after.out
171171
echo ''
172172

173+
echo '###### RESTORE COMMAND TEST-0009 ######'
174+
echo '###### recovery to latest from full + ptrack backups with loads when full backup do ######'
175+
init_backup
176+
pgbench_objs 0009
177+
pgbench -p ${TEST_PGPORT} -d pgbench -c 4 -T 8 > /dev/null 2>&1 &
178+
PGBENCH_PID=$!
179+
pg_arman backup -B ${BACKUP_PATH} -b full -j 4 -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0009-run.out 2>&1;echo $?
180+
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0009-run.out 2>&1
181+
#kill $PGBENCH_PID 2> /dev/null
182+
sleep 12
183+
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT count(*) FROM pgbench_history;" > ${TEST_BASE}/TEST-0009-count1.out
184+
pg_arman backup -B ${BACKUP_PATH} -b ptrack -j 4 -p ${TEST_PGPORT} -d postgres --verbose >> ${TEST_BASE}/TEST-0009-run.out 2>&1;echo $?
185+
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1
186+
pg_ctl stop -m immediate > /dev/null 2>&1
187+
pg_arman restore -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1;echo $?
188+
pg_ctl start -w -t 600 > /dev/null 2>&1
189+
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT count(*) FROM pgbench_history;" > ${TEST_BASE}/TEST-0009-count2.out
190+
diff ${TEST_BASE}/TEST-0009-count1.out ${TEST_BASE}/TEST-0009-count2.out
191+
echo ''
192+
173193
echo '###### RESTORE COMMAND TEST-0008 ######'
174194
echo '###### recovery with target inclusive false ######'
175195
init_backup

0 commit comments

Comments
 (0)