Skip to content

Commit b26b0f0

Browse files
dmpgprodmpgpro
committed
Merge commit '9d273330d3fa633b650d30048d2822abd93ccfd0' into PGPRO9_6
2 parents 75532ba + 9d27333 commit b26b0f0

File tree

11 files changed

+135
-51
lines changed

11 files changed

+135
-51
lines changed

‎contrib/pg_arman/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ OBJS = backup.o \
2121
pgut/pgut.o \
2222
pgut/pgut-port.o
2323

24-
EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c receivelog.h streamutil.h logging.h
24+
EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c receivelog.c receivelog.h streamutil.c streamutil.h logging.h
2525

2626
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS}
2727
override CPPFLAGS := -DFRONTEND $(CPPFLAGS)

‎contrib/pg_arman/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,12 @@ some overhead to PostgreSQL performance. On our experiments it appears to be
7575
less than 3%.
7676

7777
These two approaches were implemented in this fork of pg_arman. The second
78-
approach requires [patch for PostgreSQL](https://gist.github.com/stalkerg/7ce2394c4f3b36f995895190a633b4fa).
78+
approach requires [patch for PostgreSQL 9.5](https://gist.github.com/stalkerg/44703dbcbac1da08f448b7e6966646c0).
7979

8080
Testing block level incremental backup
8181
--------------------------------------
8282

83-
You need build and install [PGPRO9_5_ptrack branch of PostgreSQL](https://github.com/postgrespro/postgrespro/tree/PGPRO9_5_ptrack) or [apply this patch to PostgreSQL 9.5](https://gist.github.com/stalkerg/7ce2394c4f3b36f995895190a633b4fa).
83+
You need build and install [PGPRO9_5 branch of PostgreSQL](https://github.com/postgrespro/postgrespro) or [apply this patch to PostgreSQL 9.5](https://gist.github.com/stalkerg/44703dbcbac1da08f448b7e6966646c0).
8484

8585
### Retrieving changed blocks from WAL archive
8686

‎contrib/pg_arman/backup.c

Lines changed: 49 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr;
3838
const char *progname = "pg_arman";
3939

4040
/* list of files contained in backup */
41-
parray *backup_files_list;
41+
parray *backup_files_list;
42+
static volatile uint32 total_copy_files_increment;
43+
static uint32 total_files_num;
4244

4345
typedef struct
4446
{
@@ -47,8 +49,6 @@ typedef struct
4749
parray *files;
4850
parray *prev_files;
4951
const XLogRecPtr *lsn;
50-
unsigned int start_file_idx;
51-
unsigned int end_file_idx;
5252
} backup_files_args;
5353

5454
/*
@@ -106,6 +106,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
106106
pthread_t stream_thread;
107107
backup_files_args *backup_threads_args[num_threads];
108108

109+
109110
/* repack the options */
110111
bool smooth_checkpoint = bkupopt.smooth_checkpoint;
111112
pgBackup *prev_backup = NULL;
@@ -146,7 +147,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
146147
}
147148

148149
/* clear ptrack files for FULL and DIFF backup */
149-
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK)
150+
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && !disable_ptrack_clear)
150151
pg_ptrack_clear();
151152

152153
/* start stream replication */
@@ -310,11 +311,21 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
310311
if (!check)
311312
dir_create_dir(dirpath, DIR_PERMISSION);
312313
}
314+
else
315+
{
316+
total_files_num++;
317+
}
318+
319+
__sync_lock_release(&file->lock);
313320
}
314321

315322
if (num_threads < 1)
316323
num_threads = 1;
317324

325+
/* sort by size for load balancing */
326+
parray_qsort(backup_files_list, pgFileCompareSize);
327+
328+
/* init thread args with own file lists */
318329
for (i = 0; i < num_threads; i++)
319330
{
320331
backup_files_args *arg = pg_malloc(sizeof(backup_files_args));
@@ -323,19 +334,17 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
323334
arg->files = backup_files_list;
324335
arg->prev_files = prev_files;
325336
arg->lsn = lsn;
326-
arg->start_file_idx = i * (parray_num(backup_files_list)/num_threads);
327-
if (i == num_threads - 1)
328-
arg->end_file_idx = parray_num(backup_files_list);
329-
else
330-
arg->end_file_idx = (i + 1) * (parray_num(backup_files_list)/num_threads);
337+
backup_threads_args[i] = arg;
338+
}
331339

340+
total_copy_files_increment = 0;
341+
342+
/* Run threads */
343+
for (i = 0; i < num_threads; i++)
344+
{
332345
if (verbose)
333-
elog(WARNING, "Start thread for start_file_idx:%i end_file_idx:%i num:%li",
334-
arg->start_file_idx,
335-
arg->end_file_idx,
336-
parray_num(backup_files_list));
337-
backup_threads_args[i] = arg;
338-
pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) backup_files, arg);
346+
elog(WARNING, "Start thread num:%li", parray_num(backup_threads_args[i]->files));
347+
pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) backup_files, backup_threads_args[i]);
339348
}
340349

341350
/* Wait theads */
@@ -345,6 +354,9 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
345354
pg_free(backup_threads_args[i]);
346355
}
347356

357+
if (progress)
358+
fprintf(stderr, "\n");
359+
348360
/* Notify end of backup */
349361
pg_stop_backup(&current);
350362

@@ -911,12 +923,14 @@ backup_files(void *arg)
911923
gettimeofday(&tv, NULL);
912924

913925
/* backup a file or create a directory */
914-
for (i = arguments->start_file_idx; i < arguments->end_file_idx; i++)
926+
for (i = 0; i < parray_num(arguments->files); i++)
915927
{
916928
int ret;
917929
struct stat buf;
918930

919931
pgFile *file = (pgFile *) parray_get(arguments->files, i);
932+
if (__sync_lock_test_and_set(&file->lock, 1) != 0)
933+
continue;
920934

921935
/* If current time is rewinded, abort this backup. */
922936
if (tv.tv_sec < file->mtime)
@@ -1007,6 +1021,9 @@ backup_files(void *arg)
10071021
}
10081022
else
10091023
elog(LOG, "unexpected file type %d", buf.st_mode);
1024+
if (progress)
1025+
fprintf(stderr, "\rProgress %i/%u", total_copy_files_increment, total_files_num-1);
1026+
__sync_fetch_and_add(&total_copy_files_increment, 1);
10101027
}
10111028
}
10121029

@@ -1089,6 +1106,20 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
10891106
continue;
10901107

10911108
file->is_datafile = true;
1109+
{
1110+
int find_dot;
1111+
char *text_segno;
1112+
for(find_dot = path_len-1; file->path[find_dot] != '.' && find_dot >= 0; find_dot--);
1113+
if (find_dot <= 0)
1114+
continue;
1115+
1116+
text_segno = file->path + find_dot + 1;
1117+
/* in future we will need check all chars */
1118+
if (!isdigit(text_segno[0]))
1119+
continue;
1120+
1121+
file->segno = (int) strtol(text_segno, NULL, 10);
1122+
}
10921123
}
10931124
parray_concat(files, list_file);
10941125
}
@@ -1334,13 +1365,13 @@ StreamLog(void *arg)
13341365
ctl.basedir = basedir;
13351366
ctl.stream_stop = stop_streaming;
13361367
ctl.standby_message_timeout = standby_message_timeout;
1337-
ctl.partial_suffix = ".partial";
1368+
ctl.partial_suffix = NULL;
13381369
ctl.synchronous = false;
13391370
ctl.mark_done = false;
13401371
ReceiveXlogStream(conn, &ctl);
13411372
#else
13421373
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
1343-
stop_streaming, standby_message_timeout, ".partial",
1374+
stop_streaming, standby_message_timeout, NULL,
13441375
false, false);
13451376
#endif
13461377

‎contrib/pg_arman/data.c

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ backup_data_file(const char *from_root, const char *to_root,
7272
size_t read_len = 0;
7373
pg_crc32 crc;
7474
off_t offset;
75+
char write_buffer[sizeof(header)+BLCKSZ];
76+
size_t write_buffer_real_size;
7577

7678
INIT_CRC32C(crc);
7779

@@ -185,12 +187,14 @@ backup_data_file(const char *from_root, const char *to_root,
185187
elog(ERROR, "File: %s blknum %u have wrong page header.", file->path, blknum);
186188
}
187189

190+
188191
if(current.checksum_version &&
189-
pg_checksum_page(page.data, header.block) != ((PageHeader) page.data)->pd_checksum)
192+
pg_checksum_page(page.data, file->segno * RELSEG_SIZE + blknum) != ((PageHeader) page.data)->pd_checksum)
190193
{
191194
if (try_checksum)
192195
{
193196
elog(WARNING, "File: %s blknum %u have wrong checksum, try again", file->path, blknum);
197+
usleep(100);
194198
fseek(in, -sizeof(page), SEEK_CUR);
195199
fread(&page, 1, sizeof(page), in);
196200
}
@@ -211,10 +215,15 @@ backup_data_file(const char *from_root, const char *to_root,
211215
upper_offset = header.hole_offset + header.hole_length;
212216
upper_length = BLCKSZ - upper_offset;
213217

218+
write_buffer_real_size = sizeof(header)+header.hole_offset+upper_length;
219+
memcpy(write_buffer, &header, sizeof(header));
220+
if (header.hole_offset)
221+
memcpy(write_buffer+sizeof(header), page.data, header.hole_offset);
222+
if (upper_length)
223+
memcpy(write_buffer+sizeof(header)+header.hole_offset, page.data + upper_offset, upper_length);
224+
214225
/* write data page excluding hole */
215-
if (fwrite(&header, 1, sizeof(header), out) != sizeof(header) ||
216-
fwrite(page.data, 1, header.hole_offset, out) != header.hole_offset ||
217-
fwrite(page.data + upper_offset, 1, upper_length, out) != upper_length)
226+
if(fwrite(write_buffer, 1, write_buffer_real_size, out) != write_buffer_real_size)
218227
{
219228
int errno_tmp = errno;
220229
/* oops */
@@ -273,8 +282,6 @@ backup_data_file(const char *from_root, const char *to_root,
273282
int i;
274283

275284
for(i=0; i<BLCKSZ && page.data[i] == 0; i++);
276-
277-
278285
if (i == BLCKSZ)
279286
{
280287
elog(WARNING, "File: %s blknum %u, empty page", file->path, blknum);
@@ -298,6 +305,7 @@ backup_data_file(const char *from_root, const char *to_root,
298305
if (try_checksum)
299306
{
300307
elog(WARNING, "File: %s blknum %u have wrong page header, try again", file->path, blknum);
308+
usleep(100);
301309
fseek(in, -sizeof(page), SEEK_CUR);
302310
fread(&page, 1, sizeof(page), in);
303311
continue;
@@ -307,7 +315,7 @@ backup_data_file(const char *from_root, const char *to_root,
307315
}
308316

309317
if(current.checksum_version &&
310-
pg_checksum_page(page.data, header.block) != ((PageHeader) page.data)->pd_checksum)
318+
pg_checksum_page(page.data, file->segno * RELSEG_SIZE + blknum) != ((PageHeader) page.data)->pd_checksum)
311319
{
312320
if (try_checksum)
313321
elog(WARNING, "File: %s blknum %u have wrong checksum, try again", file->path, blknum);
@@ -330,10 +338,15 @@ backup_data_file(const char *from_root, const char *to_root,
330338
upper_offset = header.hole_offset + header.hole_length;
331339
upper_length = BLCKSZ - upper_offset;
332340

341+
write_buffer_real_size = sizeof(header)+header.hole_offset+upper_length;
342+
memcpy(write_buffer, &header, sizeof(header));
343+
if (header.hole_offset)
344+
memcpy(write_buffer+sizeof(header), page.data, header.hole_offset);
345+
if (upper_length)
346+
memcpy(write_buffer+sizeof(header)+header.hole_offset, page.data + upper_offset, upper_length);
347+
333348
/* write data page excluding hole */
334-
if (fwrite(&header, 1, sizeof(header), out) != sizeof(header) ||
335-
fwrite(page.data, 1, header.hole_offset, out) != header.hole_offset ||
336-
fwrite(page.data + upper_offset, 1, upper_length, out) != upper_length)
349+
if(fwrite(write_buffer, 1, write_buffer_real_size, out) != write_buffer_real_size)
337350
{
338351
int errno_tmp = errno;
339352
/* oops */

‎contrib/pg_arman/dir.c

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const char *pgdata_exclude[] =
2828
NULL, /* arclog_path will be set later */
2929
NULL, /* 'pg_tblspc' will be set later */
3030
NULL, /* sentinel */
31+
NULL
3132
};
3233

3334
static pgFile *pgFileNew(const char *path, bool omit_symlink);
@@ -192,6 +193,21 @@ pgFileComparePathDesc(const void *f1, const void *f2)
192193
return -pgFileComparePath(f1, f2);
193194
}
194195

196+
/* Compare two pgFile with their size */
197+
int
198+
pgFileCompareSize(const void *f1, const void *f2)
199+
{
200+
pgFile *f1p = *(pgFile **)f1;
201+
pgFile *f2p = *(pgFile **)f2;
202+
203+
if (f1p->size > f2p->size)
204+
return 1;
205+
else if (f1p->size < f2p->size)
206+
return -1;
207+
else
208+
return 0;
209+
}
210+
195211
/* Compare two pgFile with their modify timestamp. */
196212
int
197213
pgFileCompareMtime(const void *f1, const void *f2)
@@ -260,9 +276,13 @@ dir_list_file(parray *files, const char *root, const char *exclude[], bool omit_
260276
fclose(black_list_file);
261277
parray_qsort(black_list, BlackListCompare);
262278
dir_list_file_internal(files, root, exclude, omit_symlink, add_root, black_list);
279+
parray_qsort(files, pgFileComparePath);
263280
}
264281
else
282+
{
265283
dir_list_file_internal(files, root, exclude, omit_symlink, add_root, NULL);
284+
parray_qsort(files, pgFileComparePath);
285+
}
266286
}
267287

268288
void
@@ -406,8 +426,6 @@ dir_list_file_internal(parray *files, const char *root, const char *exclude[],
406426

407427
break; /* pseudo loop */
408428
}
409-
410-
parray_qsort(files, pgFileComparePath);
411429
}
412430

413431
/* print mkdirs.sh */

‎contrib/pg_arman/doc/pg_arman.md

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,10 @@ absolute paths; relative paths are not allowed.
244244
Number of threads for backup.
245245

246246
**--stream**:
247-
Enable stream replication for save WAL during backup process.
247+
Enable stream replication for save WAL during backup process.
248+
249+
**--disable-ptrack-clear**:
250+
Disable clear ptrack files for postgres without ptrack patch.
248251

249252
### RESTORE OPTIONS
250253

@@ -272,7 +275,7 @@ the same parameters as the ones in recovery.confin recovery.conf.
272275
**--stream**:
273276
Restore without recovery.conf and use pg_xlog WALs. Before you need
274277
backup with **--stream** option. This option will disable all **--recovery-**
275-
options.
278+
options.
276279

277280
### CATALOG OPTIONS
278281

‎contrib/pg_arman/expected/option.out

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@ Common Options:
1818
-c, --check show what would have been done
1919
-j, --threads=NUM num threads for backup and restore
2020
--stream use stream for save/restore WAL during backup
21+
--progress show progress copy files
2122

2223
Backup options:
2324
-b, --backup-mode=MODE full,page,ptrack
2425
-C, --smooth-checkpoint do smooth checkpoint before backup
2526
--validate validate backup after taking it
2627
--keep-data-generations=N keep GENERATION of full data backup
2728
--keep-data-days=DAY keep enough data backup to recover to DAY days age
29+
--disable-ptrack-clear disable clear ptrack for postgres without ptrack
30+
--backup-pg-log start backup pg_log directory
2831

2932
Restore options:
3033
--recovery-target-time time stamp up to which recovery will proceed

0 commit comments

Comments
 (0)