Skip to content

Commit 269a83c

Browse files
s.logvinenkos.logvinenko
s.logvinenko
authored and
s.logvinenko
committed
Merge branch 'master' of https://git.postgrespro.ru/pgpro-dev/pg_probackup into cfs
Conflicts: tests/__init__.py
2 parents 7bfa61a + 9813471 commit 269a83c

File tree

13 files changed

+1119
-792
lines changed

13 files changed

+1119
-792
lines changed

‎src/backup.c

Lines changed: 703 additions & 384 deletions
Large diffs are not rendered by default.

‎src/data.c

Lines changed: 33 additions & 255 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,26 @@
2222
#include <common/pg_lzcompress.h>
2323
#include <zlib.h>
2424

25+
/* Implementation of zlib compression method */
2526
static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
2627
{
2728
uLongf compressed_size = dst_size;
2829
int rc = compress2(dst, &compressed_size, src, src_size, compress_level);
2930
return rc == Z_OK ? compressed_size : rc;
3031
}
3132

33+
/* Implementation of zlib compression method */
3234
static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
3335
{
3436
uLongf dest_len = dst_size;
3537
int rc = uncompress(dst, &dest_len, src, src_size);
3638
return rc == Z_OK ? dest_len : rc;
3739
}
3840

41+
/*
42+
* Compresses source into dest using algorithm. Returns the number of bytes
43+
* written in the destination buffer, or -1 if compression fails.
44+
*/
3945
static size_t
4046
do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg)
4147
{
@@ -53,6 +59,10 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre
5359
return -1;
5460
}
5561

62+
/*
63+
* Decompresses source into dest using algorithm. Returns the number of bytes
64+
* decompressed in the destination buffer, or -1 if decompression fails.
65+
*/
5666
static size_t
5767
do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg)
5868
{
@@ -70,8 +80,10 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, Comp
7080
return -1;
7181
}
7282

73-
74-
83+
/*
84+
* When copying datafiles to backup we validate and compress them block
85+
* by block. Thus special header is required for each data block.
86+
*/
7587
typedef struct BackupPageHeader
7688
{
7789
BlockNumber block; /* block number */
@@ -125,6 +137,11 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
125137
header.block = blknum;
126138
offset = blknum * BLCKSZ;
127139

140+
/*
141+
* Read the page and verify its header and checksum.
142+
* Under high write load it's possible that we've read partly
143+
* flushed page, so try several times befor throwing an error.
144+
*/
128145
while(try_checksum--)
129146
{
130147
if (fseek(in, offset, SEEK_SET) != 0)
@@ -176,14 +193,14 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
176193
elog(ERROR, "File: %s blknum %u have wrong page header.", file->path, blknum);
177194
}
178195

179-
/* If the page hasn't changed since previous backup, don't backup it. */
180-
if (!XLogRecPtrIsInvalid(prev_backup_start_lsn)
181-
&& !XLogRecPtrIsInvalid(page_lsn)
182-
&& page_lsn < prev_backup_start_lsn)
183-
{
184-
*n_skipped += 1;
185-
return;
186-
}
196+
// /* If the page hasn't changed since previous backup, don't backup it. */
197+
// if (!XLogRecPtrIsInvalid(prev_backup_start_lsn)
198+
// && !XLogRecPtrIsInvalid(page_lsn)
199+
// && page_lsn < prev_backup_start_lsn)
200+
// {
201+
// *n_skipped += 1;
202+
// return;
203+
// }
187204

188205
/* Verify checksum */
189206
if(current.checksum_version)
@@ -223,12 +240,14 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
223240
Assert (header.compressed_size <= BLCKSZ);
224241
write_buffer_size = sizeof(header);
225242

243+
/* The page was successfully compressed */
226244
if (header.compressed_size > 0)
227245
{
228246
memcpy(write_buffer, &header, sizeof(header));
229247
memcpy(write_buffer + sizeof(header), compressed_page.data, header.compressed_size);
230248
write_buffer_size += MAXALIGN(header.compressed_size);
231249
}
250+
/* The page compression failed. Write it as is. */
232251
else
233252
{
234253
header.compressed_size = BLCKSZ;
@@ -324,8 +343,7 @@ backup_data_file(const char *from_root, const char *to_root,
324343

325344
/*
326345
* Read each page, verify checksum and write it to backup.
327-
* If page map is not empty we scan only changed blocks, otherwise
328-
* backup all pages of the relation.
346+
* If page map is empty backup all pages of the relation.
329347
*/
330348
if (file->pagemap.bitmapsize == 0)
331349
{
@@ -336,6 +354,7 @@ backup_data_file(const char *from_root, const char *to_root,
336354
n_blocks_read++;
337355
}
338356
}
357+
/* If page map is not empty we scan only changed blocks, */
339358
else
340359
{
341360
datapagemap_iterator_t *iter;
@@ -371,7 +390,7 @@ backup_data_file(const char *from_root, const char *to_root,
371390
FIN_CRC32C(file->crc);
372391

373392
/*
374-
* If we have pagemap then file can't be a zero size.
393+
* If we have pagemap then file in the backup can't be a zero size.
375394
* Otherwise, we will clear the last file.
376395
*/
377396
if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped)
@@ -385,117 +404,6 @@ backup_data_file(const char *from_root, const char *to_root,
385404
return true;
386405
}
387406

388-
/*
389-
* Restore compressed file that was backed up partly.
390-
*/
391-
static void
392-
restore_file_partly(const char *from_root,const char *to_root, pgFile *file)
393-
{
394-
FILE *in;
395-
FILE *out;
396-
size_t read_len = 0;
397-
int errno_tmp;
398-
struct stat st;
399-
char to_path[MAXPGPATH];
400-
char buf[BLCKSZ];
401-
size_t write_size = 0;
402-
403-
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
404-
/* open backup mode file for read */
405-
in = fopen(file->path, "r");
406-
if (in == NULL)
407-
{
408-
elog(ERROR, "cannot open backup file \"%s\": %s", file->path,
409-
strerror(errno));
410-
}
411-
out = fopen(to_path, "r+");
412-
413-
/* stat source file to change mode of destination file */
414-
if (fstat(fileno(in), &st) == -1)
415-
{
416-
fclose(in);
417-
fclose(out);
418-
elog(ERROR, "cannot stat \"%s\": %s", file->path,
419-
strerror(errno));
420-
}
421-
422-
if (fseek(out, 0, SEEK_END) < 0)
423-
elog(ERROR, "cannot seek END of \"%s\": %s",
424-
to_path, strerror(errno));
425-
426-
/* copy everything from backup to the end of the file */
427-
for (;;)
428-
{
429-
if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf))
430-
break;
431-
432-
if (fwrite(buf, 1, read_len, out) != read_len)
433-
{
434-
errno_tmp = errno;
435-
/* oops */
436-
fclose(in);
437-
fclose(out);
438-
elog(ERROR, "cannot write to \"%s\": %s", to_path,
439-
strerror(errno_tmp));
440-
}
441-
write_size += read_len;
442-
}
443-
444-
errno_tmp = errno;
445-
if (!feof(in))
446-
{
447-
fclose(in);
448-
fclose(out);
449-
elog(ERROR, "cannot read backup mode file \"%s\": %s",
450-
file->path, strerror(errno_tmp));
451-
}
452-
453-
/* copy odd part. */
454-
if (read_len > 0)
455-
{
456-
if (fwrite(buf, 1, read_len, out) != read_len)
457-
{
458-
errno_tmp = errno;
459-
/* oops */
460-
fclose(in);
461-
fclose(out);
462-
elog(ERROR, "cannot write to \"%s\": %s", to_path,
463-
strerror(errno_tmp));
464-
}
465-
466-
write_size += read_len;
467-
}
468-
469-
470-
/* update file permission */
471-
if (chmod(to_path, file->mode) == -1)
472-
{
473-
int errno_tmp = errno;
474-
475-
fclose(in);
476-
fclose(out);
477-
elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
478-
strerror(errno_tmp));
479-
}
480-
481-
if (fflush(out) != 0 ||
482-
fsync(fileno(out)) != 0 ||
483-
fclose(out))
484-
elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno));
485-
fclose(in);
486-
}
487-
488-
void
489-
restore_compressed_file(const char *from_root,
490-
const char *to_root,
491-
pgFile *file)
492-
{
493-
if (!file->is_partial_copy)
494-
copy_file(from_root, to_root, file);
495-
else
496-
restore_file_partly(from_root, to_root, file);
497-
}
498-
499407
/*
500408
* Restore files in the from_root directory to the to_root directory with
501409
* same relative path.
@@ -522,7 +430,7 @@ restore_data_file(const char *from_root,
522430

523431
/*
524432
* Open backup file for write. We use "r+" at first to overwrite only
525-
* modified pages for differential restore. If the file is not exists,
433+
* modified pages for differential restore. If the file does not exist,
526434
* re-open it with "w" to create an empty file.
527435
*/
528436
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
@@ -856,136 +764,6 @@ copy_wal_file(const char *from_path, const char *to_path)
856764
fclose(in);
857765
}
858766

859-
/*
860-
* Save part of the file into backup.
861-
* skip_size - size of the file in previous backup. We can skip it
862-
* and copy just remaining part of the file
863-
*/
864-
bool
865-
copy_file_partly(const char *from_root, const char *to_root,
866-
pgFile *file, size_t skip_size)
867-
{
868-
char to_path[MAXPGPATH];
869-
FILE *in;
870-
FILE *out;
871-
size_t read_len = 0;
872-
int errno_tmp;
873-
struct stat st;
874-
char buf[BLCKSZ];
875-
876-
/* reset size summary */
877-
file->read_size = 0;
878-
file->write_size = 0;
879-
880-
/* open backup mode file for read */
881-
in = fopen(file->path, "r");
882-
if (in == NULL)
883-
{
884-
/* maybe deleted, it's not error */
885-
if (errno == ENOENT)
886-
return false;
887-
888-
elog(ERROR, "cannot open source file \"%s\": %s", file->path,
889-
strerror(errno));
890-
}
891-
892-
/* open backup file for write */
893-
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
894-
895-
out = fopen(to_path, "w");
896-
if (out == NULL)
897-
{
898-
int errno_tmp = errno;
899-
fclose(in);
900-
elog(ERROR, "cannot open destination file \"%s\": %s",
901-
to_path, strerror(errno_tmp));
902-
}
903-
904-
/* stat source file to change mode of destination file */
905-
if (fstat(fileno(in), &st) == -1)
906-
{
907-
fclose(in);
908-
fclose(out);
909-
elog(ERROR, "cannot stat \"%s\": %s", file->path,
910-
strerror(errno));
911-
}
912-
913-
if (fseek(in, skip_size, SEEK_SET) < 0)
914-
elog(ERROR, "cannot seek %lu of \"%s\": %s",
915-
skip_size, file->path, strerror(errno));
916-
917-
/*
918-
* copy content
919-
* NOTE: Now CRC is not computed for compressed files now.
920-
*/
921-
for (;;)
922-
{
923-
if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf))
924-
break;
925-
926-
if (fwrite(buf, 1, read_len, out) != read_len)
927-
{
928-
errno_tmp = errno;
929-
/* oops */
930-
fclose(in);
931-
fclose(out);
932-
elog(ERROR, "cannot write to \"%s\": %s", to_path,
933-
strerror(errno_tmp));
934-
}
935-
936-
file->write_size += sizeof(buf);
937-
file->read_size += sizeof(buf);
938-
}
939-
940-
errno_tmp = errno;
941-
if (!feof(in))
942-
{
943-
fclose(in);
944-
fclose(out);
945-
elog(ERROR, "cannot read backup mode file \"%s\": %s",
946-
file->path, strerror(errno_tmp));
947-
}
948-
949-
/* copy odd part. */
950-
if (read_len > 0)
951-
{
952-
if (fwrite(buf, 1, read_len, out) != read_len)
953-
{
954-
errno_tmp = errno;
955-
/* oops */
956-
fclose(in);
957-
fclose(out);
958-
elog(ERROR, "cannot write to \"%s\": %s", to_path,
959-
strerror(errno_tmp));
960-
}
961-
962-
file->write_size += read_len;
963-
file->read_size += read_len;
964-
}
965-
966-
/* update file permission */
967-
if (chmod(to_path, st.st_mode) == -1)
968-
{
969-
errno_tmp = errno;
970-
fclose(in);
971-
fclose(out);
972-
elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
973-
strerror(errno_tmp));
974-
}
975-
976-
/* add meta information needed for recovery */
977-
file->is_partial_copy = true;
978-
979-
if (fflush(out) != 0 ||
980-
fsync(fileno(out)) != 0 ||
981-
fclose(out))
982-
elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno));
983-
fclose(in);
984-
985-
return true;
986-
}
987-
988-
989767
/*
990768
* Calculate checksum of various files which are not copied from PGDATA,
991769
* but created in process of backup, such as stream XLOG files,

0 commit comments

Comments
 (0)