Skip to content

Commit 315cf3b

Browse files
alubennikovaalubennikova
committed
Read value from ptrack_control via function call. Code cleanup
1 parent 18e49a4 commit 315cf3b

File tree

5 files changed

+63
-185
lines changed

5 files changed

+63
-185
lines changed

‎src/backup.c

Lines changed: 38 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ static char *pg_ptrack_get_and_clear(Oid tablespace_oid,
112112
Oid db_oid,
113113
Oid rel_oid,
114114
size_t *result_size);
115+
static XLogRecPtr get_last_ptrack_lsn(void);
115116

116117
/* Check functions */
117118
static void check_server_version(void);
@@ -1814,25 +1815,6 @@ backup_disconnect(bool fatal, void *userdata)
18141815
pgut_disconnect(master_conn);
18151816
}
18161817

1817-
/* Count bytes in file */
1818-
static long
1819-
file_size(const char *file_path)
1820-
{
1821-
long r;
1822-
FILE *f = fopen(file_path, "r");
1823-
1824-
if (!f)
1825-
{
1826-
elog(ERROR, "%s: cannot open file \"%s\" for reading: %s\n",
1827-
PROGRAM_NAME ,file_path, strerror(errno));
1828-
return -1;
1829-
}
1830-
fseek(f, 0, SEEK_END);
1831-
r = ftell(f);
1832-
fclose(f);
1833-
return r;
1834-
}
1835-
18361818
/*
18371819
* Take a backup of the PGDATA at a file level.
18381820
* Copy all directories and files listed in backup_files_list.
@@ -2044,6 +2026,10 @@ parse_backup_filelist_filenames(parray *files, const char *root)
20442026
}
20452027
else if (isdigit(filename[0]))
20462028
{
2029+
/*
2030+
* TODO TODO TODO Files of this type can be compressed by cfs.
2031+
* Check that and do not mark them with 'is_datafile' flag.
2032+
*/
20472033
char *forkNameptr;
20482034
char *suffix = palloc(MAXPGPATH);;
20492035

@@ -2090,6 +2076,12 @@ parse_backup_filelist_filenames(parray *files, const char *root)
20902076
}
20912077
}
20922078
}
2079+
2080+
if (strcmp(filename, "pg_internal.init") == 0)
2081+
{
2082+
elog(INFO, "filename %s, path %s, dbOid %u, tblspcOid %u is_datafile %s",
2083+
filename, file->path, file->dbOid, file->tblspcOid, file->is_datafile?"true":"false");
2084+
}
20932085
}
20942086
}
20952087

@@ -2217,16 +2209,18 @@ make_pagemap_from_ptrack(parray *files)
22172209
Assert(filename != NULL);
22182210
filename++;
22192211

2212+
/* Always backup all files from template0, template1 databases */
2213+
if((file->dbOid == 1) || //dbOid of template1 daatbase
2214+
(file->dbOid == 12442)) //dbOid of template0 daatbase
2215+
{
2216+
is_template = true;
2217+
}
22202218
/*
22212219
* The function pg_ptrack_get_and_clear_db returns true
22222220
* if there was a ptrack_init file.
2223-
* And always backup all files from template0 database and global/
2221+
* Also ignore ptrack files for global tablespace,
2222+
* to avoid any possible specific errors.
22242223
*/
2225-
if((strcmp(filename, "1") == 0) ||
2226-
(strcmp(filename, "12442") == 0))
2227-
{
2228-
is_template = true;
2229-
}
22302224
else if ((file->tblspcOid == GLOBALTABLESPACE_OID) ||
22312225
pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid))
22322226
{
@@ -2441,3 +2435,22 @@ StreamLog(void *arg)
24412435
PQfinish(conn);
24422436
conn = NULL;
24432437
}
2438+
2439+
/*
2440+
* Get lsn of the moment when ptrack was enabled the last time.
2441+
*/
2442+
static XLogRecPtr
2443+
get_last_ptrack_lsn(void)
2444+
2445+
{
2446+
PGresult *res;
2447+
XLogRecPtr lsn;
2448+
2449+
res = pgut_execute(backup_conn, "select pg_ptrack_control_lsn()", 0, NULL);
2450+
2451+
lsn = atoi(PQgetvalue(res, 0, 0));
2452+
elog(INFO, "get_last_ptrack_lsn(): lsn %lu", lsn);
2453+
2454+
PQclear(res);
2455+
return lsn;
2456+
}

‎src/data.c

Lines changed: 25 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,26 @@
2222
#include <common/pg_lzcompress.h>
2323
#include <zlib.h>
2424

25+
/* Implementation of zlib compression method */
2526
static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
2627
{
2728
uLongf compressed_size = dst_size;
2829
int rc = compress2(dst, &compressed_size, src, src_size, compress_level);
2930
return rc == Z_OK ? compressed_size : rc;
3031
}
3132

33+
/* Implementation of zlib compression method */
3234
static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
3335
{
3436
uLongf dest_len = dst_size;
3537
int rc = uncompress(dst, &dest_len, src, src_size);
3638
return rc == Z_OK ? dest_len : rc;
3739
}
3840

41+
/*
42+
* Compresses source into dest using algorithm. Returns the number of bytes
43+
* written in the destination buffer, or -1 if compression fails.
44+
*/
3945
static size_t
4046
do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg)
4147
{
@@ -53,6 +59,10 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre
5359
return -1;
5460
}
5561

62+
/*
63+
* Decompresses source into dest using algorithm. Returns the number of bytes
64+
* decompressed in the destination buffer, or -1 if decompression fails.
65+
*/
5666
static size_t
5767
do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg)
5868
{
@@ -70,8 +80,10 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, Comp
7080
return -1;
7181
}
7282

73-
74-
83+
/*
84+
* When copying datafiles to backup we validate and compress them block
85+
* by block. Thus special header is required for each data block.
86+
*/
7587
typedef struct BackupPageHeader
7688
{
7789
BlockNumber block; /* block number */
@@ -125,6 +137,11 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
125137
header.block = blknum;
126138
offset = blknum * BLCKSZ;
127139

140+
/*
141+
* Read the page and verify its header and checksum.
142+
* Under high write load it's possible that we've read partly
143+
* flushed page, so try several times befor throwing an error.
144+
*/
128145
while(try_checksum--)
129146
{
130147
if (fseek(in, offset, SEEK_SET) != 0)
@@ -223,12 +240,14 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
223240
Assert (header.compressed_size <= BLCKSZ);
224241
write_buffer_size = sizeof(header);
225242

243+
/* The page was successfully compressed */
226244
if (header.compressed_size > 0)
227245
{
228246
memcpy(write_buffer, &header, sizeof(header));
229247
memcpy(write_buffer + sizeof(header), compressed_page.data, header.compressed_size);
230248
write_buffer_size += MAXALIGN(header.compressed_size);
231249
}
250+
/* The page compression failed. Write it as is. */
232251
else
233252
{
234253
header.compressed_size = BLCKSZ;
@@ -324,8 +343,7 @@ backup_data_file(const char *from_root, const char *to_root,
324343

325344
/*
326345
* Read each page, verify checksum and write it to backup.
327-
* If page map is not empty we scan only changed blocks, otherwise
328-
* backup all pages of the relation.
346+
* If page map is empty backup all pages of the relation.
329347
*/
330348
if (file->pagemap.bitmapsize == 0)
331349
{
@@ -336,6 +354,7 @@ backup_data_file(const char *from_root, const char *to_root,
336354
n_blocks_read++;
337355
}
338356
}
357+
/* If page map is not empty we scan only changed blocks, */
339358
else
340359
{
341360
datapagemap_iterator_t *iter;
@@ -371,7 +390,7 @@ backup_data_file(const char *from_root, const char *to_root,
371390
FIN_CRC32C(file->crc);
372391

373392
/*
374-
* If we have pagemap then file can't be a zero size.
393+
* If we have pagemap then file in the backup can't be a zero size.
375394
* Otherwise, we will clear the last file.
376395
*/
377396
if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped)
@@ -411,7 +430,7 @@ restore_data_file(const char *from_root,
411430

412431
/*
413432
* Open backup file for write. We use "r+" at first to overwrite only
414-
* modified pages for differential restore. If the file is not exists,
433+
* modified pages for differential restore. If the file does not exist,
415434
* re-open it with "w" to create an empty file.
416435
*/
417436
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
@@ -745,136 +764,6 @@ copy_wal_file(const char *from_path, const char *to_path)
745764
fclose(in);
746765
}
747766

748-
/*
749-
* Save part of the file into backup.
750-
* skip_size - size of the file in previous backup. We can skip it
751-
* and copy just remaining part of the file
752-
*/
753-
bool
754-
copy_file_partly(const char *from_root, const char *to_root,
755-
pgFile *file, size_t skip_size)
756-
{
757-
char to_path[MAXPGPATH];
758-
FILE *in;
759-
FILE *out;
760-
size_t read_len = 0;
761-
int errno_tmp;
762-
struct stat st;
763-
char buf[BLCKSZ];
764-
765-
/* reset size summary */
766-
file->read_size = 0;
767-
file->write_size = 0;
768-
769-
/* open backup mode file for read */
770-
in = fopen(file->path, "r");
771-
if (in == NULL)
772-
{
773-
/* maybe deleted, it's not error */
774-
if (errno == ENOENT)
775-
return false;
776-
777-
elog(ERROR, "cannot open source file \"%s\": %s", file->path,
778-
strerror(errno));
779-
}
780-
781-
/* open backup file for write */
782-
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
783-
784-
out = fopen(to_path, "w");
785-
if (out == NULL)
786-
{
787-
int errno_tmp = errno;
788-
fclose(in);
789-
elog(ERROR, "cannot open destination file \"%s\": %s",
790-
to_path, strerror(errno_tmp));
791-
}
792-
793-
/* stat source file to change mode of destination file */
794-
if (fstat(fileno(in), &st) == -1)
795-
{
796-
fclose(in);
797-
fclose(out);
798-
elog(ERROR, "cannot stat \"%s\": %s", file->path,
799-
strerror(errno));
800-
}
801-
802-
if (fseek(in, skip_size, SEEK_SET) < 0)
803-
elog(ERROR, "cannot seek %lu of \"%s\": %s",
804-
skip_size, file->path, strerror(errno));
805-
806-
/*
807-
* copy content
808-
* NOTE: Now CRC is not computed for compressed files now.
809-
*/
810-
for (;;)
811-
{
812-
if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf))
813-
break;
814-
815-
if (fwrite(buf, 1, read_len, out) != read_len)
816-
{
817-
errno_tmp = errno;
818-
/* oops */
819-
fclose(in);
820-
fclose(out);
821-
elog(ERROR, "cannot write to \"%s\": %s", to_path,
822-
strerror(errno_tmp));
823-
}
824-
825-
file->write_size += sizeof(buf);
826-
file->read_size += sizeof(buf);
827-
}
828-
829-
errno_tmp = errno;
830-
if (!feof(in))
831-
{
832-
fclose(in);
833-
fclose(out);
834-
elog(ERROR, "cannot read backup mode file \"%s\": %s",
835-
file->path, strerror(errno_tmp));
836-
}
837-
838-
/* copy odd part. */
839-
if (read_len > 0)
840-
{
841-
if (fwrite(buf, 1, read_len, out) != read_len)
842-
{
843-
errno_tmp = errno;
844-
/* oops */
845-
fclose(in);
846-
fclose(out);
847-
elog(ERROR, "cannot write to \"%s\": %s", to_path,
848-
strerror(errno_tmp));
849-
}
850-
851-
file->write_size += read_len;
852-
file->read_size += read_len;
853-
}
854-
855-
/* update file permission */
856-
if (chmod(to_path, st.st_mode) == -1)
857-
{
858-
errno_tmp = errno;
859-
fclose(in);
860-
fclose(out);
861-
elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
862-
strerror(errno_tmp));
863-
}
864-
865-
/* add meta information needed for recovery */
866-
file->is_partial_copy = true;
867-
868-
if (fflush(out) != 0 ||
869-
fsync(fileno(out)) != 0 ||
870-
fclose(out))
871-
elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno));
872-
fclose(in);
873-
874-
return true;
875-
}
876-
877-
878767
/*
879768
* Calculate checksum of various files which are not copied from PGDATA,
880769
* but created in process of backup, such as stream XLOG files,

‎src/dir.c

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,6 @@ pgFileInit(const char *path)
159159
file->path = pgut_malloc(strlen(path) + 1);
160160
strcpy(file->path, path); /* enough buffer size guaranteed */
161161
file->is_cfs = false;
162-
file->is_partial_copy = false;
163162
file->compress_alg = NOT_DEFINED_COMPRESS;
164163
return file;
165164
}

‎src/pg_probackup.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,6 @@ typedef struct pgFile
9898
int segno; /* Segment number for ptrack */
9999
bool is_cfs; /* Flag to distinguish files compressed by CFS*/
100100
bool is_database;
101-
bool is_partial_copy; /* If the file was backed up via copy_file_partly().
102-
* Only applies to is_cfs files. */
103101
CompressAlg compress_alg; /* compression algorithm applied to the file */
104102
volatile uint32 lock; /* lock for synchronization of parallel threads */
105103
datapagemap_t pagemap; /* bitmap of pages updated since previous backup */
@@ -420,8 +418,6 @@ extern void restore_data_file(const char *from_root, const char *to_root,
420418
extern bool copy_file(const char *from_root, const char *to_root,
421419
pgFile *file);
422420
extern void copy_wal_file(const char *from_root, const char *to_root);
423-
extern bool copy_file_partly(const char *from_root, const char *to_root,
424-
pgFile *file, size_t skip_size);
425421

426422
extern bool calc_file_checksum(pgFile *file);
427423

@@ -449,7 +445,6 @@ extern void time2iso(char *buf, size_t len, time_t time);
449445
extern const char *status2str(BackupStatus status);
450446
extern void remove_trailing_space(char *buf, int comment_mark);
451447
extern void remove_not_digit(char *buf, size_t len, const char *str);
452-
extern XLogRecPtr get_last_ptrack_lsn(void);
453448
extern uint32 get_data_checksum_version(bool safe);
454449
extern char *base36enc(long unsigned int value);
455450
extern long unsigned int base36dec(const char *text);

0 commit comments

Comments
 (0)