Skip to content

Commit 1e1844c

Browse files
vbwagnervbwagner
committed
Merge commit 'f1e32508c09cbc47dfdb5a6ba96b6ace9a76289e' into PGPRO9_6
2 parents 4594e73 + f1e3250 commit 1e1844c

21 files changed

+1740
-50
lines changed

‎contrib/pg_probackup/Makefile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ include $(top_srcdir)/contrib/contrib-global.mk
2424
endif # USE_PGXS
2525

2626
ifeq ($(top_srcdir),../..)
27-
srchome=$(top_srcdir)/..
27+
ifeq ($(LN_S),ln -s)
28+
srchome=$(top_srcdir)/..
29+
endif
2830
else
2931
srchome=$(top_srcdir)
3032
endif

‎contrib/pg_probackup/README.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22

33
`pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure.
44

5-
The extension is compatible with:
5+
The utility is compatible with:
66
* PostgreSQL 9.5, 9.6, 10;
77

8-
`PTRACK` backup requires:
9-
* Postgres Pro Standard 9.5, 9.6;
10-
or
11-
* Postgres Pro Enterprise;
8+
`PTRACK` backup support provided via following options:
9+
* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/3d9ad9ea2568e0f4aaeeda62b59e30f8/raw/8f28e207c1aa172a9597fcda49e69b086c6b42bd/ptrack_9.6.5_v1.3.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/0ce69df847268333b72d72079bd48315/raw/d35d49c28446637ea3fe9dfc1a400bb298bf3318/ptrack_10.0_v1.3.patch)
10+
* Postgres Pro Standard 9.5, 9.6
11+
* Postgres Pro Enterprise
1212

1313
As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data:
1414
* Choosing between full and page-level incremental backups to speed up backup and recovery

‎contrib/pg_probackup/src/backup.c

Lines changed: 102 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ static XLogRecPtr get_last_ptrack_lsn(void);
118118
static void check_server_version(void);
119119
static void check_system_identifiers(void);
120120
static void confirm_block_size(const char *name, int blcksz);
121+
static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i);
121122

122123

123124
#define disconnect_and_exit(code) \
@@ -1878,6 +1879,7 @@ backup_files(void *arg)
18781879
struct stat buf;
18791880

18801881
pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i);
1882+
elog(VERBOSE, "Copying file: \"%s\" ", file->path);
18811883
if (__sync_lock_test_and_set(&file->lock, 1) != 0)
18821884
continue;
18831885

@@ -1918,8 +1920,9 @@ backup_files(void *arg)
19181920
if (S_ISREG(buf.st_mode))
19191921
{
19201922
/* copy the file into backup */
1921-
if (file->is_datafile)
1923+
if (file->is_datafile && !file->is_cfs)
19221924
{
1925+
/* backup block by block if datafile AND not compressed by cfs*/
19231926
if (!backup_data_file(arguments->from_root,
19241927
arguments->to_root, file,
19251928
arguments->prev_backup_start_lsn))
@@ -1961,16 +1964,18 @@ parse_backup_filelist_filenames(parray *files, const char *root)
19611964
{
19621965
pgFile *file = (pgFile *) parray_get(files, i);
19631966
char *relative;
1964-
char *filename = palloc(MAXPGPATH);
1967+
char filename[MAXPGPATH];
19651968
int sscanf_result;
19661969

19671970
relative = GetRelativePath(file->path, root);
1971+
filename[0] = '\0';
19681972

19691973
elog(VERBOSE, "-----------------------------------------------------: %s", relative);
19701974
if (path_is_prefix_of_path("global", relative))
19711975
{
19721976
file->tblspcOid = GLOBALTABLESPACE_OID;
19731977
sscanf_result = sscanf(relative, "global/%s", filename);
1978+
elog(VERBOSE, "global sscanf result: %i", sscanf_result);
19741979
if (strcmp(relative, "global") == 0)
19751980
{
19761981
Assert(S_ISDIR(file->mode));
@@ -1986,6 +1991,7 @@ parse_backup_filelist_filenames(parray *files, const char *root)
19861991
{
19871992
file->tblspcOid = DEFAULTTABLESPACE_OID;
19881993
sscanf_result = sscanf(relative, "base/%u/%s", &(file->dbOid), filename);
1994+
elog(VERBOSE, "base sscanf result: %i", sscanf_result);
19891995
if (strcmp(relative, "base") == 0)
19901996
{
19911997
Assert(S_ISDIR(file->mode));
@@ -2004,9 +2010,11 @@ parse_backup_filelist_filenames(parray *files, const char *root)
20042010
}
20052011
else if (path_is_prefix_of_path(PG_TBLSPC_DIR, relative))
20062012
{
2007-
char *temp_relative_path = palloc(MAXPGPATH);
2013+
char temp_relative_path[MAXPGPATH];
20082014

20092015
sscanf_result = sscanf(relative, "pg_tblspc/%u/%s", &(file->tblspcOid), temp_relative_path);
2016+
elog(VERBOSE, "pg_tblspc sscanf result: %i", sscanf_result);
2017+
20102018
if (strcmp(relative, "pg_tblspc") == 0)
20112019
{
20122020
Assert(S_ISDIR(file->mode));
@@ -2022,21 +2030,43 @@ parse_backup_filelist_filenames(parray *files, const char *root)
20222030
/*continue parsing */
20232031
sscanf_result = sscanf(temp_relative_path+strlen(TABLESPACE_VERSION_DIRECTORY)+1, "%u/%s",
20242032
&(file->dbOid), filename);
2033+
elog(VERBOSE, "TABLESPACE_VERSION_DIRECTORY sscanf result: %i", sscanf_result);
20252034

2026-
if (sscanf_result == 0)
2035+
if (sscanf_result == -1)
2036+
{
2037+
elog(VERBOSE, "The TABLESPACE_VERSION_DIRECTORY itself, filepath %s", relative);
2038+
}
2039+
else if (sscanf_result == 0)
20272040
{
2028-
elog(VERBOSE, "the TABLESPACE_VERSION_DIRECTORY itself, filepath %s", relative);
2041+
/* Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
2042+
Legal only in case of 'pg_compression'
2043+
*/
2044+
if (strcmp(file->name, "pg_compression") == 0)
2045+
{
2046+
elog(VERBOSE, "Found pg_compression file in TABLESPACE_VERSION_DIRECTORY, filepath %s", relative);
2047+
/*Set every datafile in tablespace as is_cfs */
2048+
set_cfs_datafiles(files, root, relative, i);
2049+
}
2050+
else
2051+
{
2052+
elog(VERBOSE, "Found illegal file in TABLESPACE_VERSION_DIRECTORY, filepath %s", relative);
2053+
}
2054+
20292055
}
20302056
else if (sscanf_result == 1)
20312057
{
20322058
Assert(S_ISDIR(file->mode));
20332059
elog(VERBOSE, "dboid %u, filepath %s", file->dbOid, relative);
20342060
file->is_database = true;
20352061
}
2036-
else
2062+
else if (sscanf_result == 2)
20372063
{
20382064
elog(VERBOSE, "dboid %u, filename %s, filepath %s", file->dbOid, filename, relative);
20392065
}
2066+
else
2067+
{
2068+
elog(VERBOSE, "Illegal file filepath %s", relative);
2069+
}
20402070
}
20412071
}
20422072
else
@@ -2054,7 +2084,7 @@ parse_backup_filelist_filenames(parray *files, const char *root)
20542084
}
20552085

20562086
/* Check files located inside database directories */
2057-
if (filename && file->dbOid != 0)
2087+
if (filename[0] != '\0' && file->dbOid != 0)
20582088
{
20592089
if (strcmp(filename, "pg_internal.init") == 0)
20602090
{
@@ -2081,7 +2111,7 @@ parse_backup_filelist_filenames(parray *files, const char *root)
20812111
* Check that and do not mark them with 'is_datafile' flag.
20822112
*/
20832113
char *forkNameptr;
2084-
char *suffix = palloc(MAXPGPATH);;
2114+
char suffix[MAXPGPATH];
20852115

20862116
forkNameptr = strstr(filename, "_");
20872117
if (forkNameptr != NULL)
@@ -2108,7 +2138,16 @@ parse_backup_filelist_filenames(parray *files, const char *root)
21082138
{
21092139
/* first segment of the relfile */
21102140
elog(VERBOSE, "relOid %u, segno %d, filepath %s", file->relOid, 0, relative);
2111-
file->is_datafile = true;
2141+
if (strcmp(relative + strlen(relative) - strlen("cfm"), "cfm") == 0)
2142+
{
2143+
/* reloid.cfm */
2144+
elog(VERBOSE, "Found cfm file %s", relative);
2145+
}
2146+
else
2147+
{
2148+
elog(VERBOSE, "Found first segment of the relfile %s", relative);
2149+
file->is_datafile = true;
2150+
}
21122151
}
21132152
else if (sscanf_result == 2)
21142153
{
@@ -2126,9 +2165,62 @@ parse_backup_filelist_filenames(parray *files, const char *root)
21262165
}
21272166
}
21282167
}
2129-
}
2168+
}
2169+
}
2170+
2171+
/* If file is equal to pg_compression, then we consider this tablespace as
2172+
* cfs-compressed and should mark every file in this tablespace as cfs-file
2173+
* Setting is_cfs is done via going back through 'files' set every file
2174+
* that contain cfs_tablespace in his path as 'is_cfs'
2175+
* Goings back through array 'files' is valid option possible because of current
2176+
* sort rules:
2177+
* tblspcOid/TABLESPACE_VERSION_DIRECTORY
2178+
* tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid
2179+
* tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1
2180+
* tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm
2181+
* tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression
2182+
*/
2183+
static void
2184+
set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
2185+
{
2186+
int len;
2187+
size_t p;
2188+
pgFile *prev_file;
2189+
char *cfs_tblspc_path;
2190+
char *relative_prev_file;
2191+
2192+
cfs_tblspc_path = strdup(relative);
2193+
len = strlen("/pg_compression");
2194+
cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0;
2195+
elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative);
2196+
2197+
for (p = i; p >= 0; p--)
2198+
{
2199+
prev_file = (pgFile *) parray_get(files, p);
2200+
relative_prev_file = GetRelativePath(prev_file->path, root);
2201+
2202+
//elog(VERBOSE, "P: %lu, Checking file in cfs tablespace %s", p, relative_prev_file);
2203+
elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file);
2204+
2205+
if (strstr(relative_prev_file, cfs_tblspc_path) != NULL)
2206+
{
2207+
if (S_ISREG(prev_file->mode) && prev_file->is_datafile)
2208+
{
2209+
elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s",
2210+
relative_prev_file, prev_file->name);
2211+
prev_file->is_cfs = true;
2212+
}
2213+
}
2214+
else
2215+
{
2216+
elog(VERBOSE, "Breaking on %s", relative_prev_file);
2217+
break;
2218+
}
2219+
}
2220+
free(cfs_tblspc_path);
21302221
}
21312222

2223+
21322224
/*
21332225
* Output the list of files to backup catalog DATABASE_FILE_LIST
21342226
*/

‎contrib/pg_probackup/src/data.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,7 @@ restore_data_file(const char *from_root,
470470
if (header.block < blknum)
471471
elog(ERROR, "backup is broken at block %u", blknum);
472472

473+
elog(VERBOSE, "file %s, header compressed size %d", file->path, header.compressed_size);
473474
Assert(header.compressed_size <= BLCKSZ);
474475

475476
read_len = fread(compressed_page.data, 1,

‎contrib/pg_probackup/src/dir.c

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -393,15 +393,20 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
393393
if (file_name == NULL)
394394
file_name = file->path;
395395
else
396+
{
396397
file_name++;
398+
file->name = file_name;
399+
}
397400

398401
/* Check if we need to exclude file by name */
399402
for (i = 0; pgdata_exclude_files[i]; i++)
400-
if (strcmp(file_name, pgdata_exclude_files[i]) == 0)
403+
if (strcmp(file->name, pgdata_exclude_files[i]) == 0)
404+
{
401405
/* Skip */
406+
elog(VERBOSE, "Excluding file: %s", file->name);
402407
return;
408+
}
403409
}
404-
405410
parray_append(files, file);
406411
}
407412

@@ -463,7 +468,10 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
463468
if (dirname == NULL)
464469
dirname = file->path;
465470
else
471+
{
466472
dirname++;
473+
file->name = dirname;
474+
}
467475

468476
/*
469477
* If the item in the exclude list starts with '/', compare to the
@@ -472,6 +480,7 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
472480
*/
473481
for (i = 0; exclude && pgdata_exclude_dir[i]; i++)
474482
{
483+
/* Full-path exclude*/
475484
if (pgdata_exclude_dir[i][0] == '/')
476485
{
477486
if (strcmp(file->path, pgdata_exclude_dir[i]) == 0)
@@ -480,14 +489,17 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
480489
break;
481490
}
482491
}
483-
else if (strcmp(dirname, pgdata_exclude_dir[i]) == 0)
492+
else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0)
484493
{
485494
skip = true;
486495
break;
487496
}
488497
}
489498
if (skip)
499+
{
500+
elog(VERBOSE, "Excluding directory content: %s", file->name);
490501
break;
502+
}
491503
}
492504

493505
/* open directory and list contents */
@@ -678,9 +690,11 @@ print_file_list(FILE *out, const parray *files, const char *root)
678690
path = GetRelativePath(path, root);
679691

680692
fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\","
681-
"\"is_datafile\":\"%u\", \"crc\":\"%u\", \"compress_alg\":\"%s\"",
693+
"\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\","
694+
"\"compress_alg\":\"%s\"",
682695
path, (unsigned long) file->write_size, file->mode,
683-
file->is_datafile?1:0, file->crc, deparse_compress_alg(file->compress_alg));
696+
file->is_datafile?1:0, file->is_cfs?1:0, file->crc,
697+
deparse_compress_alg(file->compress_alg));
684698

685699
if (file->is_datafile)
686700
fprintf(out, ",\"segno\":\"%d\"", file->segno);
@@ -853,6 +867,7 @@ dir_read_file_list(const char *root, const char *file_txt)
853867
uint64 write_size,
854868
mode, /* bit length of mode_t depends on platforms */
855869
is_datafile,
870+
is_cfs,
856871
crc,
857872
segno;
858873
pgFile *file;
@@ -861,6 +876,7 @@ dir_read_file_list(const char *root, const char *file_txt)
861876
get_control_value(buf, "size", NULL, &write_size, true);
862877
get_control_value(buf, "mode", NULL, &mode, true);
863878
get_control_value(buf, "is_datafile", NULL, &is_datafile, true);
879+
get_control_value(buf, "is_cfs", NULL, &is_cfs, false);
864880
get_control_value(buf, "crc", NULL, &crc, true);
865881

866882
/* optional fields */
@@ -878,6 +894,7 @@ dir_read_file_list(const char *root, const char *file_txt)
878894
file->write_size = (size_t) write_size;
879895
file->mode = (mode_t) mode;
880896
file->is_datafile = is_datafile ? true : false;
897+
file->is_cfs = is_cfs ? true : false;
881898
file->crc = (pg_crc32) crc;
882899
file->compress_alg = parse_compress_alg(compress_alg_string);
883900
if (linked[0])

‎contrib/pg_probackup/src/pg_probackup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
#include <sys/stat.h>
1818
#include <unistd.h>
1919

20-
const char *PROGRAM_VERSION = "2.0.8";
20+
const char *PROGRAM_VERSION = "2.0.10";
2121
const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup";
2222
const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues";
2323

‎contrib/pg_probackup/src/pg_probackup.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ typedef enum CompressAlg
8080
/* Information about single file (or dir) in backup */
8181
typedef struct pgFile
8282
{
83+
char *name; /* file or directory name */
8384
mode_t mode; /* protection (file type and permission) */
8485
size_t size; /* size of the file */
8586
size_t read_size; /* size of the portion read (if only some pages are
@@ -90,7 +91,7 @@ typedef struct pgFile
9091
pg_crc32 crc; /* CRC value of the file, regular file only */
9192
char *linked; /* path of the linked file */
9293
bool is_datafile; /* true if the file is PostgreSQL data file */
93-
char *path; /* path of the file */
94+
char *path; /* absolute path of the file */
9495
Oid tblspcOid; /* tblspcOid extracted from path, if applicable */
9596
Oid dbOid; /* dbOid extracted from path, if applicable */
9697
Oid relOid; /* relOid extracted from path, if applicable */

‎contrib/pg_probackup/src/restore.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,8 @@ restore_files(void *arg)
686686
* block and have BackupPageHeader meta information, so we cannot just
687687
* copy the file from backup.
688688
*/
689-
if (file->is_datafile)
689+
elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0);
690+
if (file->is_datafile && !file->is_cfs)
690691
restore_data_file(from_root, pgdata, file, arguments->backup);
691692
else
692693
copy_file(from_root, pgdata, file);

‎contrib/pg_probackup/src/utils/logger.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ write_elevel(FILE *stream, int elevel)
8686
{
8787
switch (elevel)
8888
{
89+
case VERBOSE:
90+
fputs("VERBOSE: ", stream);
91+
break;
8992
case LOG:
9093
fputs("LOG: ", stream);
9194
break;

0 commit comments

Comments
 (0)