Skip to content

Commit f1da429

Browse files
dmpgprodmpgpro
committed
Add 'contrib/pg_arman/' from commit 'a0fb3912a8be3ddd553688dd89e02cb57c3e55dc'
git-subtree-dir: contrib/pg_arman git-subtree-mainline: de798cf git-subtree-split: a0fb391
2 parents de798cf + a0fb391 commit f1da429

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+10463
-0
lines changed

‎contrib/pg_arman/.gitignore

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Object files
2+
*.o
3+
4+
# Libraries
5+
*.lib
6+
*.a
7+
8+
# Shared objects (inc. Windows DLLs)
9+
*.dll
10+
*.so
11+
*.so.*
12+
*.dylib
13+
14+
# Executables
15+
*.exe
16+
*.app
17+
18+
# Dependencies
19+
.deps
20+
21+
# Binaries
22+
/pg_arman
23+
24+
# Generated by test suite
25+
/regression.diffs
26+
/regression.out
27+
/results
28+
29+
# Extra files
30+
/datapagemap.c
31+
/datapagemap.h
32+
/xlogreader.c

‎contrib/pg_arman/COPYRIGHT

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
2+
3+
Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
4+
Portions Copyright (c) 1994, The Regents of the University of California
5+
6+
Redistribution and use in source and binary forms, with or without
7+
modification, are permitted provided that the following conditions are met:
8+
9+
* Redistributions of source code must retain the above copyright notice,
10+
this list of conditions and the following disclaimer.
11+
* Redistributions in binary form must reproduce the above copyright
12+
notice, this list of conditions and the following disclaimer in the
13+
documentation and/or other materials provided with the distribution.
14+
* Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION
15+
(NTT) nor the names of its contributors may be used to endorse or
16+
promote products derived from this software without specific prior
17+
written permission.
18+
19+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎contrib/pg_arman/Makefile

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
PROGRAM = pg_arman
2+
OBJS = backup.o \
3+
catalog.o \
4+
data.o \
5+
delete.o \
6+
dir.o \
7+
fetch.o \
8+
init.o \
9+
parray.o \
10+
pg_arman.o \
11+
restore.o \
12+
show.o \
13+
status.o \
14+
util.o \
15+
validate.o \
16+
datapagemap.o \
17+
parsexlog.o \
18+
xlogreader.o \
19+
streamutil.o \
20+
receivelog.o \
21+
pgut/pgut.o \
22+
pgut/pgut-port.o
23+
24+
EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c receivelog.h streamutil.h logging.h
25+
26+
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS}
27+
override CPPFLAGS := -DFRONTEND $(CPPFLAGS)
28+
PG_LIBS = $(libpq_pgport) ${PTHREAD_LIBS} ${PTHREAD_CFLAGS}
29+
30+
REGRESS = init option show delete backup restore
31+
32+
all: checksrcdir datapagemap.h logging.h receivelog.h streamutil.h pg_arman
33+
34+
# This rule's only purpose is to give the user instructions on how to pass
35+
# the path to PostgreSQL source tree to the makefile.
36+
.PHONY: checksrcdir
37+
checksrcdir:
38+
ifndef top_srcdir
39+
@echo "You must have PostgreSQL source tree available to compile."
40+
@echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir"
41+
@echo "variable: \"make top_srcdir=<path to PostgreSQL source tree>\""
42+
@exit 1
43+
endif
44+
45+
# Those files are symlinked from the PostgreSQL sources.
46+
xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/%
47+
rm -f $@ && $(LN_S) $< .
48+
datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/%
49+
rm -f $@ && $(LN_S) $< .
50+
datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/%
51+
rm -f && $(LN_S) $< .
52+
#logging.c: % : $(top_srcdir)/src/bin/pg_rewind/%
53+
# rm -f && $(LN_S) $< .
54+
logging.h: % : $(top_srcdir)/src/bin/pg_rewind/%
55+
rm -f && $(LN_S) $< .
56+
receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/%
57+
rm -f && $(LN_S) $< .
58+
receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/%
59+
rm -f && $(LN_S) $< .
60+
streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/%
61+
rm -f && $(LN_S) $< .
62+
streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/%
63+
rm -f && $(LN_S) $< .
64+
65+
PG_CONFIG = pg_config
66+
PGXS := $(shell $(PG_CONFIG) --pgxs)
67+
include $(PGXS)
68+

‎contrib/pg_arman/README.md

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
pg_arman fork from Postgres Professional
2+
========================================
3+
4+
This repository contains fork of pg_arman by Postgres Professional with
5+
block level incremental backup support.
6+
7+
pg_arman is a backup and recovery manager for PostgreSQL servers able to do
8+
differential and full backup as well as restore a cluster to a
9+
state defined by a given recovery target. It is designed to perform
10+
periodic backups of an existing PostgreSQL server, combined with WAL
11+
archives to provide a way to recover a server in case of failure of
12+
server because of a reason or another. Its differential backup
13+
facility reduces the amount of data necessary to be taken between
14+
two consecutive backups.
15+
16+
Download
17+
--------
18+
19+
The latest version of this software can be found on the project website at
20+
https://github.com/postgrespro/pg_arman. Original fork of pg_arman can be
21+
found at https://github.com/michaelpq/pg_arman.
22+
23+
Installation
24+
------------
25+
26+
Compiling pg_arman requires a PostgreSQL installation to be in place
27+
as well as a raw source tree. Pass the path to the PostgreSQL source tree
28+
to make, in the top_srcdir variable:
29+
30+
make USE_PGXS=1 top_srcdir=<path to PostgreSQL source tree>
31+
32+
In addition, you must have pg_config in $PATH.
33+
34+
The current version of pg_arman is compatible with PostgreSQL 9.5 and
35+
upper versions.
36+
37+
Platforms
38+
---------
39+
40+
pg_arman has been tested on Linux and Unix-based platforms.
41+
42+
Documentation
43+
-------------
44+
45+
All the documentation you can find [here](doc/pg_arman.md).
46+
47+
Regression tests
48+
----------------
49+
50+
The test suite of pg_arman is available in the code tree and can be
51+
launched in a way similar to common PostgreSQL extensions and modules:
52+
53+
make installcheck
54+
55+
Block level incremental backup
56+
------------------------------
57+
58+
Idea of block level incremental backup is that you may backup only blocks
59+
changed since last full backup. It gives two major benefits: taking backups
60+
faster and making backups smaller.
61+
62+
The major question here is how to get the list of changed blocks. Since
63+
each block contains LSN number, changed blocks could be retrieved by full scan
64+
of all the blocks. But this approach consumes as much server IO as full
65+
backup.
66+
67+
This is why we implemented alternative approaches to retrieve
68+
list of changed blocks.
69+
70+
1. Scan WAL archive and extract changed blocks from it. However, shortcoming
71+
of these approach is requirement to have WAL archive.
72+
73+
2. Track bitmap of changes blocks inside PostgreSQL (ptrack). It introduces
74+
some overhead to PostgreSQL performance. On our experiments it appears to be
75+
less than 3%.
76+
77+
These two approaches were implemented in this fork of pg_arman. The second
78+
approach requires [patch for PostgreSQL](https://gist.github.com/stalkerg/7ce2394c4f3b36f995895190a633b4fa).
79+
80+
Testing block level incremental backup
81+
--------------------------------------
82+
83+
You need build and install [PGPRO9_5_ptrack branch of PostgreSQL](https://github.com/postgrespro/postgrespro/tree/PGPRO9_5_ptrack) or [apply this patch to PostgreSQL 9.5](https://gist.github.com/stalkerg/7ce2394c4f3b36f995895190a633b4fa).
84+
85+
### Retrieving changed blocks from WAL archive
86+
87+
You need to enable WAL archive by adding following lines to postgresql.conf:
88+
89+
```
90+
wal_level = archive
91+
archive_command = 'test ! -f /home/postgres/backup/arman/wal/%f && cp %p /home/postgres/backup/arman/wal/%f'
92+
wal_log_hints = on
93+
```
94+
95+
Example backup (assuming PostgreSQL is running):
96+
```bash
97+
# Init pg_aramn backup folder
98+
pg_arman init -B /home/postgres/backup/pgarman
99+
cat << __EOF__ >> /home/postgres/backup/pgarman/pg_arman.ini
100+
ARCLOG_PATH = '/home/postgres/backup/arman/wal'
101+
__EOF__
102+
# Make full backup with 2 thread and verbose mode.
103+
pg_arman backup -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman -b full -v -j 2
104+
# Validate backup
105+
pg_arman validate -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman
106+
# Show backups information
107+
pg_arman show -B /home/postgres/backup/pgarman
108+
109+
# Now you can insert or update some data in your database
110+
111+
# Then start the incremental backup.
112+
pg_arman backup -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman -b page -v -j 2
113+
pg_arman validate -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman
114+
# You should see that increment is really small
115+
pg_arman show -B /home/postgres/backup/pgarman
116+
```
117+
118+
For restore after remove your pgdata you can use:
119+
```
120+
pg_arman restore -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman -j 4 --verbose
121+
```
122+
123+
### Retrieving changed blocks from ptrack
124+
125+
The advantage of this approach is that you don't have to save WAL archive. You will need to enable ptrack in postgresql.conf (restart required).
126+
127+
```
128+
ptrack_enable = on
129+
```
130+
131+
Also, some WALs still need to be fetched in order to get consistent backup. pg_arman can fetch them trough the streaming replication protocol. Thus, you also need to [enable streaming replication connection](https://wiki.postgresql.org/wiki/Streaming_Replication).
132+
133+
Example backup (assuming PostgreSQL is running):
134+
```bash
135+
# Init pg_aramn backup folder
136+
pg_arman init -B /home/postgres/backup/pgarman
137+
cat << __EOF__ >> /home/postgres/backup/pgarman/pg_arman.ini
138+
ARCLOG_PATH = '/home/postgres/backup/arman/wal'
139+
__EOF__
140+
# Make full backup with 2 thread and verbose mode.
141+
pg_arman backup -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman -b full -v -j 2 --stream
142+
# Validate backup
143+
pg_arman validate -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman
144+
# Show backups information
145+
pg_arman show -B /home/postgres/backup/pgarman
146+
147+
# Now you can insert or update some data in your database
148+
149+
# Then start the incremental backup.
150+
pg_arman backup -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman -b ptrack -v -j 2 --stream
151+
pg_arman validate -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman
152+
# You should see that increment is really small
153+
pg_arman show -B /home/postgres/backup/pgarman
154+
```
155+
156+
For restore after remove your pgdata you can use:
157+
```
158+
pg_arman restore -B /home/postgres/backup/pgarman -D /home/postgres/pgdata/arman -j 4 --verbose --stream
159+
```
160+
161+
License
162+
-------
163+
164+
pg_arman can be distributed under the PostgreSQL license. See COPYRIGHT
165+
file for more information. pg_arman is a fork of the existing project
166+
pg_rman, initially created and maintained by NTT and Itagaki Takahiro.

0 commit comments

Comments
 (0)