diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f99d0f27..c3ad89568 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,8 +72,7 @@ jobs: - name: Install Testgres run: | git clone -b no-port-for --single-branch --depth 1 https://github.com/postgrespro/testgres.git - cd testgres - python setup.py install + pip3 install psycopg2 ./testgres # Grant the Github runner user full control of the workspace for initdb to successfully process the data folder - name: Test Probackup diff --git a/LICENSE b/LICENSE index 66476e8a9..4fed760f8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2023, Postgres Professional +Copyright (c) 2015-2025, Postgres Professional Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group diff --git a/README.md b/README.md index 2279b97a4..16aec1dcc 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 11, 12, 13, 14, 15, 16 +* PostgreSQL 13, 14, 15, 16, 17 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -41,9 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL 11, 12, 13, 14, 15, 16 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13, 14, 15, 16 -* Postgres Pro Enterprise 11, 12, 13, 14, 15, 16 +* vanilla PostgreSQL 13, 14, 15, 16, 17 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 13, 14, 15, 16, 17 +* Postgres Pro Enterprise 13, 14, 15, 16, 17 ## Limitations diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 1491059c5..4f05da26a 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -529,14 +529,14 @@ doc/src/sgml/pgprobackup.sgml Initialize the backup catalog: -backup_user@backup_host:~$ pg_probackup init -B /mnt/backups +backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully initialized Add a backup instance called mydb to the backup catalog: -backup_user@backup_host:~$ pg_probackup add-instance \ +backup_user@backup_host:~$ pg_probackup-16 add-instance \ -B /mnt/backups \ -D /var/lib/pgpro/std-16/data \ --instance=node \ @@ -548,7 +548,7 @@ INFO: Instance 'node' successfully initialized Make a FULL backup: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b FULL \ --instance=node \ @@ -582,7 +582,7 @@ INFO: Backup SCUN1Q completed List the backups of the instance: -backup_user@backup_host:~$ pg_probackup show \ +backup_user@backup_host:~$ pg_probackup-16 show \ -B /mnt/backups \ --instance=node ================================================================================================================================ @@ -594,7 +594,7 @@ backup_user@backup_host:~$ pg_probackup show \ Make an incremental backup in the DELTA mode: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b DELTA \ --instance=node \ @@ -631,7 +631,7 @@ INFO: Backup SCUN22 completed Add or modify some parameters in the pg_probackup configuration file, so that you do not have to specify them each time on the command line: -backup_user@backup_host:~$ pg_probackup set-config \ +backup_user@backup_host:~$ pg_probackup-16 set-config \ -B /mnt/backups \ --instance=node \ --remote-host=postgres_host \ @@ -643,7 +643,7 @@ backup_user@backup_host:~$ pg_probackup set-config \ Check the configuration of the instance: -backup_user@backup_host:~$ pg_probackup show-config \ +backup_user@backup_host:~$ pg_probackup-16 show-config \ -B /mnt/backups \ --instance=node # Backup instance information @@ -686,7 +686,7 @@ remote-user = postgres Make another incremental backup in the DELTA mode, omitting the parameters stored in the configuration file earlier: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b DELTA \ --instance=node \ @@ -718,7 +718,7 @@ INFO: Backup SCUN2C completed List the backups of the instance again: -backup_user@backup_host:~$ pg_probackup show \ +backup_user@backup_host:~$ pg_probackup-16 show \ -B /mnt/backups \ --instance=node =================================================================================================================================== @@ -732,7 +732,7 @@ backup_user@backup_host:~$ pg_probackup show \ Restore the data from the latest available backup to an arbitrary location: -backup_user@backup_host:~$ pg_probackup restore \ +backup_user@backup_host:~$ pg_probackup-16 restore \ -B /mnt/backups \ -D /var/lib/pgpro/std-16/staging-data \ --instance=node @@ -924,7 +924,7 @@ yumdownloader --source pg_probackup-16 . /etc/os-release -echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p$VERSION_ID x86_64 vanilla" | \ +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p${VERSION_ID%%.*} x86_64 vanilla" | \ sudo tee /etc/apt/sources.list.d/pg_probackup.list @@ -934,7 +934,7 @@ sudo tee /etc/apt/sources.list.d/pg_probackup.list . /etc/os-release -echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-$VERSION_ID x86_64 vanilla" | \ +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-${VERSION_ID%%.*} x86_64 vanilla" | \ sudo tee /etc/apt/sources.list.d/pg_probackup.list @@ -2332,7 +2332,7 @@ pg_probackup restore -B backup_dir --instance=conn = pgut_connect(dbhost, dbport, dbname, dbuser); + check_server_version(instanceState->conn, &nodeInfo); + return do_add_instance(instanceState, &instance_config); + } case DELETE_INSTANCE_CMD: return do_delete_instance(instanceState); case INIT_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ae99e0605..175aa5589 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ @@ -1225,6 +1225,7 @@ extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); +void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); extern bool parse_page(Page page, XLogRecPtr *lsn); extern int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h index 56d852537..a1b221f46 100644 --- a/src/pg_probackup_state.h +++ b/src/pg_probackup_state.h @@ -2,7 +2,7 @@ * * pg_probackup_state.h: Definitions of internal pg_probackup states * - * Portions Copyright (c) 2021, Postgres Professional + * Portions Copyright (c) 2021-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/ptrack.c b/src/ptrack.c index d27629e45..ba97088c1 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -2,7 +2,7 @@ * * ptrack.c: support functions for ptrack backups * - * Copyright (c) 2021 Postgres Professional + * Copyright (c) 2021-2025 Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/restore.c b/src/restore.c index f9310dcee..0be151a99 100644 --- a/src/restore.c +++ b/src/restore.c @@ -3,7 +3,7 @@ * restore.c: restore DB cluster and archived WAL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/show.c b/src/show.c index 810262df6..0732c6a7a 100644 --- a/src/show.c +++ b/src/show.c @@ -3,7 +3,7 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/stream.c b/src/stream.c index 77453e997..d94c9564e 100644 --- a/src/stream.c +++ b/src/stream.c @@ -2,7 +2,7 @@ * * stream.c: pg_probackup specific code for WAL streaming * - * Portions Copyright (c) 2015-2020, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/util.c b/src/util.c index 3c0a33453..019d20644 100644 --- a/src/util.c +++ b/src/util.c @@ -3,7 +3,7 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/configuration.c b/src/utils/configuration.c index f049aa1be..47497850f 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -3,7 +3,7 @@ * configuration.c: - function implementations to work with pg_probackup * configurations. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 59da29bd5..da86b9db0 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -3,7 +3,7 @@ * configuration.h: - prototypes of functions and structures for * configuration. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/json.c b/src/utils/json.c index 2c8e0fe9b..1bcb4e644 100644 --- a/src/utils/json.c +++ b/src/utils/json.c @@ -2,7 +2,7 @@ * * json.c: - make json document. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/json.h b/src/utils/json.h index f80832e69..f635e1f3c 100644 --- a/src/utils/json.h +++ b/src/utils/json.h @@ -2,7 +2,7 @@ * * json.h: - prototypes of json output functions. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 7ea41f74e..ade57e529 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -2,7 +2,7 @@ * * logger.c: - log events into log file or stderr. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.h b/src/utils/logger.h index adc5061e0..9e2cb958f 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -2,7 +2,7 @@ * * logger.h: - prototypes of logger functions. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 9559fa644..df09dbdc6 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -3,7 +3,7 @@ * pgut.c * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 1b7b7864c..6fea4d022 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/remote.h b/src/utils/remote.h index dc98644ab..582210142 100644 --- a/src/utils/remote.h +++ b/src/utils/remote.h @@ -2,7 +2,7 @@ * * remote.h: - prototypes of remote functions. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/thread.c b/src/utils/thread.c index 1c469bd29..4127701f0 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -2,7 +2,7 @@ * * thread.c: - multi-platform pthread implementations. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/thread.h b/src/utils/thread.h index 2eaa5fb45..d79e2d8d0 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -2,7 +2,7 @@ * * thread.h: - multi-platform pthread implementations. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/validate.c b/src/validate.c index 0887b2e7a..3bff3f756 100644 --- a/src/validate.c +++ b/src/validate.c @@ -3,7 +3,7 @@ * validate.c: validate backup files. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/tests/archive_test.py b/tests/archive_test.py index 00fd1f592..9bd37aa55 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -8,7 +8,6 @@ import subprocess from sys import exit from time import sleep -from distutils.dir_util import copy_tree class ArchiveTest(ProbackupTest, unittest.TestCase): @@ -1243,10 +1242,6 @@ def test_archive_catalog(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) # FULL backup replica diff --git a/tests/backup_test.py b/tests/backup_test.py index dc60228b5..5ec6f1c6e 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -4,7 +4,6 @@ from time import sleep, time from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException import shutil -from distutils.dir_util import copy_tree from testgres import ProcessType, QueryException import subprocess @@ -2330,14 +2329,13 @@ def test_backup_with_less_privileges_role(self): # bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] # gdb_checkpointer = self.gdb_attach(bgwriter_pid) - copy_tree( - os.path.join(backup_dir, 'wal', 'node'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) - # self.switch_wal_segment(node) - # self.switch_wal_segment(node) + # make sure replica will archive wal segment with backup start point + lsn = self.switch_wal_segment(node, and_tx=True) + replica.poll_query_until(f"select pg_last_wal_replay_lsn() >= '{lsn}'") + replica.execute('CHECKPOINT') + replica.poll_query_until(f"select redo_lsn >= '{lsn}' from pg_control_checkpoint()") self.backup_node( backup_dir, 'replica', replica, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 27d982856..2420be462 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1701,29 +1701,26 @@ def version_to_num(self, version): num = num * 100 + int(re.sub(r"[^\d]", "", part)) return num - def switch_wal_segment(self, node): + def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): """ - Execute pg_switch_wal/xlog() in given node + Execute pg_switch_wal() in given node Args: node: an instance of PostgresNode or NodeConnection class """ if isinstance(node, testgres.PostgresNode): - if self.version_to_num( - node.safe_psql('postgres', 'show server_version').decode('utf-8') - ) >= self.version_to_num('10.0'): - node.safe_psql('postgres', 'select pg_switch_wal()') - else: - node.safe_psql('postgres', 'select pg_switch_xlog()') + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + con.execute('select pg_switch_wal()') + lsn = con.execute('select pg_switch_wal()')[0][0] else: - if self.version_to_num( - node.execute('show server_version')[0][0] - ) >= self.version_to_num('10.0'): - node.execute('select pg_switch_wal()') - else: - node.execute('select pg_switch_xlog()') + node.execute('select pg_switch_wal()') + lsn = node.execute('select pg_switch_wal()')[0][0] - sleep(1) + if sleep_seconds > 0: + sleep(sleep_seconds) + return lsn def wait_until_replica_catch_with_master(self, master, replica): diff --git a/tests/replica_test.py b/tests/replica_test.py index 17fc5a823..e96e0dfa7 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta import subprocess import time -from distutils.dir_util import copy_tree from testgres import ProcessType from time import sleep @@ -718,10 +717,6 @@ def test_replica_stop_lsn_null_offset_next_record(self): self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) self.switch_wal_segment(master) @@ -980,10 +975,6 @@ def test_replica_toast(self): self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) self.switch_wal_segment(master) diff --git a/tests/retention_test.py b/tests/retention_test.py index 88432a00f..cf422fe04 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from time import sleep -from distutils.dir_util import copy_tree class RetentionTest(ProbackupTest, unittest.TestCase):