From 33615789b01419462da414e58078bea4b4c754f8 Mon Sep 17 00:00:00 2001 From: Sergey Fukanchik Date: Fri, 17 Mar 2023 09:20:09 +0300 Subject: [PATCH 1/8] test NOJIRA --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index f93cc37a4..e449c9e52 100644 --- a/Makefile +++ b/Makefile @@ -87,3 +87,4 @@ endif include packaging/Makefile.pkg include packaging/Makefile.repo include packaging/Makefile.test + From a2b03fb633bbd3ecae5905534525f4738621ec7f Mon Sep 17 00:00:00 2001 From: Viktoria Shepard Date: Mon, 24 Nov 2025 21:36:07 +0100 Subject: [PATCH 2/8] Revert "[PBCKP-1238] Warning when using the community Version of pg_pro backup 2 for ENT" This reverts commit 98863ae0cdf1b5e7a10a9cb65db5d417527a8761 --- src/backup.c | 3 ++- src/pg_probackup.c | 8 +------- src/pg_probackup.h | 1 - 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/backup.c b/src/backup.c index e1ebdb2c7..add001a73 100644 --- a/src/backup.c +++ b/src/backup.c @@ -66,6 +66,7 @@ static bool pgpro_support(PGconn *conn); static bool pg_is_checksum_enabled(PGconn *conn); static bool pg_is_in_recovery(PGconn *conn); static bool pg_is_superuser(PGconn *conn); +static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); static void rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); static bool remove_excluded_files_criterion(void *value, void *exclude_args); @@ -946,7 +947,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, /* * Confirm that this server version is supported */ -void +static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) { PGresult *res = NULL; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b72efa93a..5cc604be9 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -980,13 +980,7 @@ main(int argc, char *argv[]) wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: - { - PGNodeInfo nodeInfo; - pgNodeInit(&nodeInfo); - instanceState->conn = pgut_connect(dbhost, dbport, dbname, dbuser); - check_server_version(instanceState->conn, &nodeInfo); - return do_add_instance(instanceState, &instance_config); - } + return do_add_instance(instanceState, &instance_config); case DELETE_INSTANCE_CMD: return do_delete_instance(instanceState); case INIT_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 9ae66a3a2..22a43a700 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1245,7 +1245,6 @@ extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); -void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); extern bool parse_page(Page page, XLogRecPtr *lsn); extern int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); From 3689d50d58cc7b9f8bc3a83d48baffed0628a7a0 Mon Sep 17 00:00:00 2001 From: Daria Date: Thu, 6 Nov 2025 22:28:19 +0100 Subject: [PATCH 3/8] PBCKP-2705: Adjust signal handling and compatibility for PostgreSQL 18 This patch adds compatibility changes for PostgreSQL 18: - Updated signal handler initialization in pgut.c to use sigaction() instead of pqsignal(), as pqsignal() now returns void. - Updated process_block_change() in backup.c to handle the new RelPathStr structure returned by relpathperm() in v18. - Updated CreateReplicationSlot_compat() in stream.c to support the new CreateReplicationSlot() signature introduced in v18. These changes ensure successful build and runtime behavior with PostgreSQL 18 while preserving backward compatibility with earlier versions. --- src/backup.c | 9 ++++++++- src/stream.c | 6 +++++- src/utils/pgut.c | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index add001a73..3cbd4fbf0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2518,11 +2518,16 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) int segno; pgFile **file_item; pgFile f; +#if PG_VERSION_NUM >= 180000 + RelPathStr rel_path_str = relpathperm(rnode, forknum); + rel_path = rel_path_str.str; +#else + rel_path = relpathperm(rnode, forknum); +#endif segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; - rel_path = relpathperm(rnode, forknum); if (segno > 0) f.rel_path = psprintf("%s.%u", rel_path, segno); else @@ -2554,7 +2559,9 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) if (segno > 0) pg_free(f.rel_path); +#if PG_VERSION_NUM < 180000 pg_free(rel_path); +#endif } diff --git a/src/stream.c b/src/stream.c index 25429f9a7..f136ed151 100644 --- a/src/stream.c +++ b/src/stream.c @@ -188,7 +188,11 @@ CreateReplicationSlot_compat(PGconn *conn, const char *slot_name, const char *pl bool is_temporary, bool is_physical, bool slot_exists_ok) { -#if PG_VERSION_NUM >= 150000 +#if PG_VERSION_NUM >= 180000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, + /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false, /* failover = */ false); +#elif PG_VERSION_NUM >= 150000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false); #elif PG_VERSION_NUM >= 110000 diff --git a/src/utils/pgut.c b/src/utils/pgut.c index df09dbdc6..003f0a559 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1062,7 +1062,23 @@ handle_interrupt(SIGNAL_ARGS) static void init_cancel_handler(void) { +#if PG_VERSION_NUM < 180000 oldhandler = pqsignal(SIGINT, handle_interrupt); +#else + { + struct sigaction act, oldact; + + act.sa_handler = handle_interrupt; + sigemptyset(&act.sa_mask); + act.sa_flags = SA_RESTART; + + /* Get the previous handler and set the new one */ + if (sigaction(SIGINT, &act, &oldact) < 0) + elog(ERROR, "sigaction(SIGINT) failed: %m"); + + oldhandler = oldact.sa_handler; + } +#endif pqsignal(SIGQUIT, handle_interrupt); pqsignal(SIGTERM, handle_interrupt); pqsignal(SIGPIPE, handle_interrupt); From 57339ae3aa33f922a546d7bd660fd39dac33ce91 Mon Sep 17 00:00:00 2001 From: Daria Date: Mon, 17 Nov 2025 03:40:52 +0100 Subject: [PATCH 4/8] PBCKP-2705: Add PostgreSQL core patch for fixing build with REL_18 In commit 760162f user-callable CRC functions were added. Since pg_probackup uses pg_crc.c from the PostgreSQL source, this change is leading to the undefined reference to pg_detoast_datum_packed errors during the build. See also https://www.postgresql.org/message-id/flat/ME0P300MB0445018F5C924B2D88983F45B66CA%40ME0P300MB0445.AUSP300.PROD.OUTLOOK.COM --- README.md | 6 +++-- patches/REL_18_STABLE_pg_probackup.patch | 29 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 patches/REL_18_STABLE_pg_probackup.patch diff --git a/README.md b/README.md index 16aec1dcc..da62364f3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 13, 14, 15, 16, 17 +* PostgreSQL 13, 14, 15, 16, 17, 18 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -79,7 +79,7 @@ For users of Postgres Pro products, commercial editions of pg_probackup are avai ## Building from source ### Linux -To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. Execute this in the module's directory: +To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. For versions under 18 execute this in the module's directory: ```shell make USE_PGXS=1 PG_CONFIG= top_srcdir= @@ -91,6 +91,8 @@ The alternative way, without using the PGXS infrastructure, is to place `pg_prob cd && git clone https://github.com/postgrespro/pg_probackup contrib/pg_probackup && cd contrib/pg_probackup && make ``` +For version 18 you have to apply PostgreSQL core patch (patches/REL_18_STABLE_pg_probackup.patch) first and recompile and reinstall PostgreSQL + ### Windows Currently pg_probackup can be build using only MSVC 2013. diff --git a/patches/REL_18_STABLE_pg_probackup.patch b/patches/REL_18_STABLE_pg_probackup.patch new file mode 100644 index 000000000..d47f0708b --- /dev/null +++ b/patches/REL_18_STABLE_pg_probackup.patch @@ -0,0 +1,29 @@ +From 5809673569d3d95c7bce75ea0aa6f9723e303c7d Mon Sep 17 00:00:00 2001 +From: Daria +Date: Mon, 17 Nov 2025 02:57:38 +0100 +Subject: [PATCH] REL_18_STABLE_pg_probackup + +--- + src/backend/utils/hash/pg_crc.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/backend/utils/hash/pg_crc.c b/src/backend/utils/hash/pg_crc.c +index e67a74ef852..6a474e804b5 100644 +--- a/src/backend/utils/hash/pg_crc.c ++++ b/src/backend/utils/hash/pg_crc.c +@@ -99,6 +99,7 @@ const uint32 pg_crc32_table[256] = { + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }; + ++#ifndef FRONTEND + /* + * SQL-callable functions + */ +@@ -128,3 +129,4 @@ crc32c_bytea(PG_FUNCTION_ARGS) + + PG_RETURN_INT64(crc); + } ++#endif +-- +2.39.5 (Apple Git-154) + From 21fc4aa21a3e93dbbf905408d137c3b2a970a6ad Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Fri, 5 Sep 2025 11:59:24 +0300 Subject: [PATCH 5/8] [PBCKP-2473] Skip block validation fixup for local mode only. Unfortunately remote mode needs to extend ssh agent protocol, that will broke backward compatibility. --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index d88b06b62..544adf182 100644 --- a/src/data.c +++ b/src/data.c @@ -1531,7 +1531,7 @@ validate_one_page(Page page, BlockNumber absolute_blkno, /* Verify checksum */ page_st->checksum = pg_checksum_page(page, absolute_blkno); - if (checksum_version) + if (checksum_version && !skip_block_validation) { /* Checksums are enabled, so check them. */ if (page_st->checksum != ((PageHeader) page)->pd_checksum) From 19f50332ac3e7aa643b22067f089515632a95620 Mon Sep 17 00:00:00 2001 From: Viktoria Shepard Date: Tue, 25 Nov 2025 13:26:48 +0100 Subject: [PATCH 6/8] PBCKP-2559 add error message to add instance for incompatible Postgres version --- src/util.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/util.c b/src/util.c index 019d20644..5189ba3b0 100644 --- a/src/util.c +++ b/src/util.c @@ -93,8 +93,13 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) #endif if (size != ControlFileSize) + { + if (size == 16384) + elog(ERROR, "Unexpected control file size %d, expected %d. Probably you trying to connect Postgres Pro using %s built with PostgreSQL. ", + (int) size, ControlFileSize, PROGRAM_NAME); elog(ERROR, "Unexpected control file size %d, expected %d", (int) size, ControlFileSize); + } memcpy(ControlFile, src, sizeof(ControlFileData)); From ebf71447aff8da91ebbb6bc38f196dde964bb215 Mon Sep 17 00:00:00 2001 From: oleg gurev Date: Thu, 27 Nov 2025 15:06:45 +0300 Subject: [PATCH 7/8] [PBCKP-2775] Test test_incr_lsn_long_xact_1 fix --- src/pg_probackup.c | 2 +- tests/archive_test.py | 6 +++ tests/backup_test.py | 4 ++ tests/catchup_test.py | 10 +++-- tests/checkdb_test.py | 4 +- tests/helpers/ptrack_helpers.py | 73 ++++++++++++++++++++++----------- tests/incr_restore_test.py | 14 +++---- tests/ptrack_test.py | 5 ++- tests/requirements.txt | 2 +- tests/set_backup_test.py | 3 ++ tests/validate_test.py | 17 ++++---- 11 files changed, 91 insertions(+), 49 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5cc604be9..030d64b00 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -687,7 +687,7 @@ main(int argc, char *argv[]) if (instance_config.pgdata != NULL && (backup_subcmd != ARCHIVE_GET_CMD && backup_subcmd != CATCHUP_CMD) && !is_absolute_path(instance_config.pgdata)) - elog(ERROR, "-D, --pgdata must be an absolute path"); + elog(ERROR, "-D, --pgdata must be an absolute path: %s", instance_config.pgdata); #if PG_VERSION_NUM >= 110000 /* Check xlog-seg-size option */ diff --git a/tests/archive_test.py b/tests/archive_test.py index 9bd37aa55..d2a141cb7 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -2437,6 +2437,9 @@ def test_archive_get_prefetch_corruption(self): prefetch_line = 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename) restored_line = 'LOG: restored log file "{0}" from archive'.format(filename) + + self.wait_server_wal_exists(replica.data_dir, wal_dir, filename) + tailer = tail_file(os.path.join(replica.logs_dir, 'postgresql.log')) tailer.wait(contains=prefetch_line) tailer.wait(contains=restored_line) @@ -2483,6 +2486,9 @@ def test_archive_show_partial_files_handling(self): self.switch_wal_segment(node) + self.wait_instance_wal_exists(backup_dir, 'node', + f"{filename}.gz") + os.rename( os.path.join(wals_dir, filename), os.path.join(wals_dir, '{0}.part'.format(filename))) diff --git a/tests/backup_test.py b/tests/backup_test.py index 5ec6f1c6e..5e4e4fc32 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -3218,6 +3218,10 @@ def test_missing_replication_permission_1(self): replica.promote() + # Wait for replica to catch up with master before promoting + # to ensure 'backup' role is replicated + self.wait_until_replica_catch_with_master(node, replica) + # PAGE output = self.backup_node( backup_dir, 'node', replica, backup_type='page', diff --git a/tests/catchup_test.py b/tests/catchup_test.py index cf8388dd2..2997a9c4d 100644 --- a/tests/catchup_test.py +++ b/tests/catchup_test.py @@ -1586,18 +1586,22 @@ def test_dry_run_catchup_delta(self): # Cleanup src_pg.stop() + # Skip test, because it's PGDATA is global variable and has impact for other tests + # e.g. test_checkdb_amcheck_only_sanity + @unittest.skip("skip") def test_pgdata_is_ignored(self): """ In catchup we still allow PGDATA to be set either from command line or from the env var. This test that PGDATA is actually ignored and --source-pgadta is used instead """ - node = self.make_simple_node('node', + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication = True ) node.slow_start() # do full catchup - dest = self.make_empty_node('dst') + dest = self.make_empty_node(base_dir=os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = node.data_dir, @@ -1612,7 +1616,7 @@ def test_pgdata_is_ignored(self): os.environ['PGDATA']='xxx' - dest2 = self.make_empty_node('dst') + dest2 = self.make_empty_node(base_dir=os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = node.data_dir, diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py index eb46aea19..f5c0f6895 100644 --- a/tests/checkdb_test.py +++ b/tests/checkdb_test.py @@ -18,7 +18,7 @@ def test_checkdb_amcheck_only_sanity(self): backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -227,7 +227,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): """""" backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 2420be462..76f9ea3ab 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -6,13 +6,14 @@ import signal import subprocess import shutil +from time import sleep import six import testgres import hashlib import re import getpass import select -from time import sleep +import time import re import json import random @@ -150,8 +151,9 @@ def __str__(self): class PostgresNodeExtended(testgres.PostgresNode): - def __init__(self, base_dir=None, *args, **kwargs): - super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, *args, **kwargs) + def __init__(self, base_dir=None, port = None, bin_dir=None, *args, **kwargs): + assert port is None or type(port) == int + super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, port = port, bin_dir=bin_dir, *args, **kwargs) self.is_started = False def slow_start(self, replica=False): @@ -414,25 +416,28 @@ def is_test_result_ok(test_case): # # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065 - if not isinstance(test_case, unittest.TestCase): - raise AssertionError("test_case is not instance of unittest.TestCase") - - if hasattr(test_case, '_outcome'): # Python 3.4+ - if hasattr(test_case._outcome, 'errors'): - # Python 3.4 - 3.10 (These two methods have no side effects) - result = test_case.defaultTestResult() # These two methods have no side effects - test_case._feedErrorsToResult(result, test_case._outcome.errors) - else: - # Python 3.11+ and pytest 5.3.5+ - result = test_case._outcome.result - if not hasattr(result, 'errors'): - result.errors = [] - if not hasattr(result, 'failures'): - result.failures = [] - else: # Python 2.7, 3.0-3.3 - result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) + if hasattr(test_case._outcome, 'errors'): + # Python 3.4 - 3.10 (These two methods have no side effects) + result = test_case.defaultTestResult() # These two methods have no side effects + test_case._feedErrorsToResult(result, test_case._outcome.errors) + else: + # Python 3.11+ and pytest 5.3.5+ + result = test_case._outcome.result + if not hasattr(result, 'errors'): + result.errors = [] + if not hasattr(result, 'failures'): + result.failures = [] ok = all(test != test_case for test, text in result.errors + result.failures) + # check subtests as well + ok = ok and all(getattr(test, 'test_case', None) != test_case + for test, text in result.errors + result.failures) + + # for pytest 8+ + if hasattr(result, '_excinfo'): + if result._excinfo is not None and len(result._excinfo) > 0: + # if test was successful, _excinfo will be None, else it will be non-empty list + ok = False return ok @@ -475,12 +480,14 @@ def pg_config_version(self): def make_empty_node( self, - base_dir=None): + base_dir=None, + port=None, + bin_dir=None): real_base_dir = os.path.join(self.tmp_path, base_dir) shutil.rmtree(real_base_dir, ignore_errors=True) os.makedirs(real_base_dir) - node = PostgresNodeExtended(base_dir=real_base_dir) + node = PostgresNodeExtended(base_dir=real_base_dir, port=port, bin_dir=bin_dir) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -489,12 +496,14 @@ def make_empty_node( def make_simple_node( self, base_dir=None, + port=None, + bin_dir=None, set_replication=False, ptrack_enable=False, initdb_params=[], pg_options={}): - node = self.make_empty_node(base_dir) + node = self.make_empty_node(base_dir, port=port, bin_dir=bin_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) @@ -910,6 +919,24 @@ def get_backup_filelist_diff(self, filelist_A, filelist_B): return filelist_diff + def wait_instance_wal_exists(self, backup_dir, instance, file, timeout=300): + """Wait for WAL segment appeared in the WAL archive""" + start = time.time() + fl = f'wal/{instance}/{file}' + while time.time() - start < timeout: + if os.path.exists(fl): + break + time.sleep(0.25) + + def wait_server_wal_exists(self, data_dir, wal_dir, file, timeout=300): + """Wait for WAL segment appeared in the server WAL dir""" + start = time.time() + fl = f'{data_dir}/{wal_dir}/{file}' + while time.time() - start < timeout: + if os.path.exists(fl): + return + time.sleep(0.25) + # used for partial restore def truncate_every_file_in_dir(self, path): for file in os.listdir(path): diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py index 6a2164098..1c8f49e81 100644 --- a/tests/incr_restore_test.py +++ b/tests/incr_restore_test.py @@ -1592,17 +1592,12 @@ def test_incr_checksum_long_xact(self): 'select count(*) from t1').decode('utf-8').rstrip(), '1') - # @unittest.skip("skip") - # @unittest.expectedFailure - # This test will pass with Enterprise - # because it has checksums enabled by default - @unittest.skipIf(ProbackupTest.enterprise, 'skip') def test_incr_lsn_long_xact_1(self): """ """ node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) + set_replication=True, initdb_params=['--no-data-checksums']) backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) @@ -2046,8 +2041,9 @@ def test_incremental_partial_restore_exclude_lsn(self): base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.make_empty_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2'), port=node.port) + assert node2.port == node.port node2.cleanup() # restore some data into node2 @@ -2063,7 +2059,7 @@ def test_incremental_partial_restore_exclude_lsn(self): pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore backup into node2 - node2.port = node.port + # node2.port = node.port node2.slow_start() node2.stop() self.restore_node( diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index 7b5bc416b..38317ea2c 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -1781,7 +1781,9 @@ def test_alter_database_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + port = node.port) + assert node_restored.port == node.port node_restored.cleanup() self.restore_node( backup_dir, 'node', @@ -1799,7 +1801,6 @@ def test_alter_database_set_tablespace_ptrack(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.port = node.port node_restored.slow_start() # @unittest.skip("skip") diff --git a/tests/requirements.txt b/tests/requirements.txt index 31e01aeb5..0a0331b6b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,7 +5,7 @@ # git+https://github.com/postgrespro/testgres.git@ # 3. From a local directory # /path/to/local/directory/testgres -testgres==1.8.5 +testgres==1.12.0 allure-pytest deprecation pexpect diff --git a/tests/set_backup_test.py b/tests/set_backup_test.py index 31334cfba..cda29b7a7 100644 --- a/tests/set_backup_test.py +++ b/tests/set_backup_test.py @@ -339,6 +339,9 @@ def test_wal_retention_and_pinning_1(self): node.pgbench_init(scale=2) + self.wait_instance_wal_exists(backup_dir, 'node', + "000000010000000000000004") + # Purge backups out = self.delete_expired( backup_dir, 'node', diff --git a/tests/validate_test.py b/tests/validate_test.py index 4ff44941f..e8c6587f6 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1711,7 +1711,7 @@ def test_validate_corrupt_wal_between_backups(self): wals_dir = os.path.join(backup_dir, 'wal', 'node') with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: f.seek(9000) - f.write(b"b") + f.write(b"Because the answer is 42") f.flush() f.close @@ -1722,20 +1722,17 @@ def test_validate_corrupt_wal_between_backups(self): 'node', backup_id, options=[ - "--xid={0}".format(target_xid), "-j", "4"]) + f"--xid={target_xid}", "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + f" Output: {repr(self.output)} \n CMD: {self.cmd}") except ProbackupException as e: self.assertTrue( 'ERROR: Not enough WAL records to xid' in e.message and 'WARNING: Recovery can be done up to time' in e.message and - "ERROR: Not enough WAL records to xid {0}\n".format( - target_xid), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + f"ERROR: Not enough WAL records to xid {target_xid}\n", + f'\n Unexpected Error Message: {repr(e.message)}\n CMD: {self.cmd}') self.assertEqual( 'OK', @@ -3397,6 +3394,10 @@ def test_corrupt_pg_control_via_resetxlog(self): os.path.join( backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'archive_status')) + os.mkdir( + os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'summaries')) + pg_control_path = os.path.join( backup_dir, 'backups', 'node', backup_id, 'database', 'global', 'pg_control') From 6b957f19e71c427a3be4d23da9134e9649670d65 Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Wed, 26 Nov 2025 12:30:17 +0300 Subject: [PATCH 8/8] [PBCKP-2775] Make auth_test.py multithreaded --- tests/auth_test.py | 108 ++++++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 46 deletions(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index 32cabc4a1..eaef4e582 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -174,51 +174,36 @@ def test_backup_via_unprivileged_user(self): class AuthTest(unittest.TestCase): pb = None node = None + test_path = None - # TODO move to object scope, replace module_name - @classmethod - def setUpClass(cls): + @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") + def setUp(self): - super(AuthTest, cls).setUpClass() + super().setUp() - cls.pb = ProbackupTest() - cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup') + self.pb = ProbackupTest() + self.test_path = os.path.join(self.pb.tmp_path, module_name, self._testMethodName) + self.backup_dir = os.path.join(self.test_path, 'backup') - cls.node = cls.pb.make_simple_node( - base_dir="{}/node".format(module_name), + self.node = self.pb.make_simple_node( + base_dir=os.path.join(self.test_path, 'node'), set_replication=True, initdb_params=['--data-checksums', '--auth-host=md5'] ) - cls.username = cls.pb.get_username() + self.modify_pg_hba(self.node, self.pb.get_username()) - cls.modify_pg_hba(cls.node) - - cls.pb.init_pb(cls.backup_dir) - cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) - cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) + self.pb.init_pb(self.backup_dir) + self.pb.add_instance(self.backup_dir, self.node.name, self.node) + self.pb.set_archiving(self.backup_dir, self.node.name, self.node) try: - cls.node.slow_start() + self.node.slow_start() except StartNodeException: raise unittest.skip("Node hasn't started") - if cls.pb.get_version(cls.node) < 100000: - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") - elif cls.pb.get_version(cls.node) < 150000: - cls.node.safe_psql( + version = self.pb.get_version(self.node) + if version < 150000: + self.node.safe_psql( "postgres", "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " @@ -233,7 +218,7 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") else: - cls.node.safe_psql( + self.node.safe_psql( "postgres", "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " @@ -247,16 +232,29 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") - cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') - - # TODO move to object scope, replace module_name - @classmethod - def tearDownClass(cls): - cls.node.cleanup() - cls.pb.del_test_dir(module_name, '') + if version >= 150000: + home_dir = os.path.join(self.test_path, "home") + os.makedirs(home_dir, exist_ok=True) + self.pb.test_env['HOME'] = home_dir + self.pgpass_file = os.path.join(home_dir, '.pgpass') + self.pgpass_file_lock = None + else: + # before PGv15 only true home dir were inspected. + # Since we can't have separate file per test, we have to serialize + # tests. + self.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + self.pgpass_file_lock = self.pgpass_file + '~probackup_test_lock' + # have to lock pgpass by creating file in exclusive mode + for i in range(120): + try: + open(self.pgpass_file_lock, "x").close() + except FileExistsError: + time.sleep(1) + else: + break + else: + raise TimeoutError("can't create ~/.pgpass~probackup_test_lock for 120 seconds") - @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") - def setUp(self): self.pb_cmd = ['backup', '-B', self.backup_dir, '--instance', self.node.name, @@ -268,6 +266,19 @@ def setUp(self): ] def tearDown(self): + if (self.pgpass_file_lock + and hasattr(self, "pgpass_line") + and os.path.exists(self.pgpass_file)): + with open(self.pgpass_file, 'r', encoding="utf-8") as fl: + lines = fl.readlines() + if self.pgpass_line in lines: + lines.remove(self.pgpass_line) + if len(lines) == 0: + os.remove(self.pgpass_file) + else: + with open(self.pgpass_file, 'w', encoding="utf-8") as fl: + fl.writelines(lines) + if "PGPASSWORD" in self.pb.test_env.keys(): del self.pb.test_env["PGPASSWORD"] @@ -279,6 +290,10 @@ def tearDown(self): except OSError: pass + test_path = os.path.join(self.pb.tmp_path, module_name) + self.node.cleanup() + self.pb.del_test_dir(test_path, self._testMethodName) + def test_empty_password(self): """ Test case: PGPB_AUTH03 - zero password length """ try: @@ -313,7 +328,7 @@ def test_ctrl_c_event(self): def test_pgpassfile_env(self): """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ - path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') + path = os.path.join(self.test_path, 'pgpass.conf') line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) self.create_pgpass(path, line) self.pb.test_env["PGPASSFILE"] = path @@ -367,7 +382,7 @@ def run_pb_with_auth(self, password=None, add_args = [], kill=False): @classmethod - def modify_pg_hba(cls, node): + def modify_pg_hba(self, node, username): """ Description: Add trust authentication for user postgres. Need for add new role and set grant. @@ -378,11 +393,12 @@ def modify_pg_hba(cls, node): with open(hba_conf, 'r+') as fio: data = fio.read() fio.seek(0) - fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (cls.username, data)) + fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (username, data)) def create_pgpass(self, path, line): + self.pgpass_line = line+"\n" with open(path, 'w') as passfile: # host:port:db:username:password - passfile.write(line) + passfile.write(self.pgpass_line) os.chmod(path, 0o600)