From f37c299e2f8adde503842c3dc2e95a4444917745 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 21:43:42 +0300 Subject: [PATCH 01/36] Using pytest [pytest.raises] --- tests/test_simple.py | 78 ++++++++++++++++++++----------------- tests/test_simple_remote.py | 64 ++++++++++++++++-------------- 2 files changed, 76 insertions(+), 66 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index a751f0a3..9d5388fe 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -9,6 +9,7 @@ import time import six import unittest +import pytest import psutil import logging.config @@ -134,7 +135,7 @@ def test_custom_init(self): def test_double_init(self): with get_new_node().init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): @@ -172,7 +173,7 @@ def test_init_unique_system_id(self): def test_node_exit(self): base_dir = None - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): with get_new_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') @@ -196,7 +197,7 @@ def test_double_start(self): def test_uninitialized_start(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): @@ -211,7 +212,7 @@ def test_restart(self): self.assertEqual(res, [(2, )]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() @@ -303,13 +304,13 @@ def test_psql(self): self.assertEqual(rm_carriage_returns(_sum), b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): @@ -320,11 +321,12 @@ def test_safe_psql__expect_error(self): self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) self.assertEqual(rm_carriage_returns(res), b'1\n') @@ -357,7 +359,7 @@ def test_control_data(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() @@ -374,7 +376,7 @@ def test_backup_simple(self): master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -411,15 +413,17 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): with get_new_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): @@ -551,7 +555,7 @@ def test_logical_replication(self): self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') @@ -588,7 +592,7 @@ def test_logical_catchup(self): @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): with get_new_node() as node: - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): @@ -599,7 +603,7 @@ def test_replication_slots(self): replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): @@ -607,7 +611,7 @@ def test_incorrect_catchup(self): node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): @@ -664,12 +668,12 @@ def test_poll_query_until(self): self.assertTrue(end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -682,7 +686,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -692,17 +696,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=(TimeoutException)): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -809,11 +813,11 @@ def test_pg_config(self): def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -827,7 +831,7 @@ def test_config_stack(self): stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass @@ -903,7 +907,7 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): @@ -980,7 +984,7 @@ def test_child_pids(self): with get_new_node().init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: @@ -1008,7 +1012,7 @@ def test_child_pids(self): replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender def test_child_process_dies(self): @@ -1061,11 +1065,12 @@ def test_the_same_port(self): self.assertEqual(node2.port, node.port) self.assertFalse(node2._should_free_port) - with self.assertRaises(StartNodeException) as ctx: + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): node2.init().start() - self.assertIn("Cannot start node", str(ctx.exception)) - # node is still working self.assertEqual(node.port, node_port_copy) self.assertTrue(node._should_free_port) @@ -1218,11 +1223,12 @@ def test_port_conflict(self): self.assertTrue(node2._should_free_port) self.assertEqual(node2.port, node1.port) - with self.assertRaises(StartNodeException) as ctx: + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts") + ): node2.init().start() - self.assertIn("Cannot start node after multiple attempts", str(ctx.exception)) - self.assertEqual(node2.port, node1.port) self.assertTrue(node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 2b581ac9..c548de03 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -10,6 +10,7 @@ import time import six import unittest +import pytest import psutil import logging.config @@ -195,7 +196,7 @@ def test_init__unk_LANG_and_LC_CTYPE(self): def test_double_init(self): with get_remote_node(conn_params=conn_params).init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): @@ -230,7 +231,7 @@ def test_init_unique_system_id(self): self.assertGreater(id2, id1) def test_node_exit(self): - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): with get_remote_node(conn_params=conn_params).init() as node: base_dir = node.base_dir node.safe_psql('select 1') @@ -254,7 +255,7 @@ def test_double_start(self): def test_uninitialized_start(self): with get_remote_node(conn_params=conn_params) as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): @@ -269,7 +270,7 @@ def test_restart(self): self.assertEqual(res, [(2,)]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() @@ -360,13 +361,13 @@ def test_psql(self): self.assertEqual(_sum, b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): @@ -377,11 +378,12 @@ def test_safe_psql__expect_error(self): self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) self.assertEqual(res, b'1\n') @@ -412,7 +414,7 @@ def test_transactions(self): def test_control_data(self): with get_remote_node(conn_params=conn_params) as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() @@ -428,7 +430,7 @@ def test_backup_simple(self): master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -465,15 +467,17 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): with get_remote_node(conn_params=conn_params) as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): @@ -605,7 +609,7 @@ def test_logical_replication(self): self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') @@ -642,7 +646,7 @@ def test_logical_catchup(self): @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): with get_remote_node(conn_params=conn_params) as node: - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): @@ -653,7 +657,7 @@ def test_replication_slots(self): replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): @@ -661,7 +665,7 @@ def test_incorrect_catchup(self): node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): @@ -717,12 +721,12 @@ def test_poll_query_until(self): self.assertTrue(end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -735,7 +739,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -745,17 +749,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -857,11 +861,11 @@ def test_pg_config(self): def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -875,7 +879,7 @@ def test_config_stack(self): stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass @@ -955,7 +959,7 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): @@ -1024,7 +1028,7 @@ def test_child_pids(self): with get_remote_node(conn_params=conn_params).init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: @@ -1052,7 +1056,7 @@ def test_child_pids(self): replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender def test_child_process_dies(self): From 53e468e28b2851b42c90c5886bf89f6528b62c9b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:13:45 +0300 Subject: [PATCH 02/36] Using pytest [pytest.skip] --- tests/test_simple.py | 52 ++++++++++++++++++++++++++++++++----- tests/test_simple_remote.py | 45 +++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 9d5388fe..936aed18 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -11,6 +11,7 @@ import unittest import pytest import psutil +import platform import logging.config @@ -144,10 +145,13 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): @@ -453,8 +457,10 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + with get_new_node() as master: old_version = not pg_version_ge('9.6') @@ -494,8 +500,10 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') self.assertEqual(rm_carriage_returns(res), b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -563,9 +571,11 @@ def test_logical_replication(self): res = node2.execute('select * from test2') self.assertListEqual(res, [('a', ), ('b', )]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -589,8 +599,10 @@ def test_logical_catchup(self): )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') + # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): + __class__.helper__skip_test_if_pg_version_is_ge("10") + with get_new_node() as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) @@ -766,8 +778,10 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') + # @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): + __class__.helper__skip_test_if_util_not_exist("pgbench") + with get_new_node().init().start() as node: # initialize pgbench DB and run benchmarks @@ -1312,6 +1326,30 @@ def test_set_auto_conf(self): x[0] + " stored wrong" ) + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + + if platform.system().lower() == "windows": + name2 = name + ".exe" + else: + name2 = name + + if not util_exists(name2): + pytest.skip('might be missing') + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index c548de03..28291acb 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -205,10 +205,13 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge('9.6') + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): @@ -507,8 +510,10 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + with get_remote_node(conn_params=conn_params) as master: old_version = not pg_version_ge('9.6') @@ -548,8 +553,10 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') self.assertEqual(res, b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: node1.init(allow_logical=True) node1.start() @@ -617,9 +624,11 @@ def test_logical_replication(self): res = node2.execute('select * from test2') self.assertListEqual(res, [('a',), ('b',)]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: node1.init(allow_logical=True) node1.start() @@ -643,8 +652,10 @@ def test_logical_catchup(self): )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') + # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): + __class__.helper__skip_test_if_pg_version_is_ge("10") + with get_remote_node(conn_params=conn_params) as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) @@ -820,8 +831,10 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') + # @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): + __class__.helper__skip_test_if_util_not_exist("pgbench") + with get_remote_node(conn_params=conn_params).init().start() as node: # initialize pgbench DB and run benchmarks node.pgbench_init(scale=2, foreign_keys=True, @@ -1077,6 +1090,24 @@ def helper__restore_envvar(name, prev_value): else: os.environ[name] = prev_value + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + if not util_exists(name): + pytest.skip('might be missing') + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) + if __name__ == '__main__': if os_ops.environ('ALT_CONFIG'): From 12d702ad7e653cfe29ff62790125c12c4b151feb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:29:44 +0300 Subject: [PATCH 03/36] Using pytest [assertIsNotNone] --- tests/test_simple.py | 4 ++-- tests/test_simple_remote.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 936aed18..23a0f581 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -111,7 +111,7 @@ class TestgresTests(unittest.TestCase): def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): with get_new_node() as node: @@ -370,7 +370,7 @@ def test_control_data(self): data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) + assert data is not None self.assertTrue(any('pg_control' in s for s in data.keys())) def test_backup_simple(self): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 28291acb..daad68a2 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -99,7 +99,7 @@ class TestgresRemoteTests(unittest.TestCase): def test_node_repr(self): with get_remote_node(conn_params=conn_params) as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): with get_remote_node(conn_params=conn_params) as node: @@ -424,7 +424,7 @@ def test_control_data(self): data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) + assert data is not None self.assertTrue(any('pg_control' in s for s in data.keys())) def test_backup_simple(self): @@ -1084,6 +1084,7 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + @staticmethod def helper__restore_envvar(name, prev_value): if prev_value is None: os.environ.pop(name, None) From 1cb664e5cd855cc63e0c6990dc689dc7fece8f6f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:35:58 +0300 Subject: [PATCH 04/36] Using pytest [assertFalse] --- tests/test_simple.py | 14 +++++++------- tests/test_simple_remote.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 23a0f581..293aee6c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -131,7 +131,7 @@ def test_custom_init(self): self.assertGreaterEqual(len(lines), 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_double_init(self): with get_new_node().init() as node: @@ -190,7 +190,7 @@ def test_node_exit(self): base_dir = node.base_dir # should have been removed by default - self.assertFalse(os.path.exists(base_dir)) + assert not (os.path.exists(base_dir)) def test_double_start(self): with get_new_node().init().start() as node: @@ -245,8 +245,8 @@ def test_pg_ctl(self): def test_status(self): self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation with get_new_node() as node: @@ -812,7 +812,7 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() @@ -1077,7 +1077,7 @@ def test_the_same_port(self): with get_new_node(port=node.port) as node2: self.assertEqual(type(node2.port), int) self.assertEqual(node2.port, node.port) - self.assertFalse(node2._should_free_port) + assert not (node2._should_free_port) with pytest.raises( expected_exception=StartNodeException, @@ -1247,7 +1247,7 @@ def test_port_conflict(self): self.assertTrue(node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertFalse(node2.is_started) + assert not (node2.is_started) # node2 must release our dummyPort (node1.port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index daad68a2..102f0eb5 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -118,7 +118,7 @@ def test_custom_init(self): self.assertGreaterEqual(len(lines), 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_init__LANG_ะก(self): # PBCKP-1744 @@ -247,7 +247,7 @@ def test_node_exit(self): base_dir = node.base_dir # should have been removed by default - self.assertFalse(os_ops.path_exists(base_dir)) + assert not (os_ops.path_exists(base_dir)) def test_double_start(self): with get_remote_node(conn_params=conn_params).init().start() as node: @@ -302,8 +302,8 @@ def test_pg_ctl(self): def test_status(self): self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation with get_remote_node(conn_params=conn_params) as node: @@ -859,7 +859,7 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() From e2e7e8b6d7d26bfa08a7e612a3ab4dc307b7ba11 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:38:10 +0300 Subject: [PATCH 05/36] Using pytest [assertTrue] --- tests/test_simple.py | 84 ++++++++++++++++++------------------- tests/test_simple_remote.py | 52 +++++++++++------------ 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 293aee6c..3489173b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -160,8 +160,8 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different with get_new_node().init().start() as node1, \ @@ -183,7 +183,7 @@ def test_node_exit(self): node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os.path.exists(base_dir)) + assert (os.path.exists(base_dir)) rmtree(base_dir, ignore_errors=True) with get_new_node().init() as node: @@ -196,7 +196,7 @@ def test_double_start(self): with get_new_node().init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): with get_new_node() as node: @@ -241,10 +241,10 @@ def test_pg_ctl(self): node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) + assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) @@ -320,7 +320,7 @@ def test_psql(self): def test_safe_psql__expect_error(self): with get_new_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 + assert (type(err) == str) # noqa: E721 self.assertIn('select_or_not_select', err) self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) @@ -371,7 +371,7 @@ def test_control_data(self): # check returned dict assert data is not None - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): with get_new_node() as master: @@ -651,9 +651,9 @@ def test_dump(self): with removing(node1.dump(format=format)) as dump: with get_new_node().init().start() as node3: if format == 'directory': - self.assertTrue(os.path.isdir(dump)) + assert (os.path.isdir(dump)) else: - self.assertTrue(os.path.isfile(dump)) + assert (os.path.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) @@ -677,7 +677,7 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns with pytest.raises(expected_exception=QueryException): @@ -770,13 +770,13 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) # @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): @@ -798,7 +798,7 @@ def test_pgbench(self): proc.stdout.close() - self.assertTrue('tps' in out) + assert ('tps' in out) def test_pg_config(self): # check same instances @@ -872,13 +872,13 @@ def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): from testgres.utils import file_tail @@ -957,21 +957,21 @@ def test_version_management(self): g = PgVer('15.3.1bihabeta1') k = PgVer('15.3.1') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) - self.assertTrue(h > f) - self.assertTrue(h == i) - self.assertTrue(g == k) - self.assertTrue(g > h) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) version = get_pg_version() with get_new_node() as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) self.assertEqual(node.version, PgVer(version)) def test_child_pids(self): @@ -1054,7 +1054,7 @@ def test_upgrade_node(self): node_new.init(cached=False) res = node_new.upgrade_from(old_node=node_old) node_new.start() - self.assertTrue(b'Upgrade Complete' in res) + assert (b'Upgrade Complete' in res) def test_parse_pg_version(self): # Linux Mint @@ -1069,7 +1069,7 @@ def test_parse_pg_version(self): def test_the_same_port(self): with get_new_node() as node: node.init().start() - self.assertTrue(node._should_free_port) + assert (node._should_free_port) self.assertEqual(type(node.port), int) node_port_copy = node.port self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') @@ -1087,7 +1087,7 @@ def test_the_same_port(self): # node is still working self.assertEqual(node.port, node_port_copy) - self.assertTrue(node._should_free_port) + assert (node._should_free_port) self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') class tagPortManagerProxy: @@ -1193,7 +1193,7 @@ def test_port_rereserve_during_node_start(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') @@ -1201,22 +1201,22 @@ def test_port_rereserve_during_node_start(self): with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(node2.port, node1.port) node2.init().start() self.assertNotEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertTrue(node2.is_started) + assert (node2.is_started) self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') # node1 is still working self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_port_conflict(self): @@ -1226,7 +1226,7 @@ def test_port_conflict(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') @@ -1234,7 +1234,7 @@ def test_port_conflict(self): with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(node2.port, node1.port) with pytest.raises( @@ -1244,7 +1244,7 @@ def test_port_conflict(self): node2.init().start() self.assertEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) assert not (node2.is_started) @@ -1254,7 +1254,7 @@ def test_port_conflict(self): # node1 is still working self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_simple_with_bin_dir(self): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 102f0eb5..f615e3aa 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -220,8 +220,8 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different with get_remote_node(conn_params=conn_params).init().start() as node1, \ @@ -240,7 +240,7 @@ def test_node_exit(self): node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os_ops.path_exists(base_dir)) + assert (os_ops.path_exists(base_dir)) os_ops.rmdirs(base_dir, ignore_errors=True) with get_remote_node(conn_params=conn_params).init() as node: @@ -253,7 +253,7 @@ def test_double_start(self): with get_remote_node(conn_params=conn_params).init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): with get_remote_node(conn_params=conn_params) as node: @@ -298,10 +298,10 @@ def test_pg_ctl(self): node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) + assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) @@ -376,7 +376,7 @@ def test_psql(self): def test_safe_psql__expect_error(self): with get_remote_node(conn_params=conn_params).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 + assert (type(err) == str) # noqa: E721 self.assertIn('select_or_not_select', err) self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) @@ -425,7 +425,7 @@ def test_control_data(self): # check returned dict assert data is not None - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): with get_remote_node(conn_params=conn_params) as master: @@ -704,9 +704,9 @@ def test_dump(self): with removing(node1.dump(format=format)) as dump: with get_remote_node(conn_params=conn_params).init().start() as node3: if format == 'directory': - self.assertTrue(os_ops.isdir(dump)) + assert (os_ops.isdir(dump)) else: - self.assertTrue(os_ops.isfile(dump)) + assert (os_ops.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) @@ -729,7 +729,7 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns with pytest.raises(expected_exception=QueryException): @@ -823,13 +823,13 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) # @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): @@ -845,7 +845,7 @@ def test_pgbench(self): stderr=subprocess.STDOUT, options=['-T3']) out = proc.communicate()[0] - self.assertTrue(b'tps = ' in out) + assert (b'tps = ' in out) def test_pg_config(self): # check same instances @@ -923,13 +923,13 @@ def test_auto_name(self): with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): from testgres.utils import file_tail @@ -1004,17 +1004,17 @@ def test_version_management(self): e = PgVer('15rc1') f = PgVer('15beta4') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) version = get_pg_version() with get_remote_node(conn_params=conn_params) as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) self.assertEqual(node.version, PgVer(version)) def test_child_pids(self): From 89784a82e601b06031628cb6b490c43a52cc5f00 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:57:23 +0300 Subject: [PATCH 06/36] Using pytest [assertEqual] --- tests/test_simple.py | 136 ++++++++++++++++++------------------ tests/test_simple_remote.py | 98 +++++++++++++------------- 2 files changed, 117 insertions(+), 117 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 3489173b..c3fcd495 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -210,10 +210,10 @@ def test_restart(self): # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1, )]) + assert (res == [(1, )]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2, )]) + assert (res == [(2, )]) # restart, fail with pytest.raises(expected_exception=StartNodeException): @@ -233,7 +233,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) + assert ('debug1' == cmm_new[0][0].lower()) self.assertNotEqual(cmm_old, cmm_new) def test_pg_ctl(self): @@ -250,62 +250,62 @@ def test_status(self): # check statuses after each operation with get_new_node() as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): with get_new_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(rm_carriage_returns(res), (0, b'1\n', b'')) + assert (rm_carriage_returns(res) == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(rm_carriage_returns(res), (0, b'2\n', b'')) + assert (rm_carriage_returns(res) == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), (0, b'3\n', b'')) + assert (rm_carriage_returns(res) == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(rm_carriage_returns(res), b'4\n') + assert (rm_carriage_returns(res) == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(rm_carriage_returns(res), b'5\n') + assert (rm_carriage_returns(res) == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), b'6\n') + assert (rm_carriage_returns(res) == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(rm_carriage_returns(_sum), b'6\n') + assert (rm_carriage_returns(_sum) == b'6\n') # check psql's default args, fails with pytest.raises(expected_exception=QueryException): @@ -333,7 +333,7 @@ def test_safe_psql__expect_error(self): # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_transactions(self): with get_new_node().init().start() as node: @@ -475,11 +475,11 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2)))) # yapf: disable # set synchronous_standby_names @@ -498,7 +498,7 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(rm_carriage_returns(res), b'1000000\n') + assert (rm_carriage_returns(res) == b'1000000\n') # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): @@ -638,7 +638,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -664,7 +664,7 @@ def test_users(self): node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = rm_carriage_returns(value) - self.assertEqual(value, b'1\n') + assert (value == b'1\n') def test_poll_query_until(self): with get_new_node() as node: @@ -804,7 +804,7 @@ def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -839,7 +839,7 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) @@ -849,12 +849,12 @@ def test_config_stack(self): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): with get_new_node() as node: @@ -897,13 +897,13 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): with get_new_node().init().start() as node: @@ -926,19 +926,19 @@ def test_isolation_levels(self): def test_ports_management(self): # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) with get_new_node() as node: # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert (len(bound_ports) == 1) # check that bound_ports contains our port port_1 = list(bound_ports)[0] port_2 = node.port - self.assertEqual(port_1, port_2) + assert (port_1 == port_2) # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -972,7 +972,7 @@ def test_version_management(self): with get_new_node() as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -1018,10 +1018,10 @@ def test_child_pids(self): self.assertIn(ptype, replica_pids) # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() @@ -1034,7 +1034,7 @@ def test_child_process_dies(self): cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - self.assertEqual(process.poll(), None) + assert (process.poll() == None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid @@ -1070,13 +1070,13 @@ def test_the_same_port(self): with get_new_node() as node: node.init().start() assert (node._should_free_port) - self.assertEqual(type(node.port), int) + assert (type(node.port) == int) node_port_copy = node.port - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') with get_new_node(port=node.port) as node2: - self.assertEqual(type(node2.port), int) - self.assertEqual(node2.port, node.port) + assert (type(node2.port) == int) + assert (node2.port == node.port) assert not (node2._should_free_port) with pytest.raises( @@ -1086,9 +1086,9 @@ def test_the_same_port(self): node2.init().start() # node is still working - self.assertEqual(node.port, node_port_copy) + assert (node.port == node_port_copy) assert (node._should_free_port) - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') + assert (rm_carriage_returns(node.safe_psql("SELECT 3;")) == b'3\n') class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -1194,30 +1194,30 @@ def test_port_rereserve_during_node_start(self): with get_new_node() as node1: node1.init().start() assert (node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: assert (node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2.port == node1.port) node2.init().start() self.assertNotEqual(node2.port, node1.port) assert (node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) assert (node2.is_started) - self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') + assert (rm_carriage_returns(node2.safe_psql("SELECT 2;")) == b'2\n') # node1 is still working - self.assertEqual(node1.port, node1_port_copy) + assert (node1.port == node1_port_copy) assert (node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_port_conflict(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 @@ -1227,15 +1227,15 @@ def test_port_conflict(self): with get_new_node() as node1: node1.init().start() assert (node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: assert (node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2.port == node1.port) with pytest.raises( expected_exception=StartNodeException, @@ -1243,19 +1243,19 @@ def test_port_conflict(self): ): node2.init().start() - self.assertEqual(node2.port, node1.port) + assert (node2.port == node1.port) assert (node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) assert not (node2.is_started) # node2 must release our dummyPort (node1.port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) # node1 is still working - self.assertEqual(node1.port, node1_port_copy) + assert (node1.port == node1_port_copy) assert (node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index f615e3aa..50fa26be 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -267,10 +267,10 @@ def test_restart(self): # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1,)]) + assert (res == [(1,)]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2,)]) + assert (res == [(2,)]) # restart, fail with pytest.raises(expected_exception=StartNodeException): @@ -290,7 +290,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) + assert ('debug1' == cmm_new[0][0].lower()) self.assertNotEqual(cmm_old, cmm_new) def test_pg_ctl(self): @@ -307,61 +307,61 @@ def test_status(self): # check statuses after each operation with get_remote_node(conn_params=conn_params) as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): with get_remote_node(conn_params=conn_params).init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) + assert (res == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) + assert (res == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) + assert (res == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') + assert (res == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') + assert (res == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') + assert (res == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') + assert (_sum == b'6\n') # check psql's default args, fails with pytest.raises(expected_exception=QueryException): @@ -389,7 +389,7 @@ def test_safe_psql__expect_error(self): # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_transactions(self): with get_remote_node(conn_params=conn_params).init().start() as node: @@ -528,11 +528,11 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2)))) # yapf: disable # set synchronous_standby_names @@ -551,7 +551,7 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') + assert (res == b'1000000\n') # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): @@ -691,7 +691,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -716,7 +716,7 @@ def test_users(self): with get_remote_node(conn_params=conn_params).init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') - self.assertEqual(b'1\n', value) + assert (b'1\n' == value) def test_poll_query_until(self): with get_remote_node(conn_params=conn_params) as node: @@ -851,7 +851,7 @@ def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -886,7 +886,7 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) @@ -896,12 +896,12 @@ def test_config_stack(self): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): with get_remote_node(conn_params=conn_params) as node: @@ -910,14 +910,14 @@ def test_unix_sockets(self): res_exec = node.execute('select 1') res_psql = node.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') with node.replicate().start() as r: res_exec = r.execute('select 1') res_psql = r.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') def test_auto_name(self): with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: @@ -948,13 +948,13 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): with get_remote_node(conn_params=conn_params).init().start() as node: @@ -977,19 +977,19 @@ def test_isolation_levels(self): def test_ports_management(self): # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) with get_remote_node(conn_params=conn_params) as node: # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert (len(bound_ports) == 1) # check that bound_ports contains our port port_1 = list(bound_ports)[0] port_2 = node.port - self.assertEqual(port_1, port_2) + assert (port_1 == port_2) # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -1015,7 +1015,7 @@ def test_version_management(self): with get_remote_node(conn_params=conn_params) as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -1061,10 +1061,10 @@ def test_child_pids(self): self.assertIn(ptype, replica_pids) # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() @@ -1075,7 +1075,7 @@ def test_child_pids(self): def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function with subprocess.Popen(["sleep", "60"]) as process: - self.assertEqual(process.poll(), None) + assert (process.poll() == None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid From 2252d60fd25cdf9fedd1d1a99fa8adc4a2f1100c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:04:19 +0300 Subject: [PATCH 07/36] Using pytest [assertNotEqual] --- tests/test_simple.py | 16 ++++++++-------- tests/test_simple_remote.py | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index c3fcd495..70b053c7 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -234,7 +234,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') assert ('debug1' == cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert (cmm_old != cmm_new) def test_pg_ctl(self): with get_new_node() as node: @@ -260,7 +260,7 @@ def test_status(self): node.start() - self.assertNotEqual(node.pid, 0) + assert (node.pid != 0) assert (node.status() == NodeStatus.Running) node.stop() @@ -400,12 +400,12 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): with get_new_node() as node: @@ -818,12 +818,12 @@ def test_pg_config(self): c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option @@ -876,7 +876,7 @@ def test_auto_name(self): assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) + assert (m.name != r.name) assert ('testgres' in m.name) assert ('testgres' in r.name) @@ -1206,7 +1206,7 @@ def test_port_rereserve_during_node_start(self): node2.init().start() - self.assertNotEqual(node2.port, node1.port) + assert (node2.port != node1.port) assert (node2._should_free_port) assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 50fa26be..0a72162b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -291,7 +291,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') assert ('debug1' == cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert (cmm_old != cmm_new) def test_pg_ctl(self): with get_remote_node(conn_params=conn_params) as node: @@ -317,7 +317,7 @@ def test_status(self): node.start() - self.assertNotEqual(node.pid, 0) + assert (node.pid != 0) assert (node.status() == NodeStatus.Running) node.stop() @@ -453,12 +453,12 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): with get_remote_node(conn_params=conn_params) as node: @@ -865,12 +865,12 @@ def test_pg_config(self): c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option @@ -927,7 +927,7 @@ def test_auto_name(self): assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) + assert (m.name != r.name) assert ('testgres' in m.name) assert ('testgres' in r.name) From e8aad3d46cb39a77b284af0bb099a93983a26d0b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:06:05 +0300 Subject: [PATCH 08/36] Using pytest [assertGreaterEqual] --- tests/test_simple.py | 2 +- tests/test_simple_remote.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 70b053c7..448fbf01 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -128,7 +128,7 @@ def test_custom_init(self): lines = conf.readlines() # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all assert not (any('trust' in s for s in lines)) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 0a72162b..8f36a33b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -115,7 +115,7 @@ def test_custom_init(self): lines = os_ops.readlines(hba_file) # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all assert not (any('trust' in s for s in lines)) From eaaa276eef770e5612bffcb255ac2d888b608eca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:13:34 +0300 Subject: [PATCH 09/36] Using pytest [assertGreater] --- tests/test_simple.py | 6 +++--- tests/test_simple_remote.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 448fbf01..5db5b20e 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -171,8 +171,8 @@ def test_init_unique_system_id(self): id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): base_dir = None @@ -1002,7 +1002,7 @@ def test_child_pids(self): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 8f36a33b..79e5c133 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -230,8 +230,8 @@ def test_init_unique_system_id(self): id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): with pytest.raises(expected_exception=QueryException): @@ -1045,7 +1045,7 @@ def test_child_pids(self): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: From 4afbe00257832ac87c0ba114a279a187a8858851 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:16:26 +0300 Subject: [PATCH 10/36] Using pytest [assertIn] --- tests/test_simple.py | 15 +++++---------- tests/test_simple_remote.py | 8 ++++---- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 5db5b20e..ea7494b9 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -321,8 +321,8 @@ def test_safe_psql__expect_error(self): with get_new_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- with pytest.raises( @@ -1011,11 +1011,11 @@ def test_child_pids(self): master_pids = master.auxiliary_pids for ptype in master_processes: - self.assertIn(ptype, master_pids) + assert (ptype in master_pids) replica_pids = replica.auxiliary_pids for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + assert (ptype in replica_pids) # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) @@ -1320,12 +1320,7 @@ def test_set_auto_conf(self): content = f.read() for x in testData: - self.assertIn( - x[0] + " = " + x[2], - content, - x[0] + " stored wrong" - ) - + assert x[0] + " = " + x[2] in content @staticmethod def helper__skip_test_if_util_not_exist(name: str): assert type(name) == str # noqa: E721 diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 79e5c133..5bcb7755 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -377,8 +377,8 @@ def test_safe_psql__expect_error(self): with get_remote_node(conn_params=conn_params).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- with pytest.raises( @@ -1054,11 +1054,11 @@ def test_child_pids(self): master_pids = master.auxiliary_pids for ptype in master_processes: - self.assertIn(ptype, master_pids) + assert (ptype in master_pids) replica_pids = replica.auxiliary_pids for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + assert (ptype in replica_pids) # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) From a8e57ae01cba107a3ef46599c4d901d809be7a93 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:21:01 +0300 Subject: [PATCH 11/36] Using pytest [assertListEqual] --- tests/test_simple.py | 27 ++++++++++++--------------- tests/test_simple_remote.py | 27 ++++++++++++--------------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index ea7494b9..fc5a5300 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -347,12 +347,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) con.rollback() con.begin() @@ -392,7 +392,7 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) + assert (res == [(1, ), (2, ), (3, ), (4, )]) def test_backup_multiple(self): with get_new_node() as node: @@ -448,14 +448,14 @@ def test_replicate(self): with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): @@ -522,7 +522,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -532,7 +532,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -546,7 +546,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) # drop subscription sub.drop() @@ -560,7 +560,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table with pytest.raises(expected_exception=ValueError): @@ -569,7 +569,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): @@ -593,10 +593,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i,i,)]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -657,7 +654,7 @@ def test_dump(self): # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) def test_users(self): with get_new_node().init().start() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 5bcb7755..b833077b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -402,12 +402,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) con.rollback() con.begin() @@ -445,7 +445,7 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1,), (2,), (3,), (4,)]) + assert (res == [(1,), (2,), (3,), (4,)]) def test_backup_multiple(self): with get_remote_node(conn_params=conn_params) as node: @@ -501,14 +501,14 @@ def test_replicate(self): with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): @@ -575,7 +575,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -585,7 +585,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -599,7 +599,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) # drop subscription sub.drop() @@ -613,7 +613,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table with pytest.raises(expected_exception=ValueError): @@ -622,7 +622,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): @@ -646,10 +646,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i,i,)]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -710,7 +707,7 @@ def test_dump(self): # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) def test_users(self): with get_remote_node(conn_params=conn_params).init().start() as node: From df199b92d7d9534dc4ddf9fcdb491d1a559ab44c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:25:39 +0300 Subject: [PATCH 12/36] unittest is not used --- tests/test_simple.py | 23 +---------------------- tests/test_simple_remote.py | 23 +---------------------- 2 files changed, 2 insertions(+), 44 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index fc5a5300..36f67537 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -8,7 +8,6 @@ import testgres import time import six -import unittest import pytest import psutil import platform @@ -107,7 +106,7 @@ def removing(f): rmtree(f, ignore_errors=True) -class TestgresTests(unittest.TestCase): +class TestgresTests: def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" @@ -1342,23 +1341,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - -if __name__ == '__main__': - if os.environ.get('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresTests('test_pg_config')) - suite.addTest(TestgresTests('test_pg_ctl')) - suite.addTest(TestgresTests('test_psql')) - suite.addTest(TestgresTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index b833077b..9bd7216c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -9,7 +9,6 @@ import testgres import time import six -import unittest import pytest import psutil @@ -95,7 +94,7 @@ def removing(f): os_ops.rmdirs(f, ignore_errors=True) -class TestgresRemoteTests(unittest.TestCase): +class TestgresRemoteTests: def test_node_repr(self): with get_remote_node(conn_params=conn_params) as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" @@ -1106,23 +1105,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - -if __name__ == '__main__': - if os_ops.environ('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresRemoteTests('test_pg_config')) - suite.addTest(TestgresRemoteTests('test_pg_ctl')) - suite.addTest(TestgresRemoteTests('test_psql')) - suite.addTest(TestgresRemoteTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() From a82cf3dbd159a4a52446673a2f687dbc48324128 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:33:55 +0300 Subject: [PATCH 13/36] Code style (flake8) --- tests/test_simple.py | 18 +++++++++--------- tests/test_simple_remote.py | 13 ++++++------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 36f67537..278a63a3 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -475,11 +475,11 @@ def test_synchronous_replication(self): # check formatting assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(First(1, (standby1, standby2)))) # yapf: disable + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(Any(1, (standby1, standby2)))) # yapf: disable + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -592,7 +592,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - assert (res == [(i,i,)]) + assert (res == [(i, i, )]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -1030,7 +1030,7 @@ def test_child_process_dies(self): cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - assert (process.poll() == None) + assert (process.poll() is None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid @@ -1066,12 +1066,12 @@ def test_the_same_port(self): with get_new_node() as node: node.init().start() assert (node._should_free_port) - assert (type(node.port) == int) + assert (type(node.port) == int) # noqa: E721 node_port_copy = node.port assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') with get_new_node(port=node.port) as node2: - assert (type(node2.port) == int) + assert (type(node2.port) == int) # noqa: E721 assert (node2.port == node.port) assert not (node2._should_free_port) @@ -1317,6 +1317,7 @@ def test_set_auto_conf(self): for x in testData: assert x[0] + " = " + x[2] in content + @staticmethod def helper__skip_test_if_util_not_exist(name: str): assert type(name) == str # noqa: E721 @@ -1340,4 +1341,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): assert type(version) == str # noqa: E721 if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 9bd7216c..e9be7ac4 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -528,11 +528,11 @@ def test_synchronous_replication(self): # check formatting assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(First(1, (standby1, standby2)))) # yapf: disable + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(Any(1, (standby1, standby2)))) # yapf: disable + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -645,7 +645,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - assert (res == [(i,i,)]) + assert (res == [(i, i, )]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -1071,7 +1071,7 @@ def test_child_pids(self): def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function with subprocess.Popen(["sleep", "60"]) as process: - assert (process.poll() == None) + assert (process.poll() is None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid @@ -1104,4 +1104,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): assert type(version) == str # noqa: E721 if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - From 27c4f408c92565a9752faee193f8eaf3a2839f80 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 06:45:47 +0300 Subject: [PATCH 14/36] Execution signature is removed --- tests/test_simple.py | 3 --- tests/test_simple_remote.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 278a63a3..55659e86 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# coding: utf-8 - import os import re import subprocess diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index e9be7ac4..cfd92ac6 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# coding: utf-8 - import os import re import subprocess From ae41d166df57a7122cfbeef8d09b27c8aff21b5e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 07:03:00 +0300 Subject: [PATCH 15/36] run_tests.sh installs pytest --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 73c459be..a3549242 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,7 +22,7 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx +$PIP install coverage flake8 psutil Sphinx pytest # install testgres' dependencies export PYTHONPATH=$(pwd) From a14b3d787d3ce56244399053f43f0baebfa89230 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 10:21:30 +0300 Subject: [PATCH 16/36] run_tests.sh is updated run tests through pytest explicitly --- run_tests.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index a3549242..5e7def86 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,7 +22,7 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx pytest +$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist # install testgres' dependencies export PYTHONPATH=$(pwd) @@ -38,21 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a tests/test_simple.py +time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # show coverage From e6f4448bd992cbe28f67cccfae40c008bc5a3596 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 11:51:28 +0300 Subject: [PATCH 17/36] Total refactoring of tests - TestgresRemoteTests does not use global variables and code - Explicit work with ..testgres folder --- run_tests.sh | 4 +- tests/test_local.py | 6 +- tests/test_remote.py | 8 +- tests/test_simple.py | 22 ++--- tests/test_simple_remote.py | 175 ++++++++++++++++++++---------------- 5 files changed, 116 insertions(+), 99 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 5e7def86..6995b7e4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,11 +22,11 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist +$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil # install testgres' dependencies export PYTHONPATH=$(pwd) -$PIP install . +# $PIP install . # test code quality flake8 . diff --git a/tests/test_local.py b/tests/test_local.py index 4051bfb5..346336bd 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -4,9 +4,9 @@ import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import LocalOperations +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import LocalOperations from .helpers.run_conditions import RunConditions diff --git a/tests/test_remote.py b/tests/test_remote.py index 4330b92f..21bc9e8f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -4,10 +4,10 @@ import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import RemoteOperations -from testgres import ConnectionParams +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import RemoteOperations +from ..testgres import ConnectionParams class TestRemoteOperations: diff --git a/tests/test_simple.py b/tests/test_simple.py index 55659e86..30440a72 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,7 +2,6 @@ import re import subprocess import tempfile -import testgres import time import six import pytest @@ -14,7 +13,9 @@ from contextlib import contextmanager from shutil import rmtree -from testgres import \ +from .. import testgres + +from ..testgres import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -25,31 +26,32 @@ InvalidOperationException, \ NodeApp -from testgres import \ +from ..testgres import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_new_node -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer, parse_pg_version -from testgres.node import ProcessProxy +from ..testgres import bound_ports +from ..testgres.utils import PgVer, parse_pg_version +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy def pg_version_ge(version): @@ -874,8 +876,6 @@ def test_auto_name(self): assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index cfd92ac6..403b3c49 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -3,7 +3,6 @@ import subprocess import tempfile -import testgres import time import six import pytest @@ -13,7 +12,9 @@ from contextlib import contextmanager -from testgres.exceptions import \ +from .. import testgres + +from ..testgres.exceptions import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -23,39 +24,33 @@ TestgresException, \ InvalidOperationException -from testgres.config import \ +from ..testgres.config import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config, testgres_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_remote_node, \ RemoteOperations -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer -from testgres.node import ProcessProxy, ConnectionParams - -conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) -os_ops = RemoteOperations(conn_params) -testgres_config.set_os_ops(os_ops=os_ops) - +from ..testgres import bound_ports +from ..testgres.utils import PgVer +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy, ConnectionParams def pg_version_ge(version): cur_ver = PgVer(get_pg_version()) @@ -65,16 +60,16 @@ def pg_version_ge(version): def util_exists(util): def good_properties(f): - return (os_ops.path_exists(f) and # noqa: W504 - os_ops.isfile(f) and # noqa: W504 - os_ops.is_executable(f)) # yapf: disable + return (testgres_config.os_ops.path_exists(f) and # noqa: W504 + testgres_config.os_ops.isfile(f) and # noqa: W504 + testgres_config.os_ops.is_executable(f)) # yapf: disable # try to resolve it if good_properties(get_bin_path(util)): return True # check if util is in PATH - for path in os_ops.environ("PATH").split(os_ops.pathsep): + for path in testgres_config.os_ops.environ("PATH").split(testgres_config.os_ops.pathsep): if good_properties(os.path.join(path, util)): return True @@ -84,31 +79,50 @@ def removing(f): try: yield f finally: - if os_ops.isfile(f): - os_ops.remove_file(f) + if testgres_config.os_ops.isfile(f): + testgres_config.os_ops.remove_file(f) - elif os_ops.isdir(f): - os_ops.rmdirs(f, ignore_errors=True) + elif testgres_config.os_ops.isdir(f): + testgres_config.os_ops.rmdirs(f, ignore_errors=True) class TestgresRemoteTests: + sm_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_os_ops = RemoteOperations(sm_conn_params) + + @pytest.fixture(autouse=True, scope="class") + def implicit_fixture(self): + prev_ops = testgres_config.os_ops + assert prev_ops is not None + assert __class__.sm_os_ops is not None + testgres_config.set_os_ops(os_ops=__class__.sm_os_ops) + assert testgres_config.os_ops is __class__.sm_os_ops + yield + assert testgres_config.os_ops is __class__.sm_os_ops + testgres_config.set_os_ops(os_ops=prev_ops) + assert testgres_config.os_ops is prev_ops + def test_node_repr(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" assert re.match(pattern, str(node)) is not None def test_custom_init(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # enable page checksums node.init(initdb_params=['-k']).start() - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init( allow_streaming=True, initdb_params=['--auth-local=reject', '--auth-host=reject']) hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - lines = os_ops.readlines(hba_file) + lines = node.os_ops.readlines(hba_file) # check number of lines assert (len(lines) >= 6) @@ -123,7 +137,7 @@ def test_init__LANG_ะก(self): try: os.environ["LANG"] = "C" - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() finally: __class__.helper__restore_envvar("LANG", prev_LANG) @@ -164,9 +178,9 @@ def test_init__unk_LANG_and_LC_CTYPE(self): while True: try: - with get_remote_node(conn_params=conn_params): + with __class__.helper__get_node(): pass - except testgres.exceptions.ExecUtilException as e: + except ExecUtilException as e: # # Example of an error message: # @@ -190,13 +204,13 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) def test_double_init(self): - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: # can't initialize node more than once with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') @@ -211,7 +225,7 @@ def test_init_unique_system_id(self): query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): - with get_remote_node(conn_params=conn_params).init().start() as node0: + with __class__.helper__get_node().init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, @@ -220,8 +234,8 @@ def test_init_unique_system_id(self): assert (config.cached_initdb_unique) # spawn two nodes; ids must be different - with get_remote_node(conn_params=conn_params).init().start() as node1, \ - get_remote_node(conn_params=conn_params).init().start() as node2: + with __class__.helper__get_node().init().start() as node1, \ + __class__.helper__get_node().init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] @@ -231,34 +245,34 @@ def test_init_unique_system_id(self): def test_node_exit(self): with pytest.raises(expected_exception=QueryException): - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - assert (os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) + assert (__class__.sm_os_ops.path_exists(base_dir)) + __class__.sm_os_ops.rmdirs(base_dir, ignore_errors=True) - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: base_dir = node.base_dir # should have been removed by default - assert not (os_ops.path_exists(base_dir)) + assert not (__class__.sm_os_ops.path_exists(base_dir)) def test_double_start(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # can't start node more than once node.start() assert (node.is_started) def test_uninitialized_start(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # restart, ok @@ -274,7 +288,7 @@ def test_restart(self): node.restart() def test_reload(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # change client_min_messages and save old value @@ -290,7 +304,7 @@ def test_reload(self): assert (cmm_old != cmm_new) def test_pg_ctl(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() status = node.pg_ctl(['status']) @@ -302,7 +316,7 @@ def test_status(self): assert not (NodeStatus.Uninitialized) # check statuses after each operation - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) @@ -327,7 +341,7 @@ def test_status(self): assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') assert (res == (0, b'1\n', b'')) @@ -370,7 +384,7 @@ def test_psql(self): node.safe_psql('select 1') def test_safe_psql__expect_error(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 assert ('select_or_not_select' in err) @@ -388,7 +402,7 @@ def test_safe_psql__expect_error(self): assert (res == b'1\n') def test_transactions(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: con.begin() con.execute('create table test(val int)') @@ -411,7 +425,7 @@ def test_transactions(self): con.commit() def test_control_data(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() @@ -424,7 +438,7 @@ def test_control_data(self): assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: # enable streaming for backups master.init(allow_streaming=True) @@ -444,7 +458,7 @@ def test_backup_simple(self): assert (res == [(1,), (2,), (3,), (4,)]) def test_backup_multiple(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ @@ -457,7 +471,7 @@ def test_backup_multiple(self): assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: @@ -470,7 +484,7 @@ def test_backup_exhaust(self): backup.spawn_primary() def test_backup_wrong_xlog_method(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with pytest.raises( @@ -480,7 +494,7 @@ def test_backup_wrong_xlog_method(self): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start(wait=False) while True: try: @@ -492,7 +506,7 @@ def test_pg_ctl_wait_option(self): pass def test_replicate(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: @@ -510,7 +524,7 @@ def test_replicate(self): def test_synchronous_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: old_version = not pg_version_ge('9.6') master.init(allow_streaming=True).start() @@ -553,7 +567,7 @@ def test_synchronous_replication(self): def test_logical_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("10") - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -625,7 +639,7 @@ def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ __class__.helper__skip_test_if_pg_version_is_not_ge("10") - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -649,12 +663,12 @@ def test_logical_catchup(self): def test_logical_replication_fail(self): __class__.helper__skip_test_if_pg_version_is_ge("10") - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: @@ -665,7 +679,7 @@ def test_replication_slots(self): node.replicate(slot='slot1') def test_incorrect_catchup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() # node has no master, can't catch up @@ -673,7 +687,7 @@ def test_incorrect_catchup(self): node.catchup() def test_promotion(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: master.init().start() master.safe_psql('create table abc(id serial)') @@ -690,29 +704,29 @@ def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' - with get_remote_node(conn_params=conn_params).init().start() as node1: + with __class__.helper__get_node().init().start() as node1: node1.execute(query_create) for format in ['plain', 'custom', 'directory', 'tar']: with removing(node1.dump(format=format)) as dump: - with get_remote_node(conn_params=conn_params).init().start() as node3: + with __class__.helper__get_node().init().start() as node3: if format == 'directory': - assert (os_ops.isdir(dump)) + assert (node1.os_ops.isdir(dump)) else: - assert (os_ops.isfile(dump)) + assert (node1.os_ops.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) assert (res == [(1,), (2,)]) def test_users(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') assert (b'1\n' == value) def test_poll_query_until(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -828,7 +842,7 @@ def test_logging(self): def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # initialize pgbench DB and run benchmarks node.pgbench_init(scale=2, foreign_keys=True, options=['-q']).pgbench_run(time=2) @@ -897,7 +911,7 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(unix_sockets=False, allow_streaming=True) node.start() @@ -913,7 +927,7 @@ def test_unix_sockets(self): assert (res_psql == b'1\n') def test_auto_name(self): - with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: + with __class__.helper__get_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running assert (m.status()) @@ -925,8 +939,6 @@ def test_auto_name(self): assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" @@ -950,7 +962,7 @@ def test_file_tail(self): assert (lines[0] == s3) def test_isolation_levels(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -972,7 +984,7 @@ def test_ports_management(self): # check that no ports have been bound yet assert (len(bound_ports) == 0) - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # check that we've just bound a port assert (len(bound_ports) == 1) @@ -1005,7 +1017,7 @@ def test_version_management(self): assert (d > f) version = get_pg_version() - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) @@ -1031,7 +1043,7 @@ def test_child_pids(self): ProcessType.WalReceiver, ] - with get_remote_node(conn_params=conn_params).init().start() as master: + with __class__.helper__get_node().init().start() as master: # master node doesn't have a source walsender! with pytest.raises(expected_exception=TestgresException): @@ -1077,6 +1089,11 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + @staticmethod + def helper__get_node(): + assert __class__.sm_conn_params is not None + return get_remote_node(conn_params=__class__.sm_conn_params) + @staticmethod def helper__restore_envvar(name, prev_value): if prev_value is None: From 842bc086ba77140bce470b3e5de82f991121a5c2 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 11:53:26 +0300 Subject: [PATCH 18/36] Code style (flake8) --- tests/test_simple_remote.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 403b3c49..e0bc265e 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -52,6 +52,7 @@ from ..testgres.utils import file_tail from ..testgres.node import ProcessProxy, ConnectionParams + def pg_version_ge(version): cur_ver = PgVer(get_pg_version()) min_ver = PgVer(version) @@ -93,7 +94,7 @@ class TestgresRemoteTests: ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) sm_os_ops = RemoteOperations(sm_conn_params) - + @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): prev_ops = testgres_config.os_ops From 969e49d49ec4d942d5a1a37acda78d5e3cc299ed Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 16:26:06 +0300 Subject: [PATCH 19/36] Root __init__.py is added It is required for tests. --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b From 6e2820df688c70a57ceaf670b39bebb075ec4ed8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 16:34:41 +0300 Subject: [PATCH 20/36] Code style (flake8) --- testgres/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 61d44868..3ac63326 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -32,11 +32,11 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo def make_utility_path(name): assert name is not None - assert type(name) == str + assert type(name) == str # noqa: E721 if bin_path: return os.path.join(bin_path, name) - + return get_bin_path2(os_ops, name) def call_initdb(initdb_dir, log=logfile): From e4e890905ebfe7d34bc1037adfe25897c18b5a47 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 17:43:56 +0300 Subject: [PATCH 21/36] pytest.ini is added --- pytest.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..61d4f9bc --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = ./tests +addopts = --strict-markers +markers = +#log_file = logs/pytest.log +log_file_level = NOTSET +log_file_format = %(levelname)8s [%(asctime)s] %(message)s +log_file_date_format=%Y-%m-%d %H:%M:%S + From 66860d0a76ca924e5aa8821542db636bdaa5f540 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 18:51:26 +0300 Subject: [PATCH 22/36] TestgresTests::test_ports_management is corrected Let's send warning about a garbage in the container "bound_ports" and continue working. --- tests/test_simple.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 30440a72..15a26e1b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -918,20 +918,38 @@ def test_isolation_levels(self): con.begin('Garbage').commit() def test_ports_management(self): - # check that no ports have been bound yet - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + if len(bound_ports) != 0: + logging.warning("bound_ports is not empty: {0}".format(bound_ports)) + + stage0__bound_ports = bound_ports.copy() with get_new_node() as node: - # check that we've just bound a port - assert (len(bound_ports) == 1) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + assert node.port is not None + assert type(node.port) == int # noqa: E721 + + logging.info("node port is {0}".format(node.port)) + + assert node.port in bound_ports + assert node.port not in stage0__bound_ports + + assert stage0__bound_ports <= bound_ports + assert len(stage0__bound_ports) + 1 == len(bound_ports) + + stage1__bound_ports = stage0__bound_ports.copy() + stage1__bound_ports.add(node.port) - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - assert (port_1 == port_2) + assert stage1__bound_ports == bound_ports # check that port has been freed successfully - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + assert bound_ports == stage0__bound_ports def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) From aaaa677e20ac97af667a381c29246b021f3077a4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 18:53:20 +0300 Subject: [PATCH 23/36] coding: utf-8 --- tests/helpers/run_conditions.py | 1 + tests/test_local.py | 1 + tests/test_remote.py | 1 + tests/test_simple.py | 1 + tests/test_simple_remote.py | 1 + 5 files changed, 5 insertions(+) diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py index 8d57f753..11357c30 100644 --- a/tests/helpers/run_conditions.py +++ b/tests/helpers/run_conditions.py @@ -1,3 +1,4 @@ +# coding: utf-8 import pytest import platform diff --git a/tests/test_local.py b/tests/test_local.py index 346336bd..60a96c18 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import pytest diff --git a/tests/test_remote.py b/tests/test_remote.py index 21bc9e8f..8b167e9f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import pytest diff --git a/tests/test_simple.py b/tests/test_simple.py index 15a26e1b..b9da8ad5 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import re import subprocess diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 6d681190..99b024f4 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import re import subprocess From 60bb4c52355db7b62812e2b96a0a41d3eda1a67f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 20:01:47 +0300 Subject: [PATCH 24/36] Cleanup --- tests/test_simple.py | 7 ------- tests/test_simple_remote.py | 7 ------- 2 files changed, 14 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index b9da8ad5..4cb63e4f 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -144,8 +144,6 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - # @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ __class__.helper__skip_test_if_util_not_exist("pg_resetwal") @@ -456,7 +454,6 @@ def test_replicate(self): res = node.execute('select * from test') assert (res == []) - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") @@ -499,7 +496,6 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') assert (rm_carriage_returns(res) == b'1000000\n') - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -570,7 +566,6 @@ def test_logical_replication(self): res = node2.execute('select * from test2') assert (res == [('a', ), ('b', )]) - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -595,7 +590,6 @@ def test_logical_catchup(self): assert (res == [(i, i, )]) node1.execute('delete from test') - # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): __class__.helper__skip_test_if_pg_version_is_ge("10") @@ -774,7 +768,6 @@ def test_logging(self): master.restart() assert (master._logger.is_alive()) - # @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 99b024f4..27852428 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -217,8 +217,6 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - # @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ __class__.helper__skip_test_if_util_not_exist("pg_resetwal") @@ -522,7 +520,6 @@ def test_replicate(self): res = node.execute('select * from test') assert (res == []) - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") @@ -565,7 +562,6 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') assert (res == b'1000000\n') - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -636,7 +632,6 @@ def test_logical_replication(self): res = node2.execute('select * from test2') assert (res == [('a',), ('b',)]) - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -661,7 +656,6 @@ def test_logical_catchup(self): assert (res == [(i, i, )]) node1.execute('delete from test') - # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): __class__.helper__skip_test_if_pg_version_is_ge("10") @@ -840,7 +834,6 @@ def test_logging(self): master.restart() assert (master._logger.is_alive()) - # @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") From 2d2532c77e8d7521552c0f3511c119e90d55573e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 20:04:22 +0300 Subject: [PATCH 25/36] CI runs all the tests of testgres. --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 6995b7e4..010bfe91 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -38,19 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" +time coverage run -a -m pytest -l -v -n 4 # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 # show coverage From 537a9acb9dfb26d82251d2d68796a55989be8317 Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 24 Feb 2025 09:07:50 +0100 Subject: [PATCH 26/36] Add install ssh (cherry picked from commit fec1e7ac9d6e2bfb43a01f0e370336ba5ed8e971) --- Dockerfile.tmpl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index dc5878b6..6749483c 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -9,6 +9,15 @@ RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ apk add --no-cache curl python3 python3-dev build-base musl-dev \ linux-headers py-virtualenv; \ fi + +# Install OpenSSH +RUN apk add --no-cache openssh + +# Configure SSH +RUN echo "postgres:postgres" | chpasswd && \ + mkdir -p /var/run/sshd && \ + sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config + ENV LANG=C.UTF-8 RUN mkdir -p /pg @@ -19,5 +28,8 @@ ADD . /pg/testgres WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg +# Expose SSH port +EXPOSE 22 + USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh +ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh & /usr/sbin/sshd -D From 1f998133a08e149f8b678e867e3615e5cebc8a2d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 09:46:03 +0300 Subject: [PATCH 27/36] Revert "Add install ssh" This reverts commit 537a9acb9dfb26d82251d2d68796a55989be8317. --- Dockerfile.tmpl | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 6749483c..dc5878b6 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -9,15 +9,6 @@ RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ apk add --no-cache curl python3 python3-dev build-base musl-dev \ linux-headers py-virtualenv; \ fi - -# Install OpenSSH -RUN apk add --no-cache openssh - -# Configure SSH -RUN echo "postgres:postgres" | chpasswd && \ - mkdir -p /var/run/sshd && \ - sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config - ENV LANG=C.UTF-8 RUN mkdir -p /pg @@ -28,8 +19,5 @@ ADD . /pg/testgres WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg -# Expose SSH port -EXPOSE 22 - USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh & /usr/sbin/sshd -D +ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh From 41455d676af49431dfb4d04066294889bff49540 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 09:46:03 +0300 Subject: [PATCH 28/36] Revert "CI runs all the tests of testgres." This reverts commit 2d2532c77e8d7521552c0f3511c119e90d55573e. --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 010bfe91..6995b7e4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -38,19 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 4 +time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - coverage run -a -m pytest -l -v -n 4 + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - coverage run -a -m pytest -l -v -n 4 + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # show coverage From 47a51a82b2572e29a4794f392ff482c6464ee692 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 12:32:35 +0300 Subject: [PATCH 29/36] Test of probackup plugin is restored It works now (was runned with a fresh probackup2 and vanilla 18devel). --- testgres/plugins/__init__.py | 8 ++++---- .../pg_probackup2/pg_probackup2/tests/__init__.py | 0 .../pg_probackup2/tests/basic_test.py | 14 ++++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py index 8c19a23b..824eadc6 100644 --- a/testgres/plugins/__init__.py +++ b/testgres/plugins/__init__.py @@ -1,7 +1,7 @@ -from pg_probackup2.gdb import GDBobj -from pg_probackup2.app import ProbackupApp, ProbackupException -from pg_probackup2.init_helpers import init_params -from pg_probackup2.storage.fs_backup import FSTestBackupDir +from .pg_probackup2.pg_probackup2.gdb import GDBobj +from .pg_probackup2.pg_probackup2.app import ProbackupApp, ProbackupException +from .pg_probackup2.pg_probackup2.init_helpers import init_params +from .pg_probackup2.pg_probackup2.storage.fs_backup import FSTestBackupDir __all__ = [ "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py index b63531ec..ececd6e7 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -2,10 +2,10 @@ import os import shutil import unittest -import testgres -from pg_probackup2.app import ProbackupApp -from pg_probackup2.init_helpers import Init, init_params -from pg_probackup2.app import build_backup_dir +from ...... import testgres +from ...pg_probackup2.app import ProbackupApp +from ...pg_probackup2.init_helpers import Init, init_params +from ..storage.fs_backup import FSTestBackupDir class TestUtils: @@ -37,11 +37,11 @@ def setup_test_environment(self): def setup_test_paths(self): self.rel_path = os.path.join(self.module_name, self.fname) self.test_path = os.path.join(init_params.tmp_path, self.rel_path) - os.makedirs(self.test_path) + os.makedirs(self.test_path, exist_ok=True) self.pb_log_path = os.path.join(self.test_path, "pb_log") def setup_backup_dir(self): - self.backup_dir = build_backup_dir(self, 'backup') + self.backup_dir = self.build_backup_dir('backup') self.backup_dir.cleanup() def setup_probackup(self): @@ -53,6 +53,8 @@ def tearDown(self): if os.path.exists(self.test_path): shutil.rmtree(self.test_path) + def build_backup_dir(self, backup='backup'): + return FSTestBackupDir(rel_path=self.rel_path, backup=backup) class BasicTest(ProbackupTest): def test_full_backup(self): From f7f163d2186ff314bd54f9ab67fa68ea0bab2721 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 13:34:54 +0300 Subject: [PATCH 30/36] The test suite of a probackup plugin is based on pytest --- .../tests/{basic_test.py => test_basic.py} | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) rename testgres/plugins/pg_probackup2/pg_probackup2/tests/{basic_test.py => test_basic.py} (57%) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py similarity index 57% rename from testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py rename to testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index ececd6e7..32b29498 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -1,62 +1,66 @@ -import logging +from __future__ import annotations + import os import shutil -import unittest +import pytest + from ...... import testgres from ...pg_probackup2.app import ProbackupApp from ...pg_probackup2.init_helpers import Init, init_params from ..storage.fs_backup import FSTestBackupDir -class TestUtils: - @staticmethod - def get_module_and_function_name(test_id): - try: - module_name = test_id.split('.')[-2] - fname = test_id.split('.')[-1] - except IndexError: - logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") - module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] - return module_name, fname +class ProbackupTest: + pg_node: testgres.PostgresNode + + @pytest.fixture(autouse=True, scope="function") + def implicit_fixture(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + self.helper__setUp(request) + yield + self.helper__tearDown() + + def helper__setUp(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + self.helper__setup_test_environment(request) + self.helper__setup_test_paths() + self.helper__setup_backup_dir() + self.helper__setup_probackup() -class ProbackupTest(unittest.TestCase): - def setUp(self): - self.setup_test_environment() - self.setup_test_paths() - self.setup_backup_dir() - self.setup_probackup() + def helper__setup_test_environment(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) - def setup_test_environment(self): self.output = None self.cmd = None self.nodes_to_cleanup = [] - self.module_name, self.fname = TestUtils.get_module_and_function_name(self.id()) + self.module_name, self.fname = request.node.cls.__name__, request.node.name self.test_env = Init().test_env() - def setup_test_paths(self): + def helper__setup_test_paths(self): self.rel_path = os.path.join(self.module_name, self.fname) self.test_path = os.path.join(init_params.tmp_path, self.rel_path) os.makedirs(self.test_path, exist_ok=True) self.pb_log_path = os.path.join(self.test_path, "pb_log") - def setup_backup_dir(self): - self.backup_dir = self.build_backup_dir('backup') + def helper__setup_backup_dir(self): + self.backup_dir = self.helper__build_backup_dir('backup') self.backup_dir.cleanup() - def setup_probackup(self): + def helper__setup_probackup(self): self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, auto_compress_alg='zlib', backup_dir=self.backup_dir) - def tearDown(self): + def helper__tearDown(self): if os.path.exists(self.test_path): shutil.rmtree(self.test_path) - def build_backup_dir(self, backup='backup'): + def helper__build_backup_dir(self, backup='backup'): return FSTestBackupDir(rel_path=self.rel_path, backup=backup) -class BasicTest(ProbackupTest): + +class TestBasic(ProbackupTest): def test_full_backup(self): # Setting up a simple test node node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) @@ -75,8 +79,4 @@ def test_full_backup(self): out = self.pb.validate('node', backup_id) # Check if the backup is valid - self.assertIn(f"INFO: Backup {backup_id} is valid", out) - - -if __name__ == "__main__": - unittest.main() + assert f"INFO: Backup {backup_id} is valid" in out From ed7b1874cf9f5f654eb859fa07cad0fa11cff178 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 14:20:45 +0300 Subject: [PATCH 31/36] Probackup plugin is updated Probackup plugin tests - They are skipped if PGPROBACKUPBIN is not defined Global variable init_params is None when PGPROBACKUPBIN is not defined or version is not processed --- .../pg_probackup2/init_helpers.py | 146 ++++++++++-------- .../pg_probackup2/tests/test_basic.py | 13 ++ 2 files changed, 91 insertions(+), 68 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 078fdbab..08f3a067 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from functools import reduce import getpass @@ -37,21 +39,27 @@ class Init(object): def __init__(self): + pass + + @staticmethod + def create() -> Init: + SELF = __class__() + if '-v' in sys.argv or '--verbose' in sys.argv: - self.verbose = True + SELF.verbose = True else: - self.verbose = False - - self._pg_config = testgres.get_pg_config() - self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' - self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' - self.is_pgpro = 'PGPRO_EDITION' in self._pg_config - self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] - self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] - version = self._pg_config['VERSION'].rstrip('develalphabetapre') + SELF.verbose = False + + SELF._pg_config = testgres.get_pg_config() + SELF.is_enterprise = SELF._pg_config.get('PGPRO_EDITION', None) == 'enterprise' + SELF.is_shardman = SELF._pg_config.get('PGPRO_EDITION', None) == 'shardman' + SELF.is_pgpro = 'PGPRO_EDITION' in SELF._pg_config + SELF.is_nls_enabled = 'enable-nls' in SELF._pg_config['CONFIGURE'] + SELF.is_lz4_enabled = '-llz4' in SELF._pg_config['LIBS'] + version = SELF._pg_config['VERSION'].rstrip('develalphabetapre') parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] parts[0] = re.match(r'\d+', parts[0]).group() - self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + SELF.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale test_env = os.environ.copy() @@ -75,31 +83,31 @@ def __init__(self): test_env['LC_MESSAGES'] = 'C' test_env['LC_TIME'] = 'C' - self._test_env = test_env + SELF._test_env = test_env # Get the directory from which the script was executed - self.source_path = os.getcwd() + SELF.source_path = os.getcwd() tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') if tmp_path and os.path.isabs(tmp_path): - self.tmp_path = tmp_path + SELF.tmp_path = tmp_path else: - self.tmp_path = os.path.abspath( - os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) + SELF.tmp_path = os.path.abspath( + os.path.join(SELF.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) ) - os.makedirs(self.tmp_path, exist_ok=True) + os.makedirs(SELF.tmp_path, exist_ok=True) - self.username = getpass.getuser() + SELF.username = getpass.getuser() - self.probackup_path = None + SELF.probackup_path = None if 'PGPROBACKUPBIN' in test_env: if shutil.which(test_env["PGPROBACKUPBIN"]): - self.probackup_path = test_env["PGPROBACKUPBIN"] + SELF.probackup_path = test_env["PGPROBACKUPBIN"] else: - if self.verbose: + if SELF.verbose: print('PGPROBACKUPBIN is not an executable file') - if not self.probackup_path: + if not SELF.probackup_path: probackup_path_tmp = os.path.join( testgres.get_pg_config()['BINDIR'], 'pg_probackup') @@ -108,116 +116,118 @@ def __init__(self): logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - self.probackup_path = probackup_path_tmp + SELF.probackup_path = probackup_path_tmp - if not self.probackup_path: - probackup_path_tmp = self.source_path + if not SELF.probackup_path: + probackup_path_tmp = SELF.source_path if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - self.probackup_path = probackup_path_tmp + SELF.probackup_path = probackup_path_tmp - if not self.probackup_path: + if not SELF.probackup_path: logging.error('pg_probackup binary is not found') - exit(1) + return None if os.name == 'posix': - self.EXTERNAL_DIRECTORY_DELIMITER = ':' + SELF.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ':' + os.environ['PATH'] + SELF.probackup_path) + ':' + os.environ['PATH'] elif os.name == 'nt': - self.EXTERNAL_DIRECTORY_DELIMITER = ';' + SELF.EXTERNAL_DIRECTORY_DELIMITER = ';' os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ';' + os.environ['PATH'] + SELF.probackup_path) + ';' + os.environ['PATH'] - self.probackup_old_path = None + SELF.probackup_old_path = None if 'PGPROBACKUPBIN_OLD' in test_env: if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): - self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] + SELF.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] else: - if self.verbose: + if SELF.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') - self.probackup_version = None - self.old_probackup_version = None + SELF.probackup_version = None + SELF.old_probackup_version = None probackup_version_output = subprocess.check_output( - [self.probackup_path, "--version"], + [SELF.probackup_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", probackup_version_output) - self.probackup_version = match.group(0) if match else None + SELF.probackup_version = match.group(0) if match else None match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) compressions = match.group(1) if match else None if compressions: - self.probackup_compressions = {s.strip() for s in compressions.split(',')} + SELF.probackup_compressions = {s.strip() for s in compressions.split(',')} else: - self.probackup_compressions = [] + SELF.probackup_compressions = [] - if self.probackup_old_path: + if SELF.probackup_old_path: old_probackup_version_output = subprocess.check_output( - [self.probackup_old_path, "--version"], + [SELF.probackup_old_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", old_probackup_version_output) - self.old_probackup_version = match.group(0) if match else None + SELF.old_probackup_version = match.group(0) if match else None - self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' - self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 - self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' + SELF.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' + SELF.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and SELF.pg_config_version >= 110000 + SELF.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' - self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() - if self.bckp_source not in ('base', 'direct', 'pro'): + SELF.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() + if SELF.bckp_source not in ('base', 'direct', 'pro'): raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") - self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' + SELF.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) if env_compress: env_compress = env_compress.lower() if env_compress in ('on', 'zlib'): - self.compress_suffix = '.gz' - self.archive_compress = 'zlib' + SELF.compress_suffix = '.gz' + SELF.archive_compress = 'zlib' elif env_compress == 'lz4': if not HAVE_LZ4: raise LZ4_error - if 'lz4' not in self.probackup_compressions: + if 'lz4' not in SELF.probackup_compressions: raise Exception("pg_probackup is not compiled with lz4 support") - self.compress_suffix = '.lz4' - self.archive_compress = 'lz4' + SELF.compress_suffix = '.lz4' + SELF.archive_compress = 'lz4' elif env_compress == 'zstd': if not HAVE_ZSTD: raise ZSTD_error - if 'zstd' not in self.probackup_compressions: + if 'zstd' not in SELF.probackup_compressions: raise Exception("pg_probackup is not compiled with zstd support") - self.compress_suffix = '.zst' - self.archive_compress = 'zstd' + SELF.compress_suffix = '.zst' + SELF.archive_compress = 'zstd' else: - self.compress_suffix = '' - self.archive_compress = False + SELF.compress_suffix = '' + SELF.archive_compress = False cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) if cfs_compress: - self.cfs_compress = cfs_compress.lower() + SELF.cfs_compress = cfs_compress.lower() else: - self.cfs_compress = self.archive_compress + SELF.cfs_compress = SELF.archive_compress os.environ["PGAPPNAME"] = "pg_probackup" - self.delete_logs = delete_logs + SELF.delete_logs = delete_logs - if self.probackup_version.split('.')[0].isdigit(): - self.major_version = int(self.probackup_version.split('.')[0]) + if SELF.probackup_version.split('.')[0].isdigit(): + SELF.major_version = int(SELF.probackup_version.split('.')[0]) else: - logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) - sys.exit(1) + logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(SELF.probackup_version)) + return None + + return SELF def test_env(self): return self._test_env.copy() -init_params = Init() +init_params = Init.create() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index 32b29498..ba788623 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -13,6 +13,18 @@ class ProbackupTest: pg_node: testgres.PostgresNode + @staticmethod + def probackup_is_available() -> bool: + p = os.environ.get("PGPROBACKUPBIN") + + if p is None: + return False + + if not os.path.exists(p): + return False + + return True + @pytest.fixture(autouse=True, scope="function") def implicit_fixture(self, request: pytest.FixtureRequest): assert isinstance(request, pytest.FixtureRequest) @@ -60,6 +72,7 @@ def helper__build_backup_dir(self, backup='backup'): return FSTestBackupDir(rel_path=self.rel_path, backup=backup) +@pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") class TestBasic(ProbackupTest): def test_full_backup(self): # Setting up a simple test node From 8af4a49d3eb2eb94e776db4271619263dce5e87b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 14:24:34 +0300 Subject: [PATCH 32/36] CI test use 4 cores --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 6995b7e4..e9d58b54 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -38,19 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" +time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # show coverage From d35ca43acdc07bad1540002a50b263283947af86 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 15:09:54 +0300 Subject: [PATCH 33/36] testgres.plugins.probackup2.Init was restored [thanks to Yuri Sokolov] --- .../pg_probackup2/init_helpers.py | 151 +++++++++--------- 1 file changed, 72 insertions(+), 79 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 08f3a067..c4570a39 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import logging from functools import reduce import getpass @@ -39,27 +37,21 @@ class Init(object): def __init__(self): - pass - - @staticmethod - def create() -> Init: - SELF = __class__() - if '-v' in sys.argv or '--verbose' in sys.argv: - SELF.verbose = True + self.verbose = True else: - SELF.verbose = False - - SELF._pg_config = testgres.get_pg_config() - SELF.is_enterprise = SELF._pg_config.get('PGPRO_EDITION', None) == 'enterprise' - SELF.is_shardman = SELF._pg_config.get('PGPRO_EDITION', None) == 'shardman' - SELF.is_pgpro = 'PGPRO_EDITION' in SELF._pg_config - SELF.is_nls_enabled = 'enable-nls' in SELF._pg_config['CONFIGURE'] - SELF.is_lz4_enabled = '-llz4' in SELF._pg_config['LIBS'] - version = SELF._pg_config['VERSION'].rstrip('develalphabetapre') + self.verbose = False + + self._pg_config = testgres.get_pg_config() + self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' + self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' + self.is_pgpro = 'PGPRO_EDITION' in self._pg_config + self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] + self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] + version = self._pg_config['VERSION'].rstrip('develalphabetapre') parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] parts[0] = re.match(r'\d+', parts[0]).group() - SELF.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale test_env = os.environ.copy() @@ -83,31 +75,31 @@ def create() -> Init: test_env['LC_MESSAGES'] = 'C' test_env['LC_TIME'] = 'C' - SELF._test_env = test_env + self._test_env = test_env # Get the directory from which the script was executed - SELF.source_path = os.getcwd() + self.source_path = os.getcwd() tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') if tmp_path and os.path.isabs(tmp_path): - SELF.tmp_path = tmp_path + self.tmp_path = tmp_path else: - SELF.tmp_path = os.path.abspath( - os.path.join(SELF.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) + self.tmp_path = os.path.abspath( + os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) ) - os.makedirs(SELF.tmp_path, exist_ok=True) + os.makedirs(self.tmp_path, exist_ok=True) - SELF.username = getpass.getuser() + self.username = getpass.getuser() - SELF.probackup_path = None + self.probackup_path = None if 'PGPROBACKUPBIN' in test_env: if shutil.which(test_env["PGPROBACKUPBIN"]): - SELF.probackup_path = test_env["PGPROBACKUPBIN"] + self.probackup_path = test_env["PGPROBACKUPBIN"] else: - if SELF.verbose: + if self.verbose: print('PGPROBACKUPBIN is not an executable file') - if not SELF.probackup_path: + if not self.probackup_path: probackup_path_tmp = os.path.join( testgres.get_pg_config()['BINDIR'], 'pg_probackup') @@ -116,118 +108,119 @@ def create() -> Init: logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - SELF.probackup_path = probackup_path_tmp + self.probackup_path = probackup_path_tmp - if not SELF.probackup_path: - probackup_path_tmp = SELF.source_path + if not self.probackup_path: + probackup_path_tmp = self.source_path if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - SELF.probackup_path = probackup_path_tmp + self.probackup_path = probackup_path_tmp - if not SELF.probackup_path: - logging.error('pg_probackup binary is not found') - return None + if not self.probackup_path: + raise Exception('pg_probackup binary is not found') if os.name == 'posix': - SELF.EXTERNAL_DIRECTORY_DELIMITER = ':' + self.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( - SELF.probackup_path) + ':' + os.environ['PATH'] + self.probackup_path) + ':' + os.environ['PATH'] elif os.name == 'nt': - SELF.EXTERNAL_DIRECTORY_DELIMITER = ';' + self.EXTERNAL_DIRECTORY_DELIMITER = ';' os.environ['PATH'] = os.path.dirname( - SELF.probackup_path) + ';' + os.environ['PATH'] + self.probackup_path) + ';' + os.environ['PATH'] - SELF.probackup_old_path = None + self.probackup_old_path = None if 'PGPROBACKUPBIN_OLD' in test_env: if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): - SELF.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] + self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] else: - if SELF.verbose: + if self.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') - SELF.probackup_version = None - SELF.old_probackup_version = None + self.probackup_version = None + self.old_probackup_version = None probackup_version_output = subprocess.check_output( - [SELF.probackup_path, "--version"], + [self.probackup_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", probackup_version_output) - SELF.probackup_version = match.group(0) if match else None + self.probackup_version = match.group(0) if match else None match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) compressions = match.group(1) if match else None if compressions: - SELF.probackup_compressions = {s.strip() for s in compressions.split(',')} + self.probackup_compressions = {s.strip() for s in compressions.split(',')} else: - SELF.probackup_compressions = [] + self.probackup_compressions = [] - if SELF.probackup_old_path: + if self.probackup_old_path: old_probackup_version_output = subprocess.check_output( - [SELF.probackup_old_path, "--version"], + [self.probackup_old_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", old_probackup_version_output) - SELF.old_probackup_version = match.group(0) if match else None + self.old_probackup_version = match.group(0) if match else None - SELF.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' - SELF.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and SELF.pg_config_version >= 110000 - SELF.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' + self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' + self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' - SELF.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() - if SELF.bckp_source not in ('base', 'direct', 'pro'): + self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() + if self.bckp_source not in ('base', 'direct', 'pro'): raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") - SELF.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' + self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) if env_compress: env_compress = env_compress.lower() if env_compress in ('on', 'zlib'): - SELF.compress_suffix = '.gz' - SELF.archive_compress = 'zlib' + self.compress_suffix = '.gz' + self.archive_compress = 'zlib' elif env_compress == 'lz4': if not HAVE_LZ4: raise LZ4_error - if 'lz4' not in SELF.probackup_compressions: + if 'lz4' not in self.probackup_compressions: raise Exception("pg_probackup is not compiled with lz4 support") - SELF.compress_suffix = '.lz4' - SELF.archive_compress = 'lz4' + self.compress_suffix = '.lz4' + self.archive_compress = 'lz4' elif env_compress == 'zstd': if not HAVE_ZSTD: raise ZSTD_error - if 'zstd' not in SELF.probackup_compressions: + if 'zstd' not in self.probackup_compressions: raise Exception("pg_probackup is not compiled with zstd support") - SELF.compress_suffix = '.zst' - SELF.archive_compress = 'zstd' + self.compress_suffix = '.zst' + self.archive_compress = 'zstd' else: - SELF.compress_suffix = '' - SELF.archive_compress = False + self.compress_suffix = '' + self.archive_compress = False cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) if cfs_compress: - SELF.cfs_compress = cfs_compress.lower() + self.cfs_compress = cfs_compress.lower() else: - SELF.cfs_compress = SELF.archive_compress + self.cfs_compress = self.archive_compress os.environ["PGAPPNAME"] = "pg_probackup" - SELF.delete_logs = delete_logs + self.delete_logs = delete_logs - if SELF.probackup_version.split('.')[0].isdigit(): - SELF.major_version = int(SELF.probackup_version.split('.')[0]) + if self.probackup_version.split('.')[0].isdigit(): + self.major_version = int(self.probackup_version.split('.')[0]) else: - logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(SELF.probackup_version)) - return None - - return SELF + raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) def test_env(self): return self._test_env.copy() -init_params = Init.create() +try: + init_params = Init() +except Exception as e: + logging.error(str(e)) + logging.warning("testgres.plugins.probackup2.init_params is set to None.") + init_params = None From 181126bbbddb108f0c17a201f9c78e070c26f8f9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 16:09:36 +0300 Subject: [PATCH 34/36] pytest.ini is updated [testpaths] Enumeration of all the known folders with tests. --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 61d4f9bc..c94eabc2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -testpaths = ./tests +testpaths = ["./tests", "./testgres/plugins/pg_probackup2/pg_probackup2/tests"] addopts = --strict-markers markers = #log_file = logs/pytest.log From d08f659dffd4af3dc412868f50fe6b75b1dd1ff9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 17:17:42 +0300 Subject: [PATCH 35/36] test_child_pids (local, remote) is updated Multiple attempts and logging are added. --- tests/test_simple.py | 70 +++++++++++++++++++++++++++++++++---- tests/test_simple_remote.py | 70 +++++++++++++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 4cb63e4f..c470b33d 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1000,6 +1000,62 @@ def test_child_pids(self): ProcessType.WalReceiver, ] + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + with get_new_node().init().start() as master: # master node doesn't have a source walsender! @@ -1014,13 +1070,15 @@ def test_child_pids(self): # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - assert (ptype in master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - assert (ptype in replica_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 27852428..67c98bf8 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1038,6 +1038,62 @@ def test_child_pids(self): ProcessType.WalReceiver, ] + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + with __class__.helper__get_node().init().start() as master: # master node doesn't have a source walsender! @@ -1052,13 +1108,15 @@ def test_child_pids(self): # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - assert (ptype in master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - assert (ptype in replica_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) From e80314648d001356d40520bf3bd3345a5c9868d4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 18:02:49 +0300 Subject: [PATCH 36/36] test_child_process_dies is updated Multiple attempts are added. --- tests/test_simple.py | 35 +++++++++++++++++++++++++--------- tests/test_simple_remote.py | 38 +++++++++++++++++++++++++++---------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index c470b33d..6c433cd4 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1096,15 +1096,32 @@ def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] - with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - assert (process.poll() is None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 67c98bf8..e7cc5e5c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1130,17 +1130,35 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( with pytest.raises(expected_exception=TestgresException): replica.source_walsender + # TODO: Why does not this test work with remote host? def test_child_process_dies(self): - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - assert (process.poll() is None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + # test for FileNotFound exception during child_processes() function + with subprocess.Popen(["sleep", "60"]) as process: + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break @staticmethod def helper__get_node():