diff --git a/testgres/__init__.py b/testgres/__init__.py index 8d0e38c6..665548d6 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -23,7 +23,8 @@ CatchUpException, \ StartNodeException, \ InitNodeException, \ - BackupException + BackupException, \ + InvalidOperationException from .enums import \ XLogMethod, \ @@ -60,7 +61,7 @@ "NodeBackup", "testgres_config", "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", - "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", + "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", diff --git a/testgres/exceptions.py b/testgres/exceptions.py index ee329031..d61d4691 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -9,13 +9,14 @@ class TestgresException(Exception): @six.python_2_unicode_compatible class ExecUtilException(TestgresException): - def __init__(self, message=None, command=None, exit_code=0, out=None): + def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): super(ExecUtilException, self).__init__(message) self.message = message self.command = command self.exit_code = exit_code self.out = out + self.error = error def __str__(self): msg = [] @@ -24,13 +25,17 @@ def __str__(self): msg.append(self.message) if self.command: - msg.append(u'Command: {}'.format(self.command)) + command_s = ' '.join(self.command) if isinstance(self.command, list) else self.command, + msg.append(u'Command: {}'.format(command_s)) if self.exit_code: msg.append(u'Exit code: {}'.format(self.exit_code)) + if self.error: + msg.append(u'---- Error:\n{}'.format(self.error)) + if self.out: - msg.append(u'----\n{}'.format(self.out)) + msg.append(u'---- Out:\n{}'.format(self.out)) return self.convert_and_join(msg) @@ -98,3 +103,7 @@ class InitNodeException(TestgresException): class BackupException(TestgresException): pass + + +class InvalidOperationException(TestgresException): + pass diff --git a/testgres/node.py b/testgres/node.py index 48a100a9..b5cbab27 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -74,7 +74,8 @@ TimeoutException, \ InitNodeException, \ TestgresException, \ - BackupException + BackupException, \ + InvalidOperationException from .logger import TestgresLogger @@ -987,6 +988,37 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ + return self._psql( + ignore_errors=True, + query=query, + filename=filename, + dbname=dbname, + username=username, + input=input, + **variables + ) + + def _psql( + self, + ignore_errors, + query=None, + filename=None, + dbname=None, + username=None, + input=None, + **variables): + assert type(variables) == dict # noqa: E721 + + # + # We do not support encoding. It may be added later. Ok? + # + if input is None: + pass + elif type(input) == bytes: # noqa: E721 + pass + else: + raise Exception("Input data must be None or bytes.") + dbname = dbname or default_dbname() psql_params = [ @@ -1017,20 +1049,14 @@ def psql(self, # should be the last one psql_params.append(dbname) - if not self.os_ops.remote: - # start psql process - process = subprocess.Popen(psql_params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # wait until it finishes and get stdout and stderr - out, err = process.communicate(input=input) - return process.returncode, out, err - else: - status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input) - return status_code, out, err + return self.os_ops.exec_command( + psql_params, + verbose=True, + input=input, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ignore_errors=ignore_errors) @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query=None, expect_error=False, **kwargs): @@ -1051,22 +1077,27 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): Returns: psql's output as str. """ + assert type(kwargs) == dict # noqa: E721 + assert not ("ignore_errors" in kwargs.keys()) + assert not ("expect_error" in kwargs.keys()) # force this setting kwargs['ON_ERROR_STOP'] = 1 try: - ret, out, err = self.psql(query=query, **kwargs) + ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) except ExecUtilException as e: - ret = e.exit_code - out = e.out - err = e.message - if ret: - if expect_error: - out = (err or b'').decode('utf-8') - else: - raise QueryException((err or b'').decode('utf-8'), query) - elif expect_error: - assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) + if not expect_error: + raise QueryException(e.message, query) + + if type(e.error) == bytes: # noqa: E721 + return e.error.decode("utf-8") # throw + + # [2024-12-09] This situation is not expected + assert False + return e.error + + if expect_error: + raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query)) return out diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py new file mode 100644 index 00000000..b50f0baa --- /dev/null +++ b/testgres/operations/helpers.py @@ -0,0 +1,52 @@ +import locale + + +class Helpers: + def _make_get_default_encoding_func(): + # locale.getencoding is added in Python 3.11 + if hasattr(locale, 'getencoding'): + return locale.getencoding + + # It must exist + return locale.getpreferredencoding + + # Prepared pointer on function to get a name of system codepage + _get_default_encoding_func = _make_get_default_encoding_func() + + def GetDefaultEncoding(): + # + # Original idea/source was: + # + # def os_ops.get_default_encoding(): + # if not hasattr(locale, 'getencoding'): + # locale.getencoding = locale.getpreferredencoding + # return locale.getencoding() or 'UTF-8' + # + + assert __class__._get_default_encoding_func is not None + + r = __class__._get_default_encoding_func() + + if r: + assert r is not None + assert type(r) == str # noqa: E721 + assert r != "" + return r + + # Is it an unexpected situation? + return 'UTF-8' + + def PrepareProcessInput(input, encoding): + if not input: + return None + + if type(input) == str: # noqa: E721 + if encoding is None: + return input.encode(__class__.GetDefaultEncoding()) + + assert type(encoding) == str # noqa: E721 + return input.encode(encoding) + + # It is expected! + assert type(input) == bytes # noqa: E721 + return input diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 5b7972ae..3e8ab8ca 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -11,6 +11,8 @@ from ..exceptions import ExecUtilException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from .raise_error import RaiseError +from .helpers import Helpers try: from shutil import which as find_executable @@ -47,14 +49,6 @@ def __init__(self, conn_params=None): self.remote = False self.username = conn_params.username or getpass.getuser() - @staticmethod - def _raise_exec_exception(message, command, exit_code, output): - """Raise an ExecUtilException.""" - raise ExecUtilException(message=message.format(output), - command=' '.join(command) if isinstance(command, list) else command, - exit_code=exit_code, - out=output) - @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" @@ -65,6 +59,8 @@ def _process_output(encoding, temp_file_path): return output, None # In Windows stderr writing in stdout def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + # TODO: why don't we use the data from input? + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: stdout = temp_file stderr = subprocess.STDOUT @@ -86,6 +82,12 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process return process, output, error def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + process = subprocess.Popen( cmd, shell=shell, @@ -93,18 +95,23 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.PIPE, ) + assert not (process is None) if get_process: return process, None, None try: - output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) - if encoding: - output = output.decode(encoding) - error = error.decode(encoding) - return process, output, error + output, error = process.communicate(input=input_prepared, timeout=timeout) except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + assert type(output) == bytes # noqa: E721 + assert type(error) == bytes # noqa: E721 + + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): """Execute a command and return the process and its output.""" if os.name == 'nt' and stdout is None: # Windows @@ -120,11 +127,20 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ Execute a command in a subprocess and handle the output based on the provided parameters. """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): - self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=process.returncode, + msg_arg=error or output, + error=error, + out=output + ) if verbose: return process.returncode, output, error diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py new file mode 100644 index 00000000..0e760e74 --- /dev/null +++ b/testgres/operations/raise_error.py @@ -0,0 +1,46 @@ +from ..exceptions import ExecUtilException +from .helpers import Helpers + + +class RaiseError: + def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): + assert type(exit_code) == int # noqa: E721 + + msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip() + assert type(msg_arg_s) == str # noqa: E721 + + if msg_arg_s == "": + msg_arg_s = "#no_error_message" + + message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`" + raise ExecUtilException( + message=message, + command=cmd, + exit_code=exit_code, + out=out, + error=error) + + def _TranslateDataIntoString(data): + if type(data) == bytes: # noqa: E721 + return __class__._TranslateDataIntoString__FromBinary(data) + + return str(data) + + def _TranslateDataIntoString__FromBinary(data): + assert type(data) == bytes # noqa: E721 + + try: + return data.decode(Helpers.GetDefaultEncoding()) + except UnicodeDecodeError: + pass + + return "#cannot_decode_text" + + def _BinaryIsASCII(data): + assert type(data) == bytes # noqa: E721 + + for b in data: + if not (b >= 0 and b <= 127): + return False + + return True diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 88394eb7..00c50d93 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -15,6 +15,8 @@ from ..exceptions import ExecUtilException from .os_ops import OsOperations, ConnectionParams, get_default_encoding +from .raise_error import RaiseError +from .helpers import Helpers error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -66,17 +68,27 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, Args: - cmd (str): The command to be executed. """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert not (process is None) if get_process: return process try: - result, error = process.communicate(input, timeout=timeout) + result, error = process.communicate(input=input_prepared, timeout=timeout) except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) @@ -100,10 +112,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, error = error.decode(encoding) if not ignore_errors and error_found and not expect_error: - error = normalize_error(error) - assert type(error) == str # noqa: E721 - message = "Utility exited with non-zero code. Error: " + error - raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result) + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=exit_status, + msg_arg=error, + error=error, + out=result) if verbose: return exit_status, result, error diff --git a/tests/test_local.py b/tests/test_local.py index da26468b..cb96a3bc 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -37,7 +37,7 @@ def test_exec_command_failure(self): error = e.message break raise Exception("We wait an exception!") - assert error == "Utility exited with non-zero code. Error `b'/bin/sh: 1: nonexistent_command: not found\\n'`" + assert error == "Utility exited with non-zero code. Error: `/bin/sh: 1: nonexistent_command: not found`" def test_exec_command_failure__expect_error(self): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index 565163f7..c1a91bc6 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -37,7 +37,7 @@ def test_exec_command_failure(self): error = e.message break raise Exception("We wait an exception!") - assert error == b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n' + assert error == 'Utility exited with non-zero code. Error: `bash: line 1: nonexistent_command: command not found`' def test_exec_command_failure__expect_error(self): """ @@ -98,11 +98,14 @@ def test_makedirs_and_rmdirs_failure(self): self.operations.makedirs(path) # Test rmdirs - try: - exit_status, result, error = self.operations.rmdirs(path, verbose=True) - except ExecUtilException as e: - error = e.message - assert error == b"Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n" + while True: + try: + self.operations.rmdirs(path, verbose=True) + except ExecUtilException as e: + error = e.message + break + raise Exception("We wait an exception!") + assert error == "Utility exited with non-zero code. Error: `rm: cannot remove '/root/test_dir': Permission denied`" def test_listdir(self): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 51cdc896..fade468c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,9 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException, NodeApp + TestgresException, \ + InvalidOperationException, \ + NodeApp from testgres import \ TestgresConfig, \ @@ -310,6 +312,23 @@ def test_psql(self): with self.assertRaises(QueryException): node.safe_psql('select 1') + def test_safe_psql__expect_error(self): + with get_new_node().init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + self.assertTrue(type(err) == str) # noqa: E721 + self.assertIn('select_or_not_select', err) + self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + + # --------- + with self.assertRaises(InvalidOperationException) as ctx: + node.safe_psql("select 1;", expect_error=True) + + self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + self.assertEqual(rm_carriage_returns(res), b'1\n') + def test_transactions(self): with get_new_node().init().start() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 936c31f2..26ac7c61 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -23,7 +23,8 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException + TestgresException, \ + InvalidOperationException from testgres.config import \ TestgresConfig, \ @@ -295,6 +296,23 @@ def test_psql(self): with self.assertRaises(QueryException): node.safe_psql('select 1') + def test_safe_psql__expect_error(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + self.assertTrue(type(err) == str) # noqa: E721 + self.assertIn('select_or_not_select', err) + self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + + # --------- + with self.assertRaises(InvalidOperationException) as ctx: + node.safe_psql("select 1;", expect_error=True) + + self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + self.assertEqual(res, b'1\n') + def test_transactions(self): with get_remote_node(conn_params=conn_params).init().start() as node: with node.connect() as con: