Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Proposal to fix #154 (v2) #161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Dec 10, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions testgres/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
CatchUpException, \
StartNodeException, \
InitNodeException, \
BackupException
BackupException, \
InvalidOperationException

from .enums import \
XLogMethod, \
Expand Down Expand Up @@ -60,7 +61,7 @@
"NodeBackup", "testgres_config",
"TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config",
"NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError",
"TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException",
"TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
"XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
"PostgresNode", "NodeApp",
"reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version",
Expand Down
15 changes: 12 additions & 3 deletions testgres/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@ class TestgresException(Exception):

@six.python_2_unicode_compatible
class ExecUtilException(TestgresException):
def __init__(self, message=None, command=None, exit_code=0, out=None):
def __init__(self, message=None, command=None, exit_code=0, out=None, error=None):
super(ExecUtilException, self).__init__(message)

self.message = message
self.command = command
self.exit_code = exit_code
self.out = out
self.error = error

def __str__(self):
msg = []
Expand All @@ -24,13 +25,17 @@ def __str__(self):
msg.append(self.message)

if self.command:
msg.append(u'Command: {}'.format(self.command))
command_s = ' '.join(self.command) if isinstance(self.command, list) else self.command,
msg.append(u'Command: {}'.format(command_s))

if self.exit_code:
msg.append(u'Exit code: {}'.format(self.exit_code))

if self.error:
msg.append(u'---- Error:\n{}'.format(self.error))

if self.out:
msg.append(u'----\n{}'.format(self.out))
msg.append(u'---- Out:\n{}'.format(self.out))

return self.convert_and_join(msg)

Expand Down Expand Up @@ -98,3 +103,7 @@ class InitNodeException(TestgresException):

class BackupException(TestgresException):
pass


class InvalidOperationException(TestgresException):
pass
45 changes: 45 additions & 0 deletions testgres/helpers/raise_error.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
from ..exceptions import ExecUtilException


class RaiseError:
def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out):
assert type(exit_code) == int # noqa: E721

msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip()
assert type(msg_arg_s) == str # noqa: E721

if msg_arg_s == "":
msg_arg_s = "#no_error_message"

message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`"
raise ExecUtilException(
message=message,
command=cmd,
exit_code=exit_code,
out=out,
error=error)

def _TranslateDataIntoString(data):
if type(data) == bytes: # noqa: E721
return __class__._TranslateDataIntoString__FromBinary(data)

return str(data)

def _TranslateDataIntoString__FromBinary(data):
assert type(data) == bytes # noqa: E721

try:
return data.decode('utf-8')
except UnicodeDecodeError:
pass

return "#cannot_decode_text"

def _BinaryIsASCII(data):
assert type(data) == bytes # noqa: E721

for b in data:
if not (b >= 0 and b <= 127):
return False

return True
83 changes: 57 additions & 26 deletions testgres/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
import os
import random
import signal
import subprocess
import threading
import tempfile
import subprocess
from queue import Queue

import time
Expand Down Expand Up @@ -74,7 +74,8 @@
TimeoutException, \
InitNodeException, \
TestgresException, \
BackupException
BackupException, \
InvalidOperationException

from .logger import TestgresLogger

Expand Down Expand Up @@ -987,6 +988,37 @@ def psql(self,
>>> psql(query='select 3', ON_ERROR_STOP=1)
"""

return self._psql(
ignore_errors=True,
query=query,
filename=filename,
dbname=dbname,
username=username,
input=input,
**variables
)

def _psql(
self,
ignore_errors,
query=None,
filename=None,
dbname=None,
username=None,
input=None,
**variables):
assert type(variables) == dict # noqa: E721

#
# We do not support encoding. It may be added later. Ok?
#
if input is None:
pass
elif type(input) == bytes: # noqa: E721
pass
else:
raise Exception("Input data must be None or bytes.")

dbname = dbname or default_dbname()

psql_params = [
Expand Down Expand Up @@ -1017,20 +1049,14 @@ def psql(self,

# should be the last one
psql_params.append(dbname)
if not self.os_ops.remote:
# start psql process
process = subprocess.Popen(psql_params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)

# wait until it finishes and get stdout and stderr
out, err = process.communicate(input=input)
return process.returncode, out, err
else:
status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)

return status_code, out, err
return self.os_ops.exec_command(
psql_params,
verbose=True,
input=input,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
ignore_errors=ignore_errors)

@method_decorator(positional_args_hack(['dbname', 'query']))
def safe_psql(self, query=None, expect_error=False, **kwargs):
Expand All @@ -1051,22 +1077,27 @@ def safe_psql(self, query=None, expect_error=False, **kwargs):
Returns:
psql's output as str.
"""
assert type(kwargs) == dict # noqa: E721
assert not ("ignore_errors" in kwargs.keys())
assert not ("expect_error" in kwargs.keys())

# force this setting
kwargs['ON_ERROR_STOP'] = 1
try:
ret, out, err = self.psql(query=query, **kwargs)
ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs)
except ExecUtilException as e:
ret = e.exit_code
out = e.out
err = e.message
if ret:
if expect_error:
out = (err or b'').decode('utf-8')
else:
raise QueryException((err or b'').decode('utf-8'), query)
elif expect_error:
assert False, "Exception was expected, but query finished successfully: `{}` ".format(query)
if not expect_error:
raise QueryException(e.message, query)

if type(e.error) == bytes: # noqa: E721
return e.error.decode("utf-8") # throw

# [2024-12-09] This situation is not expected
assert False
return e.error

if expect_error:
raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query))

return out

Expand Down
15 changes: 15 additions & 0 deletions testgres/operations/helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
class Helpers:
def PrepareProcessInput(input, encoding):
if not input:
return None

if type(input) == str: # noqa: E721
if encoding is None:
return input.encode()

assert type(encoding) == str # noqa: E721
return input.encode(encoding)

# It is expected!
assert type(input) == bytes # noqa: E721
return input
29 changes: 19 additions & 10 deletions testgres/operations/local_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

from ..exceptions import ExecUtilException
from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding
from .helpers import Helpers
from ..helpers.raise_error import RaiseError

try:
from shutil import which as find_executable
Expand Down Expand Up @@ -47,14 +49,6 @@ def __init__(self, conn_params=None):
self.remote = False
self.username = conn_params.username or getpass.getuser()

@staticmethod
def _raise_exec_exception(message, command, exit_code, output):
"""Raise an ExecUtilException."""
raise ExecUtilException(message=message.format(output),
command=' '.join(command) if isinstance(command, list) else command,
exit_code=exit_code,
out=output)

@staticmethod
def _process_output(encoding, temp_file_path):
"""Process the output of a command from a temporary file."""
Expand All @@ -65,6 +59,8 @@ def _process_output(encoding, temp_file_path):
return output, None # In Windows stderr writing in stdout

def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding):
# TODO: why don't we use the data from input?

with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file:
stdout = temp_file
stderr = subprocess.STDOUT
Expand All @@ -86,6 +82,10 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process
return process, output, error

def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding):
input_prepared = None
if not get_process:
input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw

process = subprocess.Popen(
cmd,
shell=shell,
Expand All @@ -96,7 +96,7 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr
if get_process:
return process, None, None
try:
output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout)
output, error = process.communicate(input=input_prepared, timeout=timeout)
if encoding:
output = output.decode(encoding)
error = error.decode(encoding)
Expand All @@ -120,11 +120,20 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False,
"""
Execute a command in a subprocess and handle the output based on the provided parameters.
"""
assert type(expect_error) == bool # noqa: E721
assert type(ignore_errors) == bool # noqa: E721

process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding)
if get_process:
return process
if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error):
self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output)
RaiseError.UtilityExitedWithNonZeroCode(
cmd=cmd,
exit_code=process.returncode,
msg_arg=error or output,
error=error,
out=output
)

if verbose:
return process.returncode, output, error
Expand Down
21 changes: 16 additions & 5 deletions testgres/operations/remote_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
raise ImportError("You must have psycopg2 or pg8000 modules installed")

from ..exceptions import ExecUtilException
from ..helpers.raise_error import RaiseError
from .os_ops import OsOperations, ConnectionParams, get_default_encoding
from .helpers import Helpers

error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory']

Expand Down Expand Up @@ -66,6 +68,13 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False,
Args:
- cmd (str): The command to be executed.
"""
assert type(expect_error) == bool # noqa: E721
assert type(ignore_errors) == bool # noqa: E721

input_prepared = None
if not get_process:
input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw

ssh_cmd = []
if isinstance(cmd, str):
ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd]
Expand All @@ -76,7 +85,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False,
return process

try:
result, error = process.communicate(input, timeout=timeout)
result, error = process.communicate(input=input_prepared, timeout=timeout)
except subprocess.TimeoutExpired:
process.kill()
raise ExecUtilException("Command timed out after {} seconds.".format(timeout))
Expand All @@ -100,10 +109,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False,
error = error.decode(encoding)

if not ignore_errors and error_found and not expect_error:
error = normalize_error(error)
assert type(error) == str # noqa: E721
message = "Utility exited with non-zero code. Error: " + error
raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result)
RaiseError.UtilityExitedWithNonZeroCode(
cmd=cmd,
exit_code=exit_status,
msg_arg=error,
error=error,
out=result)

if verbose:
return exit_status, result, error
Expand Down
2 changes: 1 addition & 1 deletion tests/test_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def test_exec_command_failure(self):
error = e.message
break
raise Exception("We wait an exception!")
assert error == "Utility exited with non-zero code. Error `b'/bin/sh: 1: nonexistent_command: not found\\n'`"
assert error == "Utility exited with non-zero code. Error: `/bin/sh: 1: nonexistent_command: not found`"

def test_exec_command_failure__expect_error(self):
"""
Expand Down
Loading