From 1618d7cad2e73a1a130535a466bc056c4c6a3170 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 8 Apr 2025 17:18:29 +0300 Subject: [PATCH 1/6] A support of Python 3.8 [typing] Python 3.8 does not support tuple[...], list[...], set[...] and so on. We will use the analogues from typing package: typing.Tuple[...], typing.List[...] and typing.Set[...]. --- testgres/operations/remote_ops.py | 5 +++-- testgres/port_manager.py | 9 +++++---- tests/conftest.py | 22 +++++++++++++--------- tests/test_os_ops_common.py | 3 ++- tests/test_testgres_common.py | 10 +++++----- tests/test_utils.py | 3 ++- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 25d02f38..33b61ac2 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -5,6 +5,7 @@ import tempfile import io import logging +import typing from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -669,8 +670,8 @@ def _is_port_free__process_1(error: str) -> bool: return True @staticmethod - def _make_exec_env_list() -> list[str]: - result = list[str]() + def _make_exec_env_list() -> typing.List[str]: + result: typing.List[str] = list() for envvar in os.environ.items(): if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): continue diff --git a/testgres/port_manager.py b/testgres/port_manager.py index 164661e7..e2530470 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -6,6 +6,7 @@ import threading import random +import typing class PortManager: @@ -50,16 +51,16 @@ class PortManager__Generic(PortManager): _os_ops: OsOperations _guard: object # TODO: is there better to use bitmap fot _available_ports? - _available_ports: set[int] - _reserved_ports: set[int] + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] def __init__(self, os_ops: OsOperations): assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports = set[int](range(1024, 65535)) - self._reserved_ports = set[int]() + self._available_ports: typing.Set[int] = set(range(1024, 65535)) + self._reserved_ports: typing.Set[int] = set() def reserve_port(self) -> int: assert self._guard is not None diff --git a/tests/conftest.py b/tests/conftest.py index ff3b3cb4..6f2f9e41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,10 @@ class TestConfigPropNames: TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" +# ///////////////////////////////////////////////////////////////////////////// + +T_TUPLE__str_int = typing.Tuple[str, int] + # ///////////////////////////////////////////////////////////////////////////// # TestStartupData__Helper @@ -110,11 +114,11 @@ class TEST_PROCESS_STATS: cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str, int]() - XFailedTests = list[str, int]() - NotXFailedTests = list[str]() - WarningTests = list[str, int]() - AchtungTests = list[str]() + FailedTests: typing.List[T_TUPLE__str_int] = list() + XFailedTests: typing.List[T_TUPLE__str_int] = list() + NotXFailedTests: typing.List[str] = list() + WarningTests: typing.List[T_TUPLE__str_int] = list() + AchtungTests: typing.List[str] = list() cTotalDuration: datetime.timedelta = datetime.timedelta() @@ -769,7 +773,7 @@ def helper__calc_W(n: int) -> int: # ------------------------------------------------------------------------ -def helper__print_test_list(tests: list[str]) -> None: +def helper__print_test_list(tests: typing.List[str]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -796,7 +800,7 @@ def helper__print_test_list(tests: list[str]) -> None: # ------------------------------------------------------------------------ -def helper__print_test_list2(tests: list[str, int]) -> None: +def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -843,7 +847,7 @@ def LOCAL__print_line1_with_header(header: str): assert header != "" logging.info(C_LINE1 + " [" + header + "]") - def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): + def LOCAL__print_test_list(header: str, test_count: int, test_list: typing.List[str]): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 assert type(test_list) == list # noqa: E721 @@ -858,7 +862,7 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): logging.info("") def LOCAL__print_test_list2( - header: str, test_count: int, test_list: list[str, int] + header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int] ): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 7d183775..ecfff5b2 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -12,13 +12,14 @@ import logging import socket import threading +import typing from ..testgres import InvalidOperationException from ..testgres import ExecUtilException class TestOsOpsCommon: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index c384dfb2..e1252de2 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -56,7 +56,7 @@ def removing(os_ops: OsOperations, f): class TestTestgresCommon: - sm_node_svcs: list[PostgresNodeService] = [ + sm_node_svcs: typing.List[PostgresNodeService] = [ PostgresNodeServices.sm_local, PostgresNodeServices.sm_local2, PostgresNodeServices.sm_remote, @@ -315,8 +315,8 @@ def test_child_pids(self, node_svc: PostgresNodeService): def LOCAL__test_auxiliary_pids( node: PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: + expectedTypes: typing.List[ProcessType] + ) -> typing.List[ProcessType]: # returns list of the absence processes assert node is not None assert type(node) == PostgresNode # noqa: E721 @@ -327,7 +327,7 @@ def LOCAL__test_auxiliary_pids( assert pids is not None # noqa: E721 assert type(pids) == dict # noqa: E721 - result = list[ProcessType]() + result: typing.List[ProcessType] = list() for ptype in expectedTypes: if not (ptype in pids): result.append(ptype) @@ -335,7 +335,7 @@ def LOCAL__test_auxiliary_pids( def LOCAL__check_auxiliary_pids__multiple_attempts( node: PostgresNode, - expectedTypes: list[ProcessType]): + expectedTypes: typing.List[ProcessType]): assert node is not None assert type(node) == PostgresNode # noqa: E721 assert expectedTypes is not None diff --git a/tests/test_utils.py b/tests/test_utils.py index d4a4c9ad..c05bd2fe 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,10 +7,11 @@ from ..testgres import scoped_config import pytest +import typing class TestUtils: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] From 96ddf19fa5218f79b2a7f9d58753ac6200eae795 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 8 Apr 2025 18:43:40 +0300 Subject: [PATCH 2/6] A support of Python 3.8 [import] This commit fixes a problem with imports. As I understand 3.8 has a problem with import across root-directory and I do not know/find how to fix it correctly. If someone know how to resolve these problems more easily (through sys.path/__init__.py?) - you are welcome! --- .../pg_probackup2/tests/test_basic.py | 7 +- tests/helpers/global_data.py | 27 +++++-- tests/test_config.py | 21 ++++-- tests/test_os_ops_common.py | 11 ++- tests/test_os_ops_remote.py | 6 +- tests/test_testgres_common.py | 73 +++++++++++++------ tests/test_testgres_local.py | 49 +++++++++---- tests/test_testgres_remote.py | 27 +++++-- tests/test_utils.py | 12 ++- 9 files changed, 166 insertions(+), 67 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index ba788623..f6eb4b7f 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -4,7 +4,12 @@ import shutil import pytest -from ...... import testgres +try: + # Python 3.8 + import testgres +except ImportError: + from ...... import testgres + from ...pg_probackup2.app import ProbackupApp from ...pg_probackup2.init_helpers import Init, init_params from ..storage.fs_backup import FSTestBackupDir diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index c21d7dd8..6c5d9db2 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -1,11 +1,22 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations - -from ...testgres.node import PortManager -from ...testgres.node import PortManager__ThisHost -from ...testgres.node import PortManager__Generic +try: + # Python 3.8 + from testgres.operations.os_ops import OsOperations + from testgres.operations.os_ops import ConnectionParams + from testgres.operations.local_ops import LocalOperations + from testgres.operations.remote_ops import RemoteOperations + + from testgres.port_manager import PortManager + from testgres.port_manager import PortManager__ThisHost + from testgres.port_manager import PortManager__Generic +except ImportError: + from ...testgres.operations.os_ops import OsOperations + from ...testgres.operations.os_ops import ConnectionParams + from ...testgres.operations.local_ops import LocalOperations + from ...testgres.operations.remote_ops import RemoteOperations + + from ...testgres.port_manager import PortManager + from ...testgres.port_manager import PortManager__ThisHost + from ...testgres.port_manager import PortManager__Generic import os diff --git a/tests/test_config.py b/tests/test_config.py index 05702e9a..65f2e9ce 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,18 @@ -from ..testgres import TestgresConfig -from ..testgres import configure_testgres -from ..testgres import scoped_config -from ..testgres import pop_config - -from .. import testgres +try: + # Python 3.8 + import testgres + + from testgres.config import TestgresConfig + from testgres import configure_testgres + from testgres import scoped_config + from testgres import pop_config +except ImportError: + from .. import testgres + + from ..testgres.config import TestgresConfig + from ..testgres import configure_testgres + from ..testgres import scoped_config + from ..testgres import pop_config import pytest diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index ecfff5b2..e81f92dd 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -4,6 +4,14 @@ from .helpers.global_data import OsOperations from .helpers.run_conditions import RunConditions +try: + # Python 3.8 + from testgres import InvalidOperationException + from testgres import ExecUtilException +except ImportError: + from ..testgres import InvalidOperationException + from ..testgres import ExecUtilException + import os import pytest @@ -14,9 +22,6 @@ import threading import typing -from ..testgres import InvalidOperationException -from ..testgres import ExecUtilException - class TestOsOpsCommon: sm_os_ops_descrs: typing.List[OsOpsDescr] = [ diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 338e49f3..64f20258 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -3,7 +3,11 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres import ExecUtilException +try: + # Python 3.8 + from testgres import ExecUtilException +except ImportError: + from ..testgres import ExecUtilException import os import pytest diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index e1252de2..cadfe784 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -3,29 +3,56 @@ from .helpers.global_data import OsOperations from .helpers.global_data import PortManager -from ..testgres.node import PgVer -from ..testgres.node import PostgresNode -from ..testgres.utils import get_pg_version2 -from ..testgres.utils import file_tail -from ..testgres.utils import get_bin_path2 -from ..testgres import ProcessType -from ..testgres import NodeStatus -from ..testgres import IsolationLevel - -# New name prevents to collect test-functions in TestgresException and fixes -# the problem with pytest warning. -from ..testgres import TestgresException as testgres_TestgresException - -from ..testgres import InitNodeException -from ..testgres import StartNodeException -from ..testgres import QueryException -from ..testgres import ExecUtilException -from ..testgres import TimeoutException -from ..testgres import InvalidOperationException -from ..testgres import BackupException -from ..testgres import ProgrammingError -from ..testgres import scoped_config -from ..testgres import First, Any +try: + # Python 3.8 + from testgres.node import PgVer + from testgres.node import PostgresNode + from testgres.utils import get_pg_version2 + from testgres.utils import file_tail + from testgres.utils import get_bin_path2 + from testgres import ProcessType + from testgres import NodeStatus + from testgres import IsolationLevel + + # New name prevents to collect test-functions in TestgresException and fixes + # the problem with pytest warning. + from testgres import TestgresException as testgres_TestgresException + + from testgres import InitNodeException + from testgres import StartNodeException + from testgres import QueryException + from testgres import ExecUtilException + from testgres import TimeoutException + from testgres import InvalidOperationException + from testgres import BackupException + from testgres import ProgrammingError + from testgres import scoped_config + from testgres import First, Any + +except ImportError: + from ..testgres.node import PgVer + from ..testgres.node import PostgresNode + from ..testgres.utils import get_pg_version2 + from ..testgres.utils import file_tail + from ..testgres.utils import get_bin_path2 + from ..testgres import ProcessType + from ..testgres import NodeStatus + from ..testgres import IsolationLevel + + # New name prevents to collect test-functions in TestgresException and fixes + # the problem with pytest warning. + from ..testgres import TestgresException as testgres_TestgresException + + from ..testgres import InitNodeException + from ..testgres import StartNodeException + from ..testgres import QueryException + from ..testgres import ExecUtilException + from ..testgres import TimeoutException + from ..testgres import InvalidOperationException + from ..testgres import BackupException + from ..testgres import ProgrammingError + from ..testgres import scoped_config + from ..testgres import First, Any from contextlib import contextmanager diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 9dbd455b..7713cac0 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -7,21 +7,40 @@ import platform import logging -from .. import testgres - -from ..testgres import StartNodeException -from ..testgres import ExecUtilException -from ..testgres import NodeApp -from ..testgres import scoped_config -from ..testgres import get_new_node -from ..testgres import get_bin_path -from ..testgres import get_pg_config -from ..testgres import get_pg_version - -# NOTE: those are ugly imports -from ..testgres.utils import bound_ports -from ..testgres.utils import PgVer -from ..testgres.node import ProcessProxy + +try: + # Python 3.8 + import testgres + + from testgres import StartNodeException + from testgres import ExecUtilException + from testgres import NodeApp + from testgres import scoped_config + from testgres import get_new_node + from testgres import get_bin_path + from testgres import get_pg_config + from testgres import get_pg_version + + # NOTE: those are ugly imports + from testgres.utils import bound_ports + from testgres.utils import PgVer + from testgres.node import ProcessProxy +except ImportError: + from .. import testgres + + from ..testgres import StartNodeException + from ..testgres import ExecUtilException + from ..testgres import NodeApp + from ..testgres import scoped_config + from ..testgres import get_new_node + from ..testgres import get_bin_path + from ..testgres import get_pg_config + from ..testgres import get_pg_version + + # NOTE: those are ugly imports + from ..testgres.utils import bound_ports + from ..testgres.utils import PgVer + from ..testgres.node import ProcessProxy def pg_version_ge(version): diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index e38099b7..e3b7647f 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -7,16 +7,29 @@ from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices -from .. import testgres +try: + # Python 3.8 + import testgres -from ..testgres.exceptions import InitNodeException -from ..testgres.exceptions import ExecUtilException + from testgres.exceptions import InitNodeException + from testgres.exceptions import ExecUtilException -from ..testgres.config import scoped_config -from ..testgres.config import testgres_config + from testgres.config import scoped_config + from testgres.config import testgres_config -from ..testgres import get_bin_path -from ..testgres import get_pg_config + from testgres import get_bin_path + from testgres import get_pg_config +except ImportError: + from .. import testgres + + from ..testgres.exceptions import InitNodeException + from ..testgres.exceptions import ExecUtilException + + from ..testgres.config import scoped_config + from ..testgres.config import testgres_config + + from ..testgres import get_bin_path + from ..testgres import get_pg_config # NOTE: those are ugly imports diff --git a/tests/test_utils.py b/tests/test_utils.py index c05bd2fe..be22bb4c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,9 +2,15 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres.utils import parse_pg_version -from ..testgres.utils import get_pg_config2 -from ..testgres import scoped_config +try: + # Python 3.8 + from testgres.utils import parse_pg_version + from testgres.utils import get_pg_config2 + from testgres import scoped_config +except ImportError: + from ..testgres.utils import parse_pg_version + from ..testgres.utils import get_pg_config2 + from ..testgres import scoped_config import pytest import typing From 208c5e39b5ca597ec1fba5a69e26e710a42e5aeb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 8 Apr 2025 23:08:12 +0300 Subject: [PATCH 3/6] [CI] Test with python 3.8, 3.9, 3.10 and 3.11 [std2-all][alpine] We will run all the tests to get a full information about compatibility. Some new tests may fail. --- TODO: may be it is better to compile/install required python version explicitly instead using pyenv. --- .travis.yml | 4 ++ Dockerfile--std2-all.tmpl | 93 +++++++++++++++++++++++++++++++++++++++ run_tests2.sh | 68 ++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 Dockerfile--std2-all.tmpl create mode 100755 run_tests2.sh diff --git a/.travis.yml b/.travis.yml index 7557a2ce..72e23915 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,10 @@ notifications: on_failure: always env: + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.10 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.11 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl new file mode 100644 index 00000000..e5689850 --- /dev/null +++ b/Dockerfile--std2-all.tmpl @@ -0,0 +1,93 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv + +# For pyenv +RUN apk add build-base gcc-doc +RUN apk add patch +RUN apk add git +RUN apk add xz-dev +RUN apk add zip +RUN apk add zlib-dev +RUN apk add libffi-dev +RUN apk add readline-dev +RUN apk add openssl openssl-dev +RUN apk add sqlite-dev +RUN apk add bzip2-dev + +# --------------------------------------------- base3_with_python-3.8 +FROM base2_with_python-3 as base3_with_python-3.8 +ENV PYTHON_VERSION=3.8 + +# --------------------------------------------- base3_with_python-3.9 +FROM base2_with_python-3 as base3_with_python-3.9 +ENV PYTHON_VERSION=3.9 + +# --------------------------------------------- base3_with_python-3.10 +FROM base2_with_python-3 as base3_with_python-3.10 +ENV PYTHON_VERSION=3.10 + +# --------------------------------------------- base3_with_python-3.11 +FROM base2_with_python-3 as base3_with_python-3.11 +ENV PYTHON_VERSION=3.11 + +# --------------------------------------------- final +FROM base3_with_python-${PYTHON_VERSION} as final + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash + +RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +export PATH=\"~/.pyenv/bin:$PATH\"; \ +TEST_FILTER=\"\" bash run_tests2.sh;" diff --git a/run_tests2.sh b/run_tests2.sh new file mode 100755 index 00000000..173b19dc --- /dev/null +++ b/run_tests2.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2025 Postgres Professional + +set -eux + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +pyenv virtualenv --force ${PYTHON_VERSION} cur +pyenv activate cur + +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +fi + +# fail early +echo check that pg_config is in PATH +command -v pg_config + +# prepare python environment +VENV_PATH="/tmp/testgres_venv" +rm -rf $VENV_PATH +python -m venv "${VENV_PATH}" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil + +# install testgres' dependencies +export PYTHONPATH=$(pwd) +# $PIP install . + +# test code quality +flake8 . + + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + + +# run tests (PATH) +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_BIN) +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_CONFIG) +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# show coverage +coverage report + +# build documentation +cd docs +make html +cd .. + +# attempt to fix codecov +set +eux + +# send coverage stats to Codecov +bash <(curl -s https://codecov.io/bash) From 48c7625298adc52dc4efbee82f1620f9f4922f7e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 9 Apr 2025 11:19:36 +0300 Subject: [PATCH 4/6] Revert "A support of Python 3.8 [import]" This reverts commit 96ddf19fa5218f79b2a7f9d58753ac6200eae795. It seems to me there was a local gluk. Let's test without these changes. --- .../pg_probackup2/tests/test_basic.py | 7 +- tests/helpers/global_data.py | 27 ++----- tests/test_config.py | 21 ++---- tests/test_os_ops_common.py | 11 +-- tests/test_os_ops_remote.py | 6 +- tests/test_testgres_common.py | 73 ++++++------------- tests/test_testgres_local.py | 49 ++++--------- tests/test_testgres_remote.py | 27 ++----- tests/test_utils.py | 12 +-- 9 files changed, 67 insertions(+), 166 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index f6eb4b7f..ba788623 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -4,12 +4,7 @@ import shutil import pytest -try: - # Python 3.8 - import testgres -except ImportError: - from ...... import testgres - +from ...... import testgres from ...pg_probackup2.app import ProbackupApp from ...pg_probackup2.init_helpers import Init, init_params from ..storage.fs_backup import FSTestBackupDir diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index 6c5d9db2..c21d7dd8 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -1,22 +1,11 @@ -try: - # Python 3.8 - from testgres.operations.os_ops import OsOperations - from testgres.operations.os_ops import ConnectionParams - from testgres.operations.local_ops import LocalOperations - from testgres.operations.remote_ops import RemoteOperations - - from testgres.port_manager import PortManager - from testgres.port_manager import PortManager__ThisHost - from testgres.port_manager import PortManager__Generic -except ImportError: - from ...testgres.operations.os_ops import OsOperations - from ...testgres.operations.os_ops import ConnectionParams - from ...testgres.operations.local_ops import LocalOperations - from ...testgres.operations.remote_ops import RemoteOperations - - from ...testgres.port_manager import PortManager - from ...testgres.port_manager import PortManager__ThisHost - from ...testgres.port_manager import PortManager__Generic +from ...testgres.operations.os_ops import OsOperations +from ...testgres.operations.os_ops import ConnectionParams +from ...testgres.operations.local_ops import LocalOperations +from ...testgres.operations.remote_ops import RemoteOperations + +from ...testgres.node import PortManager +from ...testgres.node import PortManager__ThisHost +from ...testgres.node import PortManager__Generic import os diff --git a/tests/test_config.py b/tests/test_config.py index 65f2e9ce..05702e9a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,18 +1,9 @@ -try: - # Python 3.8 - import testgres - - from testgres.config import TestgresConfig - from testgres import configure_testgres - from testgres import scoped_config - from testgres import pop_config -except ImportError: - from .. import testgres - - from ..testgres.config import TestgresConfig - from ..testgres import configure_testgres - from ..testgres import scoped_config - from ..testgres import pop_config +from ..testgres import TestgresConfig +from ..testgres import configure_testgres +from ..testgres import scoped_config +from ..testgres import pop_config + +from .. import testgres import pytest diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index e81f92dd..ecfff5b2 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -4,14 +4,6 @@ from .helpers.global_data import OsOperations from .helpers.run_conditions import RunConditions -try: - # Python 3.8 - from testgres import InvalidOperationException - from testgres import ExecUtilException -except ImportError: - from ..testgres import InvalidOperationException - from ..testgres import ExecUtilException - import os import pytest @@ -22,6 +14,9 @@ import threading import typing +from ..testgres import InvalidOperationException +from ..testgres import ExecUtilException + class TestOsOpsCommon: sm_os_ops_descrs: typing.List[OsOpsDescr] = [ diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 64f20258..338e49f3 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -3,11 +3,7 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -try: - # Python 3.8 - from testgres import ExecUtilException -except ImportError: - from ..testgres import ExecUtilException +from ..testgres import ExecUtilException import os import pytest diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index cadfe784..e1252de2 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -3,56 +3,29 @@ from .helpers.global_data import OsOperations from .helpers.global_data import PortManager -try: - # Python 3.8 - from testgres.node import PgVer - from testgres.node import PostgresNode - from testgres.utils import get_pg_version2 - from testgres.utils import file_tail - from testgres.utils import get_bin_path2 - from testgres import ProcessType - from testgres import NodeStatus - from testgres import IsolationLevel - - # New name prevents to collect test-functions in TestgresException and fixes - # the problem with pytest warning. - from testgres import TestgresException as testgres_TestgresException - - from testgres import InitNodeException - from testgres import StartNodeException - from testgres import QueryException - from testgres import ExecUtilException - from testgres import TimeoutException - from testgres import InvalidOperationException - from testgres import BackupException - from testgres import ProgrammingError - from testgres import scoped_config - from testgres import First, Any - -except ImportError: - from ..testgres.node import PgVer - from ..testgres.node import PostgresNode - from ..testgres.utils import get_pg_version2 - from ..testgres.utils import file_tail - from ..testgres.utils import get_bin_path2 - from ..testgres import ProcessType - from ..testgres import NodeStatus - from ..testgres import IsolationLevel - - # New name prevents to collect test-functions in TestgresException and fixes - # the problem with pytest warning. - from ..testgres import TestgresException as testgres_TestgresException - - from ..testgres import InitNodeException - from ..testgres import StartNodeException - from ..testgres import QueryException - from ..testgres import ExecUtilException - from ..testgres import TimeoutException - from ..testgres import InvalidOperationException - from ..testgres import BackupException - from ..testgres import ProgrammingError - from ..testgres import scoped_config - from ..testgres import First, Any +from ..testgres.node import PgVer +from ..testgres.node import PostgresNode +from ..testgres.utils import get_pg_version2 +from ..testgres.utils import file_tail +from ..testgres.utils import get_bin_path2 +from ..testgres import ProcessType +from ..testgres import NodeStatus +from ..testgres import IsolationLevel + +# New name prevents to collect test-functions in TestgresException and fixes +# the problem with pytest warning. +from ..testgres import TestgresException as testgres_TestgresException + +from ..testgres import InitNodeException +from ..testgres import StartNodeException +from ..testgres import QueryException +from ..testgres import ExecUtilException +from ..testgres import TimeoutException +from ..testgres import InvalidOperationException +from ..testgres import BackupException +from ..testgres import ProgrammingError +from ..testgres import scoped_config +from ..testgres import First, Any from contextlib import contextmanager diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 7713cac0..9dbd455b 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -7,40 +7,21 @@ import platform import logging - -try: - # Python 3.8 - import testgres - - from testgres import StartNodeException - from testgres import ExecUtilException - from testgres import NodeApp - from testgres import scoped_config - from testgres import get_new_node - from testgres import get_bin_path - from testgres import get_pg_config - from testgres import get_pg_version - - # NOTE: those are ugly imports - from testgres.utils import bound_ports - from testgres.utils import PgVer - from testgres.node import ProcessProxy -except ImportError: - from .. import testgres - - from ..testgres import StartNodeException - from ..testgres import ExecUtilException - from ..testgres import NodeApp - from ..testgres import scoped_config - from ..testgres import get_new_node - from ..testgres import get_bin_path - from ..testgres import get_pg_config - from ..testgres import get_pg_version - - # NOTE: those are ugly imports - from ..testgres.utils import bound_ports - from ..testgres.utils import PgVer - from ..testgres.node import ProcessProxy +from .. import testgres + +from ..testgres import StartNodeException +from ..testgres import ExecUtilException +from ..testgres import NodeApp +from ..testgres import scoped_config +from ..testgres import get_new_node +from ..testgres import get_bin_path +from ..testgres import get_pg_config +from ..testgres import get_pg_version + +# NOTE: those are ugly imports +from ..testgres.utils import bound_ports +from ..testgres.utils import PgVer +from ..testgres.node import ProcessProxy def pg_version_ge(version): diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index e3b7647f..e38099b7 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -7,29 +7,16 @@ from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices -try: - # Python 3.8 - import testgres +from .. import testgres - from testgres.exceptions import InitNodeException - from testgres.exceptions import ExecUtilException +from ..testgres.exceptions import InitNodeException +from ..testgres.exceptions import ExecUtilException - from testgres.config import scoped_config - from testgres.config import testgres_config +from ..testgres.config import scoped_config +from ..testgres.config import testgres_config - from testgres import get_bin_path - from testgres import get_pg_config -except ImportError: - from .. import testgres - - from ..testgres.exceptions import InitNodeException - from ..testgres.exceptions import ExecUtilException - - from ..testgres.config import scoped_config - from ..testgres.config import testgres_config - - from ..testgres import get_bin_path - from ..testgres import get_pg_config +from ..testgres import get_bin_path +from ..testgres import get_pg_config # NOTE: those are ugly imports diff --git a/tests/test_utils.py b/tests/test_utils.py index be22bb4c..c05bd2fe 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,15 +2,9 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -try: - # Python 3.8 - from testgres.utils import parse_pg_version - from testgres.utils import get_pg_config2 - from testgres import scoped_config -except ImportError: - from ..testgres.utils import parse_pg_version - from ..testgres.utils import get_pg_config2 - from ..testgres import scoped_config +from ..testgres.utils import parse_pg_version +from ..testgres.utils import get_pg_config2 +from ..testgres import scoped_config import pytest import typing From b0fd1277a068399951c9477dffa7afe373730ae3 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 9 Apr 2025 13:43:18 +0300 Subject: [PATCH 5/6] [CI] Test with Python 3.8.0 is added [std2-all] --- .travis.yml | 5 +++-- Dockerfile--std2-all.tmpl | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72e23915..55b7afa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,9 @@ notifications: on_failure: always env: - - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 - - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8.0 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.10 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.11 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl index e5689850..3abbfe52 100644 --- a/Dockerfile--std2-all.tmpl +++ b/Dockerfile--std2-all.tmpl @@ -21,6 +21,10 @@ RUN apk add openssl openssl-dev RUN apk add sqlite-dev RUN apk add bzip2-dev +# --------------------------------------------- base3_with_python-3.8.0 +FROM base2_with_python-3 as base3_with_python-3.8.0 +ENV PYTHON_VERSION=3.8.0 + # --------------------------------------------- base3_with_python-3.8 FROM base2_with_python-3 as base3_with_python-3.8 ENV PYTHON_VERSION=3.8 From 0e68acb18f60a2baa139271b379e931a2dc179af Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 9 Apr 2025 13:48:54 +0300 Subject: [PATCH 6/6] Dockerfile--std2-all.tmpl is updated [cleanup] --- Dockerfile--std2-all.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl index 3abbfe52..10d8280c 100644 --- a/Dockerfile--std2-all.tmpl +++ b/Dockerfile--std2-all.tmpl @@ -6,10 +6,9 @@ FROM postgres:${PG_VERSION}-alpine as base1 # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 -RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers # For pyenv -RUN apk add build-base gcc-doc RUN apk add patch RUN apk add git RUN apk add xz-dev