From 219764facc13d31d383431c25bd00e96c2509bbf Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 06:50:16 +0300 Subject: [PATCH 1/7] conftest.py is updated [refactoring] - Unification for pytest and unittest cases: - pytest.Function - _pytest.unittest.TestCaseFunction - Achthung tests [with problems in fixtures?] - Cleanup --- tests/conftest.py | 63 +++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e37c3c77..d4a09ac0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,18 @@ # ///////////////////////////////////////////////////////////////////////////// # PyTest Configuration -import _pytest.outcomes - import pluggy import pytest -import _pytest import os import logging import pathlib import math import datetime +import _pytest.outcomes +import _pytest.unittest +import _pytest.logging + # ///////////////////////////////////////////////////////////////////////////// C_ROOT_DIR__RELATIVE = ".." @@ -91,10 +92,6 @@ def GetCurrentTestWorkerSignature() -> str: return __class__.sm_CurrentTestWorkerSignature -# /////////////////////////////////////////////////////////////////////////////# ///////////////////////////////////////////////////////////////////////////// -# Fixtures - - # ///////////////////////////////////////////////////////////////////////////// # TEST_PROCESS_STATS @@ -198,10 +195,13 @@ def helper__makereport__setup( assert item is not None assert call is not None assert outcome is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 + C_LINE1 = "******************************************************" + # logging.info("pytest_runtest_makereport - setup") TEST_PROCESS_STATS.incrementTotalTestCount() @@ -214,10 +214,6 @@ def helper__makereport__setup( TEST_PROCESS_STATS.incrementNotExecutedTestCount() return - assert rep.outcome == "passed" - - testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() - testID = "" if item.cls is not None: @@ -225,15 +221,25 @@ def helper__makereport__setup( testID = testID + item.name - if testNumber > 1: - logging.info("") + if rep.outcome == "passed": + testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() - logging.info("******************************************************") - logging.info("* START TEST {0}".format(testID)) + logging.info(C_LINE1) + logging.info("* START TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Number: {0}".format(testNumber)) + logging.info("*") + return + + assert rep.outcome != "passed" + + logging.info(C_LINE1) + logging.info("* ACHTUNG TEST {0}".format(testID)) logging.info("*") - logging.info("* Path : {0}".format(item.path)) - logging.info("* Number: {0}".format(testNumber)) + logging.info("* Outcome is [{0}]".format(rep.outcome)) logging.info("*") + return # ------------------------------------------------------------------------ @@ -243,12 +249,11 @@ def helper__makereport__call( assert item is not None assert call is not None assert outcome is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 - # logging.info("pytest_runtest_makereport - call") - rep = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -350,6 +355,7 @@ def helper__makereport__call( logging.info("* EXIT STATUS : {0}".format(exitStatus)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) + logging.info("*") # ///////////////////////////////////////////////////////////////////////////// @@ -359,17 +365,14 @@ def helper__makereport__call( def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert item is not None assert call is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - # logging.info("[pytest_runtest_makereport][#001][{0}][{1}]".format(item.name, call.when)) - outcome: pluggy.Result = yield assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 - # logging.info("[pytest_runtest_makereport][#002][{0}][{1}]".format(item.name, call.when)) - rep: pytest.TestReport = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -503,7 +506,13 @@ def pytest_configure(config: pytest.Config) -> None: log_path.mkdir(exist_ok=True) - logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( + "logging-plugin" + ) + + assert logging_plugin is not None + assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) + logging_plugin.set_log_path(str(log_path / log_name)) From 661c519ceebf9390e216e1de5fd265d3dedfbe0f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 09:53:37 +0300 Subject: [PATCH 2/7] conftest.py is updated. list of "achtung" tests is printed. For me - it is a right and awesome thing. --- tests/conftest.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index d4a09ac0..862aafa3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -106,10 +106,12 @@ class TEST_PROCESS_STATS: cSkippedTests: int = 0 cNotXFailedTests: int = 0 cUnexpectedTests: int = 0 + cAchtungTests: int = 0 FailedTests = list[str]() XFailedTests = list[str]() NotXFailedTests = list[str]() + AchtungTests = list[str]() # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: @@ -160,6 +162,13 @@ def incrementNotXFailedTests(testID: str) -> None: def incrementUnexpectedTests() -> None: __class__.cUnexpectedTests += 1 + # -------------------------------------------------------------------- + def incrementAchtungTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.AchtungTests) == list # noqa: E721 + + __class__.AchtungTests.append(testID) # raise? + __class__.cAchtungTests += 1 # ///////////////////////////////////////////////////////////////////////////// @@ -234,10 +243,20 @@ def helper__makereport__setup( assert rep.outcome != "passed" + TEST_PROCESS_STATS.incrementAchtungTestCount(testID) + logging.info(C_LINE1) logging.info("* ACHTUNG TEST {0}".format(testID)) logging.info("*") + logging.info("* Path : {0}".format(item.path)) logging.info("* Outcome is [{0}]".format(rep.outcome)) + + if rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + logging.info("*") + logging.error(call.excinfo.value) + logging.info("*") return @@ -443,6 +462,15 @@ def run_after_tests(request: pytest.FixtureRequest): yield + logging.info("--------------------------- [ACHTUNG TESTS]") + logging.info("") + + assert len(TEST_PROCESS_STATS.AchtungTests) == TEST_PROCESS_STATS.cAchtungTests + + if len(TEST_PROCESS_STATS.AchtungTests) > 0: + helper__print_test_list(TEST_PROCESS_STATS.AchtungTests) + logging.info("") + logging.info("--------------------------- [FAILED TESTS]") logging.info("") From f95c38e2f2b5777efd5f4192c784e1f4b5361a45 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 10:14:27 +0300 Subject: [PATCH 3/7] run_after_tests is refactored --- tests/conftest.py | 74 ++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 862aafa3..fe2974fa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -462,45 +462,53 @@ def run_after_tests(request: pytest.FixtureRequest): yield - logging.info("--------------------------- [ACHTUNG TESTS]") - logging.info("") - - assert len(TEST_PROCESS_STATS.AchtungTests) == TEST_PROCESS_STATS.cAchtungTests - - if len(TEST_PROCESS_STATS.AchtungTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.AchtungTests) - logging.info("") - - logging.info("--------------------------- [FAILED TESTS]") - logging.info("") - - assert len(TEST_PROCESS_STATS.FailedTests) == TEST_PROCESS_STATS.cFailedTests - - if len(TEST_PROCESS_STATS.FailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.FailedTests) - logging.info("") - - logging.info("--------------------------- [XFAILED TESTS]") - logging.info("") - - assert len(TEST_PROCESS_STATS.XFailedTests) == TEST_PROCESS_STATS.cXFailedTests - - if len(TEST_PROCESS_STATS.XFailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.XFailedTests) + C_LINE1 = "---------------------------" + + def LOCAL__print_line1_with_header(header: str): + assert type(C_LINE1) == str # noqa: E721 + assert type(header) == str # noqa: E721 + assert header != "" + logging.info(C_LINE1 + " [" + header + "]") + + def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) logging.info("") + if len(test_list) > 0: + helper__print_test_list(TEST_PROCESS_STATS.AchtungTests) + logging.info("") + + LOCAL__print_test_list( + "ACHTUNG TESTS", + TEST_PROCESS_STATS.cAchtungTests, + TEST_PROCESS_STATS.AchtungTests + ) - logging.info("--------------------------- [NOT XFAILED TESTS]") - logging.info("") + LOCAL__print_test_list( + "FAILED TESTS", + TEST_PROCESS_STATS.cFailedTests, + TEST_PROCESS_STATS.FailedTests + ) - assert ( - len(TEST_PROCESS_STATS.NotXFailedTests) == TEST_PROCESS_STATS.cNotXFailedTests + LOCAL__print_test_list( + "XFAILED TESTS", + TEST_PROCESS_STATS.cXFailedTests, + TEST_PROCESS_STATS.XFailedTests ) - if len(TEST_PROCESS_STATS.NotXFailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.NotXFailedTests) - logging.info("") + LOCAL__print_test_list( + "NOT XFAILED TESTS", + TEST_PROCESS_STATS.cNotXFailedTests, + TEST_PROCESS_STATS.NotXFailedTests + ) - logging.info("--------------------------- [SUMMARY STATISTICS]") + LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) From f5b293faf4d8b7071c311c034e42323da47c1b95 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 10:20:59 +0300 Subject: [PATCH 4/7] conftest is updated A total count of achtung tests is printed in a summary section. I forgot about it. --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index fe2974fa..d9ee7ec4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -514,6 +514,7 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) logging.info("") logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) From f1f2bb24e60b0055425b6f63797019ec6b3578bb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 10:25:01 +0300 Subject: [PATCH 5/7] conftest is updated An assert for "unexpected" finish of test is deleted. We must avoid creating a such problem in our infrastructure. --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index d9ee7ec4..738bd9e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -365,7 +365,8 @@ def helper__makereport__call( else: TEST_PROCESS_STATS.incrementUnexpectedTests() exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) - assert False + # [2025-03-28] It may create a useless problem in new environment. + # assert False # -------- logging.info("*") From 6568eeb1b9edd8d07c43d3aedb9d92249ef52337 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 10:47:26 +0300 Subject: [PATCH 6/7] [BUG FIX] LOCAL__print_test_list is corrected it always printed TEST_PROCESS_STATS.AchtungTests. --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 738bd9e5..a500c3ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -482,7 +482,7 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): LOCAL__print_line1_with_header(header) logging.info("") if len(test_list) > 0: - helper__print_test_list(TEST_PROCESS_STATS.AchtungTests) + helper__print_test_list(test_list) logging.info("") LOCAL__print_test_list( From d85eed296888c696cd34653aae949de03a2353cb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 13:19:32 +0300 Subject: [PATCH 7/7] conftest is formatted [black] --- tests/conftest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a500c3ec..0f65838e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -170,6 +170,7 @@ def incrementAchtungTestCount(testID: str) -> None: __class__.AchtungTests.append(testID) # raise? __class__.cAchtungTests += 1 + # ///////////////////////////////////////////////////////////////////////////// @@ -485,10 +486,11 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): helper__print_test_list(test_list) logging.info("") + # fmt: off LOCAL__print_test_list( "ACHTUNG TESTS", TEST_PROCESS_STATS.cAchtungTests, - TEST_PROCESS_STATS.AchtungTests + TEST_PROCESS_STATS.AchtungTests, ) LOCAL__print_test_list( @@ -500,14 +502,15 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): LOCAL__print_test_list( "XFAILED TESTS", TEST_PROCESS_STATS.cXFailedTests, - TEST_PROCESS_STATS.XFailedTests + TEST_PROCESS_STATS.XFailedTests, ) LOCAL__print_test_list( "NOT XFAILED TESTS", TEST_PROCESS_STATS.cNotXFailedTests, - TEST_PROCESS_STATS.NotXFailedTests + TEST_PROCESS_STATS.NotXFailedTests, ) + # fmt: on LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("")