diff --git a/src/enums.py b/src/enums.py index d07d8068..a483c5b4 100644 --- a/src/enums.py +++ b/src/enums.py @@ -29,7 +29,7 @@ class NodeStatus(IntEnum): Status of a PostgresNode """ - Running, Stopped, Uninitialized = range(3) + Running, Stopped, Uninitialized, Zombie = range(4) # for Python 3.x def __bool__(self): diff --git a/src/impl/platforms/internal_platform_utils.py b/src/impl/platforms/internal_platform_utils.py index 6af2b7b0..3ae684ed 100644 --- a/src/impl/platforms/internal_platform_utils.py +++ b/src/impl/platforms/internal_platform_utils.py @@ -60,3 +60,12 @@ def FindPostmaster( assert type(bin_dir) is str assert type(data_dir) is str raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.") + + def ProcessIsZombi_soft_check( + self, + os_ops: OsOperations, + pid: int, + ) -> typing.Optional[bool]: + assert isinstance(os_ops, OsOperations) + assert type(pid) is int + raise NotImplementedError("InternalPlatformUtils::ProcessIsZombi_soft_ver is not implemented.") diff --git a/src/impl/platforms/linux/internal_platform_utils.py b/src/impl/platforms/linux/internal_platform_utils.py index b8ca8588..b0ed64c3 100644 --- a/src/impl/platforms/linux/internal_platform_utils.py +++ b/src/impl/platforms/linux/internal_platform_utils.py @@ -8,6 +8,7 @@ import re import shlex +import typing class InternalPlatformUtils(base.InternalPlatformUtils): @@ -118,3 +119,53 @@ def is_space_or_tab(ch) -> bool: assert type(pid) is int return __class__.FindPostmasterResult.create_ok(pid) + + def ProcessIsZombi_soft_check( + self, + os_ops: OsOperations, + pid: int, + ) -> typing.Optional[bool]: + assert isinstance(os_ops, OsOperations) + assert type(pid) is int + + proc_stat_file = os_ops.build_path("/proc", str(pid), "stat") + + if not os_ops.path_exists(proc_stat_file): + return False + + result: typing.Optional[bool] = None + + try: + # Read one line from /proc/PID/stat + stat_content = os_ops.read_binary(proc_stat_file, 0).decode("utf-8", errors="ignore") + + # We look for the closing parenthesis of the process name to ensure that + # we start from it and not depend on spaces inside the parentheses! + r_paren_idx = stat_content.rfind(")") + + if r_paren_idx == -1: + pass + elif len(stat_content) <= r_paren_idx + 2: + pass + else: + # The status goes exactly one space after the closing bracket + assert (r_paren_idx + 2) < len(stat_content) + proc_status = stat_content[r_paren_idx + 2] + result = proc_status == "Z" + except Exception as e: + # If the file disappeared right during reading, it means the process is completely erased + if __class__._is_file_not_found_exception(e): + result = False + + return result + + @staticmethod + def _is_file_not_found_exception(e: Exception) -> bool: + if isinstance(e, FileNotFoundError): + return True + + if isinstance(e, ExecUtilException): + if e.exit_code == 2: + return True + + return False diff --git a/src/impl/platforms/win32/internal_platform_utils.py b/src/impl/platforms/win32/internal_platform_utils.py index 846950f0..c6df6378 100644 --- a/src/impl/platforms/win32/internal_platform_utils.py +++ b/src/impl/platforms/win32/internal_platform_utils.py @@ -2,6 +2,7 @@ from .. import internal_platform_utils as base from testgres.operations.os_ops import OsOperations +import typing class InternalPlatformUtils(base.InternalPlatformUtils): @@ -15,3 +16,12 @@ def FindPostmaster( assert type(bin_dir) is str assert type(data_dir) is str return __class__.FindPostmasterResult.create_not_implemented() + + def ProcessIsZombi_soft_check( + self, + os_ops: OsOperations, + pid: int, + ) -> typing.Optional[bool]: + assert isinstance(os_ops, OsOperations) + assert type(pid) is int + return None diff --git a/src/utils.py b/src/utils.py index 227c6525..6efd9a49 100644 --- a/src/utils.py +++ b/src/utils.py @@ -411,7 +411,23 @@ def get_pg_node_state( attempt = 0 sleep_time = C_SLEEP_TIME1 - platform_utils: typing.Optional[internal_platform_utils_factory.InternalPlatformUtils] = None + class tagPlaformUtilsProvider: + T_PLATFORM_UTILS = internal_platform_utils_factory.InternalPlatformUtils + + _platform_utils: typing.Optional[T_PLATFORM_UTILS] = None + + def __init__(self): + self._platform_utils = None + + def get(self) -> T_PLATFORM_UTILS: + if self._platform_utils is None: + self._platform_utils = internal_platform_utils_factory.create_internal_platform_utils(os_ops) + assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS) + + assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS) + return self._platform_utils + + platform_utils_provider = tagPlaformUtilsProvider() while True: assert type(attempt) is int @@ -513,6 +529,13 @@ def get_pg_node_state( assert pid != 0 + # ----------------- detect zombie + if platform_utils_provider.get().ProcessIsZombi_soft_check(os_ops, pid) is True: + internal_utils.send_log_debug("Postmaster process {} is a zombie.".format( + pid, + )) + return PostgresNodeState(NodeStatus.Zombie, pid) + # ----------------- return PostgresNodeState(NodeStatus.Running, pid) @@ -543,14 +566,8 @@ def get_pg_node_state( bin_dir, )) - if platform_utils is None: - platform_utils = internal_platform_utils_factory.create_internal_platform_utils(os_ops) - assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils) - - assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils) - try: - find_postmaster_r = platform_utils.FindPostmaster( + find_postmaster_r = platform_utils_provider.get().FindPostmaster( os_ops, bin_dir, data_dir, diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 883fcf67..9be4a778 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -687,7 +687,9 @@ def test_kill__ok( ): assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(node_svc) as node: + node = __class__.helper__get_node(node_svc) + + try: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) @@ -696,6 +698,9 @@ def test_kill__ok( assert not node.is_started node.slow_start() assert node.is_started + + assert node.status() == NodeStatus.Running + node.kill() assert not node.is_started @@ -717,8 +722,19 @@ def test_kill__ok( if s == NodeStatus.Running: continue - assert s == NodeStatus.Stopped + if s == NodeStatus.Stopped: + logging.info("Node stopped") + break + + if s == NodeStatus.Zombie: + logging.info("Node is zombie") + break + + logging.error("Node has unknown status: {}.".format(s.name)) break + finally: + if node.is_started: + node.stop() return def test_kill_backgroud_writer__ok( @@ -1614,9 +1630,15 @@ def impl__test_pg_ctl_wait_option( logging.info("Attempt #{0}.".format(nAttempt)) s1 = node.status() + logging.info("Node status is {}.".format(s1.name)) + if s1 == NodeStatus.Running: continue + if s1 == NodeStatus.Zombie: + # [2026-07-12] We will wait for final stop (stabilization). OK? + continue + if s1 == NodeStatus.Stopped: break