diff --git a/pytest_reportportal/plugin.py b/pytest_reportportal/plugin.py index 6fab6aa1..d7188c6c 100644 --- a/pytest_reportportal/plugin.py +++ b/pytest_reportportal/plugin.py @@ -149,6 +149,10 @@ def pytest_sessionfinish(session: Session) -> None: return config.py_test_service.finish_suites() + + if hasattr(config.py_test_service, 'cleanup_retry_state'): + config.py_test_service.cleanup_retry_state() + if is_control(config): config.py_test_service.finish_launch() @@ -232,7 +236,7 @@ def pytest_runtestloop(session: Session) -> Generator[None, Any, None]: # noinspection PyProtectedMember -@pytest.hookimpl(hookwrapper=True) +@pytest.hookimpl(hookwrapper=True, trylast=True) def pytest_runtest_protocol(item: Item) -> Generator[None, Any, None]: """Control start and finish of pytest items. @@ -271,6 +275,10 @@ def pytest_runtest_protocol(item: Item) -> Generator[None, Any, None]: def pytest_runtest_makereport(item: Item) -> Generator[None, Any, None]: """Change runtest_makereport function. + Enhanced to detect and handle pytest-rerunfailures retry transitions. + Monitors execution_count changes to identify when a test is retried, + allowing each attempt to be reported as a separate item. + :param item: pytest.Item :return: None """ @@ -279,6 +287,7 @@ def pytest_runtest_makereport(item: Item) -> Generator[None, Any, None]: return report = result.get_result() service = item.config.py_test_service + service.handle_retry_transition(item, report) service.process_results(item, report) diff --git a/pytest_reportportal/service.py b/pytest_reportportal/service.py index fceb5f44..7b52fe66 100644 --- a/pytest_reportportal/service.py +++ b/pytest_reportportal/service.py @@ -198,6 +198,8 @@ def __init__(self, agent_config: AgentConfig) -> None: self._launch_id = None self.agent_name = "pytest-reportportal" self.agent_version = get_package_version(self.agent_name) or "None" + self._retry_tracker: dict[str, dict[str, Any]] = {} + self._active_leaves: dict[str, dict[str, Any]] = {} self.ignored_attributes = [] self.parent_item_id = None self.rp = None @@ -457,12 +459,31 @@ def _merge_leaf_types(self, test_tree: dict[str, Any], leaf_types: set, separato self._merge_leaf_types(child_leaf, leaf_types, separator) def _merge_dirs(self, test_tree: dict[str, Any]) -> None: + """Merge directory and file leaves using configured separator. + + :param test_tree: Test tree structure to merge + """ self._merge_leaf_types(test_tree, {LeafType.DIR, LeafType.FILE}, self._config.rp_dir_path_separator) - def _merge_code_with_separator(self, test_tree: dict[str, Any], separator: str) -> None: - self._merge_leaf_types(test_tree, {LeafType.CODE, LeafType.FILE, LeafType.DIR, LeafType.SUITE}, separator) + def _merge_code_with_separator(self, test_tree: dict[str, Any], separator: str, is_bdd: bool = False) -> None: + """Merge code and suite leaves, respecting hierarchy flags. + + :param test_tree: Test tree structure to merge + :param separator: Separator to use when merging names + :param is_bdd: If True, always merge FILE for BDD scenarios. Otherwise respect rp_hierarchy_test_file + """ + types_to_merge = {LeafType.CODE, LeafType.SUITE} + if is_bdd or not self._config.rp_hierarchy_test_file: + types_to_merge.add(LeafType.FILE) + if not self._config.rp_hierarchy_dirs: + types_to_merge.add(LeafType.DIR) + self._merge_leaf_types(test_tree, types_to_merge, separator) def _merge_code(self, test_tree: dict[str, Any]) -> None: + """Merge code and suite leaves using double colon separator. + + :param test_tree: Test tree structure to merge + """ self._merge_code_with_separator(test_tree, "::") def _build_item_paths(self, leaf: dict[str, Any], path: list[dict[str, Any]]) -> None: @@ -874,6 +895,8 @@ def _build_start_step_rq(self, leaf: dict[str, Any]) -> dict[str, Any]: "parameters": leaf.get("parameters", None), "parent_item_id": self._lock(leaf["parent"], lambda p: p["item_id"]), "test_case_id": leaf.get("test_case_id", None), + "retry": leaf.get("retry", False), + "retry_of": leaf.get("retry_of", None), } return payload @@ -906,6 +929,13 @@ def start_pytest_item(self, test_item: Optional[Item] = None): return self._create_suite_path(test_item) + + item_key = self._get_item_key(test_item) + if item_key in self._active_leaves: + current_leaf = self._active_leaves[item_key] + if current_leaf["item_id"] is not None: + return + current_leaf = self._tree_path[test_item][-1] self._process_metadata_item_start(current_leaf) item_id = self._start_step(self._build_start_step_rq(current_leaf)) @@ -925,7 +955,12 @@ def process_results(self, test_item: Item, report): if PYTEST_BDD and _is_pytest_bdd_scenario(test_item.location[0]): return - leaf = self._tree_path[test_item][-1] + item_key = self._get_item_key(test_item) + if item_key in self._active_leaves: + leaf = self._active_leaves[item_key] + else: + leaf = self._tree_path[test_item][-1] + # Defining test result if report.when == "setup": leaf["status"] = "PASSED" @@ -951,6 +986,8 @@ def _build_finish_step_rq(self, leaf: dict[str, Any]) -> dict[str, Any]: "status": status, "issue": issue, "item_id": leaf["item_id"], + "retry": leaf.get("retry", False), + "retry_of": leaf.get("retry_of", None), } return payload @@ -992,6 +1029,72 @@ def _finish_parents(self, leaf: dict[str, Any]) -> None: self._lock(leaf["parent"], lambda p: self._proceed_suite_finish(p)) self._finish_parents(leaf["parent"]) + def _get_item_key(self, test_item: Item) -> str: + """Get unique key for tracking an item across retries.""" + return str(id(test_item)) + + def _detect_retry_attempt(self, test_item: Item) -> int: + """Detect current retry attempt number from execution_count.""" + return getattr(test_item, 'execution_count', 1) + + def handle_retry_transition(self, test_item: Item, report) -> None: + """Detect and handle retry transitions when test is retried.""" + if report.when not in ("setup", "call"): + return + + current_execution = self._detect_retry_attempt(test_item) + item_key = self._get_item_key(test_item) + + if item_key not in self._retry_tracker: + self._retry_tracker[item_key] = { + "last_reported_execution_count": 0, + "attempts": [] + } + + tracker = self._retry_tracker[item_key] + + if current_execution > tracker["last_reported_execution_count"]: + if tracker["last_reported_execution_count"] == 0 and current_execution == 1: + tree_path_leaf = self._tree_path[test_item][-1] + self._active_leaves[item_key] = tree_path_leaf + tracker["attempts"].append({ + "execution_count": 1, + "item_id": tree_path_leaf.get("item_id") + }) + tracker["last_reported_execution_count"] = 1 + elif current_execution > 1: + if item_key in self._active_leaves: + prev_leaf = self._active_leaves[item_key] + self._process_metadata_item_finish(prev_leaf) + self._finish_step(self._build_finish_step_rq(prev_leaf)) + prev_leaf["exec"] = ExecStatus.FINISHED + + tree_path_leaf = self._tree_path[test_item][-1] + retry_leaf = { + **tree_path_leaf, + "item_id": None, + "exec": ExecStatus.CREATED, + "retry": True, + "retry_of": tracker["attempts"][-1]["item_id"] if tracker["attempts"] else None, + } + + self._active_leaves[item_key] = retry_leaf + self._process_metadata_item_start(retry_leaf) + item_id = self._start_step(self._build_start_step_rq(retry_leaf)) + retry_leaf["item_id"] = item_id + retry_leaf["exec"] = ExecStatus.IN_PROGRESS + + tracker["attempts"].append({ + "execution_count": current_execution, + "item_id": item_id + }) + tracker["last_reported_execution_count"] = current_execution + + def cleanup_retry_state(self) -> None: + """Clean up retry tracking state after session ends.""" + self._retry_tracker.clear() + self._active_leaves.clear() + @check_rp_enabled def finish_pytest_item(self, test_item: Optional[Item] = None) -> None: """Finish pytest_item. @@ -1002,7 +1105,12 @@ def finish_pytest_item(self, test_item: Optional[Item] = None) -> None: if test_item is None: return - leaf = self._tree_path[test_item][-1] + item_key = self._get_item_key(test_item) + if item_key in self._active_leaves: + leaf = self._active_leaves[item_key] + else: + leaf = self._tree_path[test_item][-1] + self._process_metadata_item_finish(leaf) if PYTEST_BDD and _is_pytest_bdd_scenario(test_item.location[0]): @@ -1011,7 +1119,13 @@ def finish_pytest_item(self, test_item: Optional[Item] = None) -> None: self._finish_step(self._build_finish_step_rq(leaf)) leaf["exec"] = ExecStatus.FINISHED - self._finish_parents(leaf) + + current_execution = self._detect_retry_attempt(test_item) + tracker = self._retry_tracker.get(item_key, {}) + last_reported = tracker.get("last_reported_execution_count", 0) + + if current_execution == last_reported or current_execution == 1: + self._finish_parents(leaf) def _get_items(self, exec_status) -> list[Item]: return [k for k, v in self._tree_path.items() if v[-1]["exec"] == exec_status] @@ -1084,7 +1198,11 @@ def post_log( LOGGER.warning( "Incorrect loglevel = %s. Force set to INFO. " "Available levels: %s.", log_level, KNOWN_LOG_LEVELS ) - item_id = self._tree_path[test_item][-1]["item_id"] + item_key = self._get_item_key(test_item) + if item_key in self._active_leaves: + item_id = self._active_leaves[item_key]["item_id"] + else: + item_id = self._tree_path[test_item][-1]["item_id"] if PYTEST_BDD: if not item_id: # Check if we are actually a BDD scenario @@ -1185,7 +1303,7 @@ def start_bdd_scenario(self, feature: Feature, scenario: Scenario) -> None: self._generate_names(root_leaf) if not self._config.rp_hierarchy_code: try: - self._merge_code_with_separator(root_leaf, " - ") + self._merge_code_with_separator(root_leaf, " - ", is_bdd=True) except Exception as e: LOGGER.exception(e) self._build_item_paths(root_leaf, []) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index d57be797..3d59848e 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -35,6 +35,7 @@ + [["examples/hierarchy/inner/test_inner_simple.py"]] * 7 + [["examples/hierarchy/test_in_class_in_class.py"]] + [["examples/test_simple.py"]] * 2 + + [["examples/hierarchy/inner/test_inner_simple.py"]] ) # noinspection PyTypeChecker @@ -65,6 +66,10 @@ dict(**utils.DEFAULT_VARIABLES), dict({"rp_hierarchy_test_file": False}, **utils.DEFAULT_VARIABLES), dict({"rp_hierarchy_test_file": False, "rp_hierarchy_dirs_level": 1}, **utils.DEFAULT_VARIABLES), + dict( + {"rp_hierarchy_dirs": True, "rp_hierarchy_test_file": True, "rp_hierarchy_code": False}, + **utils.DEFAULT_VARIABLES, + ), ] HIERARCHY_TEST_EXPECTED_ITEMS = [ @@ -271,6 +276,13 @@ ], [{"name": "examples::test_simple", "item_type": "STEP", "parent_item_id": lambda x: x is None}], [{"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x is None}], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "hierarchy", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + {"name": "inner", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("hierarchy")}, + {"name": "test_inner_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("inner")}, + {"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("test_inner_simple.py")}, + ], ] HIERARCHY_TEST_PARAMETERS = [ diff --git a/tests/integration/test_retry_rerunfailures.py b/tests/integration/test_retry_rerunfailures.py new file mode 100644 index 00000000..d0f80bea --- /dev/null +++ b/tests/integration/test_retry_rerunfailures.py @@ -0,0 +1,30 @@ +"""Integration tests for pytest-rerunfailures support.""" + +import pytest + + +@pytest.mark.flaky(reruns=2) +def test_eventual_pass_with_retries(): + """Test that passes after retries are properly reported.""" + if not hasattr(test_eventual_pass_with_retries, 'attempts'): + test_eventual_pass_with_retries.attempts = 0 + test_eventual_pass_with_retries.attempts += 1 + + # Passes on third attempt + assert test_eventual_pass_with_retries.attempts >= 3 + + +def test_without_retries(): + """Test that passes without retries.""" + assert True + + +@pytest.mark.flaky(reruns=1) +def test_passes_on_second_attempt(): + """Test that passes on second attempt.""" + if not hasattr(test_passes_on_second_attempt, 'count'): + test_passes_on_second_attempt.count = 0 + test_passes_on_second_attempt.count += 1 + + # Fails once, passes on second attempt + assert test_passes_on_second_attempt.count >= 2 diff --git a/tests/unit/test_retry_support.py b/tests/unit/test_retry_support.py new file mode 100644 index 00000000..f5c91ce6 --- /dev/null +++ b/tests/unit/test_retry_support.py @@ -0,0 +1,293 @@ +"""Tests for pytest-rerunfailures retry support.""" + +import pytest +from unittest import mock + +from pytest_reportportal.service import PyTestService + + +class TestRetryDetection: + """Tests for retry detection and state tracking.""" + + def test_detect_retry_returns_execution_count(self): + """Verify execution_count is returned.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + test_item.execution_count = 1 + + result = service._detect_retry_attempt(test_item) + assert result == 1 + + def test_detect_retry_second_execution(self): + """Verify second execution is detected.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + test_item.execution_count = 2 + + result = service._detect_retry_attempt(test_item) + assert result == 2 + + def test_detect_retry_defaults_when_missing(self): + """Verify defaults to 1 if attribute missing.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock(spec=[]) + + result = service._detect_retry_attempt(test_item) + assert result == 1 + + def test_get_item_key_from_object_id(self): + """Verify item key is based on object id.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + key = service._get_item_key(test_item) + + assert key == str(id(test_item)) + + def test_get_item_key_is_consistent(self): + """Verify same object returns same key.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + key1 = service._get_item_key(test_item) + key2 = service._get_item_key(test_item) + + assert key1 == key2 + + def test_retry_tracker_tracks_attempts(self): + """Verify retry tracker records execution counts.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + test_item.execution_count = 1 + + key = service._get_item_key(test_item) + service._retry_tracker[key] = {"last_reported_execution_count": 1} + + # Simulate second execution + test_item.execution_count = 2 + current = service._detect_retry_attempt(test_item) + last = service._retry_tracker[key]["last_reported_execution_count"] + + assert current > last + + +class TestRetryTransition: + """Tests for retry transition handling.""" + + def test_first_execution_registers_without_duplicate(self): + """Verify first execution registers tree leaf without creating duplicate.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + test_item.execution_count = 1 + test_item.location = ("test_file.py",) + + key = service._get_item_key(test_item) + tree_leaf = { + "name": "test_item", + "item_id": "item-1", + "exec": "IN_PROGRESS" + } + service._tree_path[test_item] = [tree_leaf] + + report = mock.MagicMock() + report.when = "call" + + service.handle_retry_transition(test_item, report) + + tracker = service._retry_tracker[key] + assert tracker["last_reported_execution_count"] == 1 + assert len(tracker["attempts"]) == 1 + assert tracker["attempts"][0]["execution_count"] == 1 + + def test_ignores_non_call_and_setup_phases(self): + """Verify teardown phases are ignored.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + test_item.execution_count = 1 + + key = service._get_item_key(test_item) + + report = mock.MagicMock() + report.when = "teardown" + + service.handle_retry_transition(test_item, report) + + assert key not in service._retry_tracker + + def test_setup_phase_is_processed(self): + """Verify setup phase is processed like call phase.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + test_item.execution_count = 1 + + key = service._get_item_key(test_item) + tree_leaf = { + "name": "test_item", + "item_id": "item-1", + "exec": "IN_PROGRESS" + } + service._tree_path[test_item] = [tree_leaf] + + report = mock.MagicMock() + report.when = "setup" + + service.handle_retry_transition(test_item, report) + + tracker = service._retry_tracker[key] + assert tracker["last_reported_execution_count"] == 1 + + +class TestRetryMetadata: + """Tests for retry metadata in payloads.""" + + def test_finish_payload_includes_retry_fields(self): + """Verify finish payload has retry metadata.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + leaf = { + "name": "test_retry", + "description": "Test", + "status": "PASSED", + "item_id": "item-123", + "retry": True, + "retry_of": "prev-item" + } + + payload = service._build_finish_step_rq(leaf) + + assert payload.get("retry") == True + assert payload.get("retry_of") == "prev-item" + + def test_finish_payload_defaults_retry_false(self): + """Verify retry defaults to false.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + leaf = { + "name": "test_normal", + "description": "Test", + "status": "PASSED", + "item_id": "item-456" + } + + payload = service._build_finish_step_rq(leaf) + + assert payload.get("retry") == False + assert payload.get("retry_of") is None + + +class TestRetryStateManagement: + """Tests for retry state tracking and cleanup.""" + + def test_state_initialized(self): + """Verify state dicts are initialized.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + assert hasattr(service, '_retry_tracker') + assert hasattr(service, '_active_leaves') + assert isinstance(service._retry_tracker, dict) + assert isinstance(service._active_leaves, dict) + + def test_cleanup_clears_all_state(self): + """Verify cleanup empties both dicts.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + service._retry_tracker["key1"] = {"data": "value"} + service._active_leaves["key2"] = {"leaf": "data"} + + service.cleanup_retry_state() + + assert len(service._retry_tracker) == 0 + assert len(service._active_leaves) == 0 + + def test_active_leaves_updated_for_new_attempt(self): + """Verify active leaf is replaced for new attempt.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + key = service._get_item_key(test_item) + + leaf1 = {"item_id": "attempt-1", "execution": 1} + service._active_leaves[key] = leaf1 + + assert service._active_leaves[key]["item_id"] == "attempt-1" + + leaf2 = {"item_id": "attempt-2", "execution": 2} + service._active_leaves[key] = leaf2 + + assert service._active_leaves[key]["item_id"] == "attempt-2" + + def test_post_log_uses_active_leaf(self): + """Verify post_log routes to active leaf when present.""" + from pytest_reportportal.config import AgentConfig + + config = mock.MagicMock(spec=AgentConfig) + service = PyTestService(config) + + test_item = mock.MagicMock() + key = service._get_item_key(test_item) + + # Setup mocks + service.rp = mock.MagicMock() + + active_leaf = {"item_id": "active-item-id"} + service._active_leaves[key] = active_leaf + + tree_leaf = {"item_id": "tree-item-id"} + service._tree_path[test_item] = [tree_leaf] + + service.post_log(test_item, "test message", "INFO") + + # Verify rp.log was called + assert service.rp.log.called + call_args = service.rp.log.call_args + # The item_id should come from active_leaf + assert call_args[1]["item_id"] == "active-item-id"