From df463e38de7853ef9329629acf22dc339fa659af Mon Sep 17 00:00:00 2001 From: harbinresearcher Date: Fri, 16 Oct 2026 13:43:36 +0800 Subject: [PATCH] fix: honor fail-on-flaky with xdist workers --- changes/386.bugfix.rst | 1 + src/pytest_rerunfailures.py | 9 +++++- tests/test_pytest_rerunfailures.py | 47 ++++++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 changes/386.bugfix.rst diff --git a/changes/386.bugfix.rst b/changes/386.bugfix.rst new file mode 100644 index 0000000..cb201da --- /dev/null +++ b/changes/386.bugfix.rst @@ -0,0 +1 @@ +Honor ``--fail-on-flaky`` when tests are rerun with pytest-xdist, while preserving ordinary failure exit codes. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index db3974a..b42474c 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -780,7 +780,10 @@ def pytest_sessionstart(self, session): self.session = session def pytest_runtest_logreport(self, report): - """Clean up subtest reports superseded by a worker rerun.""" + """Track worker reruns and clean up superseded subtest reports.""" + if report.outcome == "rerun": + self.session._rerunfailures_has_rerun = True + worker_id = getattr(report, "worker_id", None) item_index = getattr(report, "item_index", None) # xdist adds worker_id and item_index when forwarding a report to the @@ -1446,6 +1449,10 @@ def pytest_sessionfinish(session, exitstatus): return if session.config.option.fail_on_flaky: + if getattr(session, "_rerunfailures_has_rerun", False): + session.exitstatus = 7 + return + for item in session.items: if not hasattr(item, "execution_count"): # no rerun requested diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 2094448..6405d59 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -316,28 +316,35 @@ def test_pass(): def test_run_with_fail_on_flaky_fails_with_custom_error_code_after_pass_on_rerun( testdir, + fail_on_flaky_xdist_args, ): testdir.makepyfile( f""" def test_pass(): {temporary_failure()}""" ) - result = testdir.runpytest("--reruns", "1", "--fail-on-flaky") + result = testdir.runpytest( + "--reruns", "1", "--fail-on-flaky", *fail_on_flaky_xdist_args + ) assert_outcomes(result, passed=1, rerun=1) assert result.ret == 7 def test_run_fails_with_code_1_after_consistent_test_failure_even_with_fail_on_flaky( testdir, + fail_on_flaky_xdist_args, ): testdir.makepyfile("def test_fail(): assert False") - result = testdir.runpytest("--reruns", "1", "--fail-on-flaky") + result = testdir.runpytest( + "--reruns", "1", "--fail-on-flaky", *fail_on_flaky_xdist_args + ) assert_outcomes(result, passed=0, failed=1, rerun=1) assert result.ret == 1 def test_run_mark_and_fail_on_flaky_fails_with_custom_error_code_after_pass_on_rerun( testdir, + fail_on_flaky_xdist_args, ): testdir.makepyfile(f""" import pytest @@ -346,13 +353,14 @@ def test_run_mark_and_fail_on_flaky_fails_with_custom_error_code_after_pass_on_r def test_fail(): {temporary_failure()} """) - result = testdir.runpytest("--fail-on-flaky") + result = testdir.runpytest("--fail-on-flaky", *fail_on_flaky_xdist_args) assert_outcomes(result, passed=1, rerun=1) assert result.ret == 7 def test_run_fails_with_code_1_after_test_failure_with_fail_on_flaky_and_mark( testdir, + fail_on_flaky_xdist_args, ): testdir.makepyfile(""" import pytest @@ -361,13 +369,14 @@ def test_run_fails_with_code_1_after_test_failure_with_fail_on_flaky_and_mark( def test_fail(): assert False """) - result = testdir.runpytest("--fail-on-flaky") + result = testdir.runpytest("--fail-on-flaky", *fail_on_flaky_xdist_args) assert_outcomes(result, passed=0, failed=1, rerun=2) assert result.ret == 1 def test_run_with_mark_and_fail_on_flaky_succeeds_if_all_tests_pass_without_reruns( testdir, + fail_on_flaky_xdist_args, ): testdir.makepyfile(""" import pytest @@ -379,20 +388,46 @@ def test_marked_pass(): def test_unmarked_pass(): assert True """) - result = testdir.runpytest("--fail-on-flaky") + result = testdir.runpytest("--fail-on-flaky", *fail_on_flaky_xdist_args) assert_outcomes(result, passed=2, rerun=0) assert result.ret == pytest.ExitCode.OK def test_run_with_fail_on_flaky_succeeds_if_all_tests_pass_without_reruns( testdir, + fail_on_flaky_xdist_args, ): testdir.makepyfile("def test_pass(): assert True") - result = testdir.runpytest("--reruns", "1", "--fail-on-flaky") + result = testdir.runpytest( + "--reruns", "1", "--fail-on-flaky", *fail_on_flaky_xdist_args + ) assert_outcomes(result, passed=1, rerun=0) assert result.ret == pytest.ExitCode.OK +@pytest.fixture(params=[0, 1, 2]) +def fail_on_flaky_xdist_args(request): + if request.param and not has_xdist: + pytest.skip("requires xdist") + return ["-n", str(request.param)] if request.param else [] + + +def test_fail_on_flaky_after_skip_on_rerun(testdir, fail_on_flaky_xdist_args): + testdir.makepyfile(""" + import pytest + + def test_flaky(request): + if request.node.execution_count == 1: + assert False + pytest.skip("skipped on rerun") + """) + result = testdir.runpytest( + "--reruns", "1", "--fail-on-flaky", *fail_on_flaky_xdist_args + ) + assert_outcomes(result, passed=0, skipped=1, rerun=1) + assert result.ret == 7 + + @pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") def test_rerun_passes_after_temporary_test_crash(testdir): # note: we need two tests because there is a bug where xdist