diff --git a/CHANGELOG.md b/CHANGELOG.md index d41971349..d0256239b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +Version 0.69.1 +------------- + +**Bugfixes** +- Support `Expected Failure` and `Skip Message` test node types for XCTest results which were added in `xcresulttool` test-results schema 0.2.0 (default on Xcode 27). Fixes actions + - `xcode-project run-tests`, + - `xcode-project test-summary`, + - `xcode-project junit-test-results`. + Also restore skip messages in JUnit output when Apple emits `Skip Message` children instead of `Failure Message` with result `Skipped`. +- Emit `XCTExpectFailure` reason text in JUnit `` for test cases with expected failures, including failed test cases that also have expected failures. + Version 0.69.0 ------------- diff --git a/pyproject.toml b/pyproject.toml index c6fa3143c..5adfb9a62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codemagic-cli-tools" -version = "0.69.0" +version = "0.69.1" description = "CLI tools used in Codemagic builds" authors = [{ name = "Priit Lätt", email = "priit@nevercode.io" }] requires-python = ">=3.8,<4" diff --git a/src/codemagic/__version__.py b/src/codemagic/__version__.py index e49d2a2c7..05e914a83 100644 --- a/src/codemagic/__version__.py +++ b/src/codemagic/__version__.py @@ -1,5 +1,5 @@ __title__ = "codemagic-cli-tools" __description__ = "CLI tools used in Codemagic builds" -__version__ = "0.69.0.dev" +__version__ = "0.69.1.dev" __url__ = "https://github.com/codemagic-ci-cd/cli-tools" __licence__ = "GNU General Public License v3.0" diff --git a/src/codemagic/models/junit/definitions.py b/src/codemagic/models/junit/definitions.py index 48bba1327..c12848a03 100644 --- a/src/codemagic/models/junit/definitions.py +++ b/src/codemagic/models/junit/definitions.py @@ -159,6 +159,7 @@ class TestCase: error: Optional[Error] = None failure: Optional[Failure] = None skipped: Optional[Skipped] = None + system_out: Optional[str] = None __test__ = False # Tell Pytest not to collect this class as test @@ -190,6 +191,9 @@ def as_xml(self) -> Element: element.append(self.failure.as_xml()) if self.skipped: element.append(self.skipped.as_xml()) + if self.system_out: + system_out = SubElement(element, "system-out") + system_out.text = self.system_out return element diff --git a/src/codemagic/models/xctests/converter.py b/src/codemagic/models/xctests/converter.py index df6284696..3a70397ff 100644 --- a/src/codemagic/models/xctests/converter.py +++ b/src/codemagic/models/xctests/converter.py @@ -182,12 +182,17 @@ def convert(self) -> TestSuites: class Xcode16XcResultConverter(XcResultConverter): @classmethod - def _iter_nodes(cls, root_node: XcTestNode, node_type: XcTestNodeType) -> Iterator[XcTestNode]: + def _iter_nodes( + cls, + root_node: XcTestNode, + node_type: XcTestNodeType, + skip_subtree_node_type: Optional[XcTestNodeType] = None, + ) -> Iterator[XcTestNode]: if root_node.node_type is node_type: yield root_node - else: + elif root_node.node_type is not skip_subtree_node_type: for child in root_node.children: - yield from cls._iter_nodes(child, node_type) + yield from cls._iter_nodes(child, node_type, skip_subtree_node_type) @classmethod def _get_run_destination(cls, root_node: XcTestNode) -> Optional[XcDevice]: @@ -225,8 +230,15 @@ def _get_test_case_error(cls, xc_test_case: XcTestNode) -> Optional[Error]: if xc_test_case.result is not XcTestResult.FAILED: return None - failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE) - failure_messages = [node.name for node in failure_messages_nodes if node.name] + failure_messages_nodes = cls._iter_nodes( + xc_test_case, + XcTestNodeType.FAILURE_MESSAGE, + skip_subtree_node_type=XcTestNodeType.EXPECTED_FAILURE, + ) + unexpected_failure_message_nodes = ( + node for node in failure_messages_nodes if node.result is not XcTestResult.EXPECTED_FAILURE + ) + failure_messages = [node.name for node in unexpected_failure_message_nodes if node.name] return Error( message=failure_messages[0] if failure_messages else "", type="Error" if any("caught error" in m for m in failure_messages) else "Failure", @@ -238,11 +250,32 @@ def _get_test_case_skipped(cls, xc_test_case: XcTestNode) -> Optional[Skipped]: if xc_test_case.result is not XcTestResult.SKIPPED: return None + # Schema 0.1.0 / Xcode 16: skip reason is a Failure Message child with result Skipped. failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE) skipped_message_nodes = (node for node in failure_messages_nodes if node.result is XcTestResult.SKIPPED) skipped_messages = [node.name for node in skipped_message_nodes if node.name] - return Skipped(message="\n".join(skipped_messages)) + # Schema 0.2.0+ / Xcode 27: skip reason is a Skip Message child. + skip_message_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.SKIP_MESSAGE) + skipped_messages.extend(node.name for node in skip_message_nodes if node.name) + + unique_skipped_messages = dict.fromkeys(skipped_messages) + return Skipped(message="\n".join(unique_skipped_messages)) + + @classmethod + def _get_test_case_system_out(cls, xc_test_case: XcTestNode) -> Optional[str]: + expected_failure_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.EXPECTED_FAILURE) + reasons = [node.name for node in expected_failure_nodes if node.name] + + failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE) + expected_failure_message_nodes = ( + node for node in failure_messages_nodes if node.result is XcTestResult.EXPECTED_FAILURE + ) + reasons.extend(node.name for node in expected_failure_message_nodes if node.name) + + if not reasons: + return None + return "\n".join(dict.fromkeys(reasons)) @classmethod def parse_xcresult_test_node_duration_value(cls, xc_duration: str) -> float: @@ -292,6 +325,7 @@ def _get_test_case(cls, xc_test_case: XcTestNode, xc_test_suite: XcTestNode) -> time=cls._get_test_node_duration(xc_test_case), status=xc_test_case.result.value if xc_test_case.result else None, skipped=cls._get_test_case_skipped(xc_test_case), + system_out=cls._get_test_case_system_out(xc_test_case), ) @classmethod diff --git a/src/codemagic/models/xctests/xcresult/xcresult.py b/src/codemagic/models/xctests/xcresult/xcresult.py index e24f07498..84ef92fd8 100644 --- a/src/codemagic/models/xctests/xcresult/xcresult.py +++ b/src/codemagic/models/xctests/xcresult/xcresult.py @@ -40,6 +40,8 @@ class XcTestNodeType(str, enum.Enum): EXPRESSION = "Expression" TEST_VALUE = "Test Value" RUNTIME_WARNING = "Runtime Warning" + SKIP_MESSAGE = "Skip Message" + EXPECTED_FAILURE = "Expected Failure" @dataclasses.dataclass diff --git a/tests/models/junit/test_xml_generation.py b/tests/models/junit/test_xml_generation.py index d1b0ba795..14cb13a01 100644 --- a/tests/models/junit/test_xml_generation.py +++ b/tests/models/junit/test_xml_generation.py @@ -351,3 +351,23 @@ def test_xml(temp_dir, _testsuites, expected_xml_path): generated_xml = ElementTree.parse(xml_path) expected_xml = ElementTree.parse(expected_xml_path) _assert_elements_are_equal(generated_xml.getroot(), expected_xml.getroot()) + + +def test_testcase_emits_system_out(): + testcase = TestCase( + classname="ExpectedFailureProbeTests", + name="testExpectedFailure()", + status="Expected Failure", + time=0.001, + system_out="known bad assertion", + ) + xml = testcase.as_xml() + system_out_elements = [child for child in xml if child.tag == "system-out"] + assert len(system_out_elements) == 1 + assert system_out_elements[0].text == "known bad assertion" + + +def test_testcase_omits_system_out_when_unset(): + testcase = TestCase(classname="Suite", name="testPass()", status="Success") + xml = testcase.as_xml() + assert all(child.tag != "system-out" for child in xml) diff --git a/tests/models/xctests/converter/test_xcode_16_converter.py b/tests/models/xctests/converter/test_xcode_16_converter.py index 82da74638..5c2367c0d 100644 --- a/tests/models/xctests/converter/test_xcode_16_converter.py +++ b/tests/models/xctests/converter/test_xcode_16_converter.py @@ -12,6 +12,9 @@ from codemagic.models.junit import TestSuites from codemagic.models.xctests import XcResultTool from codemagic.models.xctests.converter import Xcode16XcResultConverter +from codemagic.models.xctests.xcresult import XcTestNode +from codemagic.models.xctests.xcresult import XcTestNodeType +from codemagic.models.xctests.xcresult import XcTestResult @pytest.fixture @@ -151,3 +154,151 @@ def test_converter(mock_datetime, expected_properties): def test_parse_xcresult_test_node_duration_value(duration, expected_value): value = Xcode16XcResultConverter.parse_xcresult_test_node_duration_value(duration) assert value == pytest.approx(expected_value) + + +def test_get_test_case_skipped_from_skip_message_node(): + xc_test_case = XcTestNode( + name="testSkip()", + node_type=XcTestNodeType.TEST_CASE, + result=XcTestResult.SKIPPED, + children=[ + XcTestNode( + name="Test skipped - skipping on purpose", + node_type=XcTestNodeType.SKIP_MESSAGE, + ), + ], + ) + assert Xcode16XcResultConverter._get_test_case_skipped(xc_test_case) == Skipped( + message="Test skipped - skipping on purpose", + ) + + +def test_get_test_case_skipped_deduplicates_identical_messages(): + message = "Test skipped - skipping on purpose" + xc_test_case = XcTestNode( + name="testSkip()", + node_type=XcTestNodeType.TEST_CASE, + result=XcTestResult.SKIPPED, + children=[ + XcTestNode( + name=message, + node_type=XcTestNodeType.FAILURE_MESSAGE, + result=XcTestResult.SKIPPED, + ), + XcTestNode( + name=message, + node_type=XcTestNodeType.SKIP_MESSAGE, + ), + ], + ) + assert Xcode16XcResultConverter._get_test_case_skipped(xc_test_case) == Skipped(message=message) + + +def test_get_test_case_error_excludes_expected_failures(): + real_failure = 'XCTAssertEqual failed: ("3") is not equal to ("4") - should be real failure' + xc_test_case = XcTestNode( + name="testExpectedFailureThenRealFailure()", + node_type=XcTestNodeType.TEST_CASE, + result=XcTestResult.FAILED, + children=[ + XcTestNode( + name=real_failure, + node_type=XcTestNodeType.FAILURE_MESSAGE, + ), + XcTestNode( + name="known bad assertion", + node_type=XcTestNodeType.EXPECTED_FAILURE, + children=[ + XcTestNode( + name="nested expected failure message", + node_type=XcTestNodeType.FAILURE_MESSAGE, + ), + ], + ), + XcTestNode( + name="expected failure via result", + node_type=XcTestNodeType.FAILURE_MESSAGE, + result=XcTestResult.EXPECTED_FAILURE, + ), + ], + ) + assert Xcode16XcResultConverter._get_test_case_error(xc_test_case) == Error( + message=real_failure, + type="Failure", + ) + + +def test_get_test_case_system_out_from_expected_failure_children(): + xc_test_case = XcTestNode( + name="testExpectedFailure()", + node_type=XcTestNodeType.TEST_CASE, + node_identifier="ExpectedFailureProbeTests/testExpectedFailure()", + result=XcTestResult.EXPECTED_FAILURE, + children=[ + XcTestNode( + name="known bad assertion", + node_type=XcTestNodeType.EXPECTED_FAILURE, + ), + ], + ) + xc_test_suite = XcTestNode(name="ExpectedFailureProbeTests", node_type=XcTestNodeType.TEST_SUITE) + + test_case = Xcode16XcResultConverter._get_test_case(xc_test_case, xc_test_suite) + + assert test_case == TestCase( + name="testExpectedFailure()", + classname="ExpectedFailureProbeTests", + error=None, + time=0.0, + status="Expected Failure", + skipped=None, + system_out="known bad assertion", + ) + + +def test_get_test_case_system_out_absent_for_success(): + xc_test_case = XcTestNode( + name="testPass()", + node_type=XcTestNodeType.TEST_CASE, + result=XcTestResult.PASSED, + ) + assert Xcode16XcResultConverter._get_test_case_system_out(xc_test_case) is None + + +def test_get_test_case_system_out_on_failed_case_with_expected_failures(): + real_failure = 'XCTAssertEqual failed: ("3") is not equal to ("4") - should be real failure' + xc_test_case = XcTestNode( + name="testExpectedFailureThenRealFailure()", + node_type=XcTestNodeType.TEST_CASE, + node_identifier="ExpectedFailureProbeTests/testExpectedFailureThenRealFailure()", + result=XcTestResult.FAILED, + children=[ + XcTestNode( + name=real_failure, + node_type=XcTestNodeType.FAILURE_MESSAGE, + ), + XcTestNode( + name="known bad assertion", + node_type=XcTestNodeType.EXPECTED_FAILURE, + children=[ + XcTestNode( + name="nested expected failure message", + node_type=XcTestNodeType.FAILURE_MESSAGE, + ), + ], + ), + XcTestNode( + name="expected failure via result", + node_type=XcTestNodeType.FAILURE_MESSAGE, + result=XcTestResult.EXPECTED_FAILURE, + ), + ], + ) + xc_test_suite = XcTestNode(name="ExpectedFailureProbeTests", node_type=XcTestNodeType.TEST_SUITE) + + test_case = Xcode16XcResultConverter._get_test_case(xc_test_case, xc_test_suite) + + assert test_case.error == Error(message=real_failure, type="Failure") + assert test_case.system_out == "known bad assertion\nexpected failure via result" + assert test_case.status == "Failed" + assert test_case.skipped is None diff --git a/tests/models/xctests/xcresult/test_xcresult.py b/tests/models/xctests/xcresult/test_xcresult.py index 6137793cc..f3b6d7487 100644 --- a/tests/models/xctests/xcresult/test_xcresult.py +++ b/tests/models/xctests/xcresult/test_xcresult.py @@ -10,6 +10,18 @@ from codemagic.models.xctests.xcresult.xcresult import XcTestStatistic +def test_parse_expected_failure_node_type(): + node = XcTestNode.from_dict({"nodeType": "Expected Failure", "name": "known bad assertion"}) + assert node.node_type is XcTestNodeType.EXPECTED_FAILURE + assert node.name == "known bad assertion" + + +def test_parse_skip_message_node_type(): + node = XcTestNode.from_dict({"nodeType": "Skip Message", "name": "Test skipped - skipping on purpose"}) + assert node.node_type is XcTestNodeType.SKIP_MESSAGE + assert node.name == "Test skipped - skipping on purpose" + + def test_load_test_results_summary(test_results_summary_dict): test_results_summary = XcSummary.from_dict(test_results_summary_dict) diff --git a/uv.lock b/uv.lock index 1dfb3c042..114adf13d 100644 --- a/uv.lock +++ b/uv.lock @@ -550,7 +550,7 @@ wheels = [ [[package]] name = "codemagic-cli-tools" -version = "0.69.0" +version = "0.69.1" source = { editable = "." } dependencies = [ { name = "cryptography" },