Fix: support xcresult node types

CHANGELOG.md

@@ -1,3 +1,14 @@
+Version 0.69.1
+-------------
+
+**Bugfixes**
+- Support `Expected Failure` and `Skip Message` test node types for XCTest results which were added in `xcresulttool` test-results schema 0.2.0 (default on Xcode 27). Fixes actions
+  - `xcode-project run-tests`,
+  - `xcode-project test-summary`,
+  - `xcode-project junit-test-results`.
+  Also restore skip messages in JUnit output when Apple emits `Skip Message` children instead of `Failure Message` with result `Skipped`.
+- Emit `XCTExpectFailure` reason text in JUnit `<system-out>` for test cases with expected failures, including failed test cases that also have expected failures.
+
 Version 0.69.0
 -------------
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codemagic-cli-tools"
-version = "0.69.0"
+version = "0.69.1"
 description = "CLI tools used in Codemagic builds"
 authors = [{ name = "Priit Lätt", email = "priit@nevercode.io" }]
 requires-python = ">=3.8,<4"

src/codemagic/__version__.py

@@ -1,5 +1,5 @@
 __title__ = "codemagic-cli-tools"
 __description__ = "CLI tools used in Codemagic builds"
-__version__ = "0.69.0.dev"
+__version__ = "0.69.1.dev"
 __url__ = "https://github.com/codemagic-ci-cd/cli-tools"
 __licence__ = "GNU General Public License v3.0"

src/codemagic/models/junit/definitions.py

@@ -159,6 +159,7 @@ class TestCase:
     error: Optional[Error] = None
     failure: Optional[Failure] = None
     skipped: Optional[Skipped] = None
+    system_out: Optional[str] = None
 
     __test__ = False  # Tell Pytest not to collect this class as test
 
@@ -190,6 +191,9 @@ def as_xml(self) -> Element:
             element.append(self.failure.as_xml())
         if self.skipped:
             element.append(self.skipped.as_xml())
+        if self.system_out:
+            system_out = SubElement(element, "system-out")
+            system_out.text = self.system_out
         return element
 
 

src/codemagic/models/xctests/converter.py

@@ -182,12 +182,17 @@ def convert(self) -> TestSuites:
 
 class Xcode16XcResultConverter(XcResultConverter):
     @classmethod
-    def _iter_nodes(cls, root_node: XcTestNode, node_type: XcTestNodeType) -> Iterator[XcTestNode]:
+    def _iter_nodes(
+        cls,
+        root_node: XcTestNode,
+        node_type: XcTestNodeType,
+        skip_subtree_node_type: Optional[XcTestNodeType] = None,
+    ) -> Iterator[XcTestNode]:
         if root_node.node_type is node_type:
             yield root_node
-        else:
+        elif root_node.node_type is not skip_subtree_node_type:
             for child in root_node.children:
-                yield from cls._iter_nodes(child, node_type)
+                yield from cls._iter_nodes(child, node_type, skip_subtree_node_type)
 
     @classmethod
     def _get_run_destination(cls, root_node: XcTestNode) -> Optional[XcDevice]:
@@ -225,8 +230,15 @@ def _get_test_case_error(cls, xc_test_case: XcTestNode) -> Optional[Error]:
         if xc_test_case.result is not XcTestResult.FAILED:
             return None
 
-        failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE)
-        failure_messages = [node.name for node in failure_messages_nodes if node.name]
+        failure_messages_nodes = cls._iter_nodes(
+            xc_test_case,
+            XcTestNodeType.FAILURE_MESSAGE,
+            skip_subtree_node_type=XcTestNodeType.EXPECTED_FAILURE,
+        )
+        unexpected_failure_message_nodes = (
+            node for node in failure_messages_nodes if node.result is not XcTestResult.EXPECTED_FAILURE
+        )
+        failure_messages = [node.name for node in unexpected_failure_message_nodes if node.name]
         return Error(
             message=failure_messages[0] if failure_messages else "",
             type="Error" if any("caught error" in m for m in failure_messages) else "Failure",
@@ -238,11 +250,32 @@ def _get_test_case_skipped(cls, xc_test_case: XcTestNode) -> Optional[Skipped]:
         if xc_test_case.result is not XcTestResult.SKIPPED:
             return None
 
+        # Schema 0.1.0 / Xcode 16: skip reason is a Failure Message child with result Skipped.
         failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE)
         skipped_message_nodes = (node for node in failure_messages_nodes if node.result is XcTestResult.SKIPPED)
         skipped_messages = [node.name for node in skipped_message_nodes if node.name]
 
-        return Skipped(message="\n".join(skipped_messages))
+        # Schema 0.2.0+ / Xcode 27: skip reason is a Skip Message child.
+        skip_message_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.SKIP_MESSAGE)
+        skipped_messages.extend(node.name for node in skip_message_nodes if node.name)
+
+        unique_skipped_messages = dict.fromkeys(skipped_messages)
+        return Skipped(message="\n".join(unique_skipped_messages))
+
+    @classmethod
+    def _get_test_case_system_out(cls, xc_test_case: XcTestNode) -> Optional[str]:
+        expected_failure_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.EXPECTED_FAILURE)
+        reasons = [node.name for node in expected_failure_nodes if node.name]
+
+        failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE)
+        expected_failure_message_nodes = (
+            node for node in failure_messages_nodes if node.result is XcTestResult.EXPECTED_FAILURE
+        )
+        reasons.extend(node.name for node in expected_failure_message_nodes if node.name)
+
+        if not reasons:
+            return None
+        return "\n".join(dict.fromkeys(reasons))
 
     @classmethod
     def parse_xcresult_test_node_duration_value(cls, xc_duration: str) -> float:
@@ -292,6 +325,7 @@ def _get_test_case(cls, xc_test_case: XcTestNode, xc_test_suite: XcTestNode) ->
             time=cls._get_test_node_duration(xc_test_case),
             status=xc_test_case.result.value if xc_test_case.result else None,
             skipped=cls._get_test_case_skipped(xc_test_case),
+            system_out=cls._get_test_case_system_out(xc_test_case),
         )
 
     @classmethod

src/codemagic/models/xctests/xcresult/xcresult.py

@@ -40,6 +40,8 @@ class XcTestNodeType(str, enum.Enum):
     EXPRESSION = "Expression"
     TEST_VALUE = "Test Value"
     RUNTIME_WARNING = "Runtime Warning"
+    SKIP_MESSAGE = "Skip Message"
+    EXPECTED_FAILURE = "Expected Failure"
 
 
 @dataclasses.dataclass

tests/models/junit/test_xml_generation.py

@@ -351,3 +351,23 @@ def test_xml(temp_dir, _testsuites, expected_xml_path):
     generated_xml = ElementTree.parse(xml_path)
     expected_xml = ElementTree.parse(expected_xml_path)
     _assert_elements_are_equal(generated_xml.getroot(), expected_xml.getroot())
+
+
+def test_testcase_emits_system_out():
+    testcase = TestCase(
+        classname="ExpectedFailureProbeTests",
+        name="testExpectedFailure()",
+        status="Expected Failure",
+        time=0.001,
+        system_out="known bad assertion",
+    )
+    xml = testcase.as_xml()
+    system_out_elements = [child for child in xml if child.tag == "system-out"]
+    assert len(system_out_elements) == 1
+    assert system_out_elements[0].text == "known bad assertion"
+
+
+def test_testcase_omits_system_out_when_unset():
+    testcase = TestCase(classname="Suite", name="testPass()", status="Success")
+    xml = testcase.as_xml()
+    assert all(child.tag != "system-out" for child in xml)

tests/models/xctests/converter/test_xcode_16_converter.py

@@ -12,6 +12,9 @@
 from codemagic.models.junit import TestSuites
 from codemagic.models.xctests import XcResultTool
 from codemagic.models.xctests.converter import Xcode16XcResultConverter
+from codemagic.models.xctests.xcresult import XcTestNode
+from codemagic.models.xctests.xcresult import XcTestNodeType
+from codemagic.models.xctests.xcresult import XcTestResult
 
 
 @pytest.fixture
@@ -151,3 +154,151 @@ def test_converter(mock_datetime, expected_properties):
 def test_parse_xcresult_test_node_duration_value(duration, expected_value):
     value = Xcode16XcResultConverter.parse_xcresult_test_node_duration_value(duration)
     assert value == pytest.approx(expected_value)
+
+
+def test_get_test_case_skipped_from_skip_message_node():
+    xc_test_case = XcTestNode(
+        name="testSkip()",
+        node_type=XcTestNodeType.TEST_CASE,
+        result=XcTestResult.SKIPPED,
+        children=[
+            XcTestNode(
+                name="Test skipped - skipping on purpose",
+                node_type=XcTestNodeType.SKIP_MESSAGE,
+            ),
+        ],
+    )
+    assert Xcode16XcResultConverter._get_test_case_skipped(xc_test_case) == Skipped(
+        message="Test skipped - skipping on purpose",
+    )
+
+
+def test_get_test_case_skipped_deduplicates_identical_messages():
+    message = "Test skipped - skipping on purpose"
+    xc_test_case = XcTestNode(
+        name="testSkip()",
+        node_type=XcTestNodeType.TEST_CASE,
+        result=XcTestResult.SKIPPED,
+        children=[
+            XcTestNode(
+                name=message,
+                node_type=XcTestNodeType.FAILURE_MESSAGE,
+                result=XcTestResult.SKIPPED,
+            ),
+            XcTestNode(
+                name=message,
+                node_type=XcTestNodeType.SKIP_MESSAGE,
+            ),
+        ],
+    )
+    assert Xcode16XcResultConverter._get_test_case_skipped(xc_test_case) == Skipped(message=message)
+
+
+def test_get_test_case_error_excludes_expected_failures():
+    real_failure = 'XCTAssertEqual failed: ("3") is not equal to ("4") - should be real failure'
+    xc_test_case = XcTestNode(
+        name="testExpectedFailureThenRealFailure()",
+        node_type=XcTestNodeType.TEST_CASE,
+        result=XcTestResult.FAILED,
+        children=[
+            XcTestNode(
+                name=real_failure,
+                node_type=XcTestNodeType.FAILURE_MESSAGE,
+            ),
+            XcTestNode(
+                name="known bad assertion",
+                node_type=XcTestNodeType.EXPECTED_FAILURE,
+                children=[
+                    XcTestNode(
+                        name="nested expected failure message",
+                        node_type=XcTestNodeType.FAILURE_MESSAGE,
+                    ),
+                ],
+            ),
+            XcTestNode(
+                name="expected failure via result",
+                node_type=XcTestNodeType.FAILURE_MESSAGE,
+                result=XcTestResult.EXPECTED_FAILURE,
+            ),
+        ],
+    )
+    assert Xcode16XcResultConverter._get_test_case_error(xc_test_case) == Error(
+        message=real_failure,
+        type="Failure",
+    )
+
+
+def test_get_test_case_system_out_from_expected_failure_children():
+    xc_test_case = XcTestNode(
+        name="testExpectedFailure()",
+        node_type=XcTestNodeType.TEST_CASE,
+        node_identifier="ExpectedFailureProbeTests/testExpectedFailure()",
+        result=XcTestResult.EXPECTED_FAILURE,
+        children=[
+            XcTestNode(
+                name="known bad assertion",
+                node_type=XcTestNodeType.EXPECTED_FAILURE,
+            ),
+        ],
+    )
+    xc_test_suite = XcTestNode(name="ExpectedFailureProbeTests", node_type=XcTestNodeType.TEST_SUITE)
+
+    test_case = Xcode16XcResultConverter._get_test_case(xc_test_case, xc_test_suite)
+
+    assert test_case == TestCase(
+        name="testExpectedFailure()",
+        classname="ExpectedFailureProbeTests",
+        error=None,
+        time=0.0,
+        status="Expected Failure",
+        skipped=None,
+        system_out="known bad assertion",
+    )
+
+
+def test_get_test_case_system_out_absent_for_success():
+    xc_test_case = XcTestNode(
+        name="testPass()",
+        node_type=XcTestNodeType.TEST_CASE,
+        result=XcTestResult.PASSED,
+    )
+    assert Xcode16XcResultConverter._get_test_case_system_out(xc_test_case) is None
+
+
+def test_get_test_case_system_out_on_failed_case_with_expected_failures():
+    real_failure = 'XCTAssertEqual failed: ("3") is not equal to ("4") - should be real failure'
+    xc_test_case = XcTestNode(
+        name="testExpectedFailureThenRealFailure()",
+        node_type=XcTestNodeType.TEST_CASE,
+        node_identifier="ExpectedFailureProbeTests/testExpectedFailureThenRealFailure()",
+        result=XcTestResult.FAILED,
+        children=[
+            XcTestNode(
+                name=real_failure,
+                node_type=XcTestNodeType.FAILURE_MESSAGE,
+            ),
+            XcTestNode(
+                name="known bad assertion",
+                node_type=XcTestNodeType.EXPECTED_FAILURE,
+                children=[
+                    XcTestNode(
+                        name="nested expected failure message",
+                        node_type=XcTestNodeType.FAILURE_MESSAGE,
+                    ),
+                ],
+            ),
+            XcTestNode(
+                name="expected failure via result",
+                node_type=XcTestNodeType.FAILURE_MESSAGE,
+                result=XcTestResult.EXPECTED_FAILURE,
+            ),
+        ],
+    )
+    xc_test_suite = XcTestNode(name="ExpectedFailureProbeTests", node_type=XcTestNodeType.TEST_SUITE)
+
+    test_case = Xcode16XcResultConverter._get_test_case(xc_test_case, xc_test_suite)
+
+    assert test_case.error == Error(message=real_failure, type="Failure")
+    assert test_case.system_out == "known bad assertion\nexpected failure via result"
+    assert test_case.status == "Failed"
+    assert test_case.skipped is None

tests/models/xctests/xcresult/test_xcresult.py

@@ -10,6 +10,18 @@
 from codemagic.models.xctests.xcresult.xcresult import XcTestStatistic
 
 
+def test_parse_expected_failure_node_type():
+    node = XcTestNode.from_dict({"nodeType": "Expected Failure", "name": "known bad assertion"})
+    assert node.node_type is XcTestNodeType.EXPECTED_FAILURE
+    assert node.name == "known bad assertion"
+
+
+def test_parse_skip_message_node_type():
+    node = XcTestNode.from_dict({"nodeType": "Skip Message", "name": "Test skipped - skipping on purpose"})
+    assert node.node_type is XcTestNodeType.SKIP_MESSAGE
+    assert node.name == "Test skipped - skipping on purpose"
+
+
 def test_load_test_results_summary(test_results_summary_dict):
     test_results_summary = XcSummary.from_dict(test_results_summary_dict)