From a9c0a001e55402664221f181f58719538dacff1d Mon Sep 17 00:00:00 2001
From: CTWalk <100585900+CTWalk@users.noreply.github.com>
Date: Thu, 30 Jul 2026 00:07:50 +0800
Subject: [PATCH] Python: fix LocalEvaluator reporting zero-check items as
 passed

LocalEvaluator.evaluate initialized item_passed to True and only ever
cleared it inside the loop over check results. With no checks configured
the loop never runs, so an item with zero scores was recorded as passed:
result_counts reported one pass, all_passed was True, and
raise_for_status() did not raise.

Initialize item_passed from bool(check_results) so an item with no
evaluated checks fails closed. This matches the .NET contract in this
repository, where AgentEvaluationResults.ItemPassed ends with
'return result.Metrics.Count > 0' and is pinned by
LocalEvaluator_WithZeroChecks_ItemsHaveZeroMetricsAndFailAsync.

Add a focused regression covering the counts, all_passed, the empty
score list, and raise_for_status(). Update the LocalEvaluator class and
evaluate() docstrings, which previously described the pass rule without
the zero-check case.

Fixes #7397
---
 python/packages/core/agent_framework/_evaluation.py | 10 ++++++----
 python/packages/core/tests/core/test_local_eval.py  | 12 ++++++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py
index 40190fc0ee6..cac000ca0b2 100644
--- a/python/packages/core/agent_framework/_evaluation.py
+++ b/python/packages/core/agent_framework/_evaluation.py
@@ -1520,7 +1520,8 @@ class LocalEvaluator:
     """Evaluation provider that runs checks locally without API calls.
 
     Implements the ``Evaluator`` protocol. Each check function is applied
-    to every item. An item passes only if all checks pass.
+    to every item. An item passes only if at least one check was evaluated
+    and all evaluated checks pass.
 
     Examples:
         Basic usage:
@@ -1560,8 +1561,9 @@ async def evaluate(
     ) -> EvalResults:
         """Run all checks on each item and return aggregated results.
 
-        An item passes only if every check passes for that item. Per-check
-        breakdowns are available in ``per_evaluator``.
+        An item passes only if every check passes for that item. An item with
+        no checks to evaluate fails, since a pass would carry no evidence.
+        Per-check breakdowns are available in ``per_evaluator``.
 
         Supports both sync and async check functions (from
         :func:`evaluator`).
@@ -1574,7 +1576,7 @@ async def evaluate(
 
         for item_idx, item in enumerate(items):
             check_results = await asyncio.gather(*[_run_check(fn, item) for fn in self._checks])
-            item_passed = True
+            item_passed = bool(check_results)
             item_scores: list[EvalScoreResult] = []
             for result in check_results:
                 counts = per_check.setdefault(result.check_name, {"passed": 0, "failed": 0, "errored": 0})
diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py
index 1c640859df6..f3713506c22 100644
--- a/python/packages/core/tests/core/test_local_eval.py
+++ b/python/packages/core/tests/core/test_local_eval.py
@@ -374,6 +374,18 @@ async def async_fn(response: str) -> bool:
 
 
 class TestLocalEvaluatorIntegration:
+    @pytest.mark.asyncio
+    async def test_zero_checks(self):
+        """A LocalEvaluator with no checks produces items with 0 scores, which count as failed."""
+        local = LocalEvaluator()
+        results = await local.evaluate([_make_item()])
+
+        assert results.result_counts == {"passed": 0, "failed": 1, "errored": 0}
+        assert results.all_passed is False
+        assert results.items[0].scores == []
+        with pytest.raises(EvalNotPassedError):
+            results.raise_for_status()
+
     @pytest.mark.asyncio
     async def test_mixed_checks(self):
         """Function evaluators mix with built-in checks in LocalEvaluator."""