feat(metrics): add JSON schema conformance metric

openagent_eval/core/pipeline.py

@@ -115,7 +115,7 @@ async def _evaluate_item(
         ground_truth = item.get("ground_truth")
         context = item.get("context")
         gt_contexts = item.get("ground_truth_contexts", []) or []
-
+        metadata = item.get("metadata", {})
         try:
             # 1. Retrieval
             contexts = await self._retrieve(question, context, gt_contexts)
@@ -134,6 +134,7 @@ async def _evaluate_item(
                 gt_contexts,
                 latency_ms,
                 token_usage,
+                metadata,
             )
 
             return EvaluationResult(
@@ -287,6 +288,7 @@ def _run_metrics(
         gt_contexts: list[str],
         latency_ms: float | None,
         token_usage: Any | None,
+        metadata: dict[str, Any],
     ) -> tuple[dict[str, float], dict[str, str]]:
         """Run every configured metric and collect scores.
 
@@ -316,6 +318,7 @@ def _run_metrics(
                     completion_tokens=completion_tokens,
                     provider=provider_name,
                     model=model_name,
+                    metadata=metadata,
                 )
                 scores[name] = result.score
             except Exception as e:

openagent_eval/metrics/__init__.py

@@ -24,6 +24,7 @@
     HallucinationDetection,
     JudgeCriteria,
     LLMJudgeMetric,
+    SchemaConformance,
     SemanticSimilarity,
 )
 from openagent_eval.metrics.nli import (
@@ -70,6 +71,7 @@
     "AsyncLLMJudgeMetric",
     "JudgeCriteria",
     "ROUGE",
+    "SchemaConformance",
     "SemanticSimilarity",
     # NLI
     "NLIJudge",
@@ -103,6 +105,7 @@
     "bleu": BLEU,
     "rouge": ROUGE,
     "bertscore": BERTScore,
+    "schema_conformance": SchemaConformance,
     "latency": LatencyMetric,
     "token_count": TokenCountMetric,
 }

openagent_eval/metrics/generation/__init__.py

@@ -20,6 +20,7 @@
 )
 from openagent_eval.metrics.generation.relevancy import AnswerRelevancy
 from openagent_eval.metrics.generation.rouge import ROUGE
+from openagent_eval.metrics.generation.schema_conformance import SchemaConformance
 from openagent_eval.metrics.generation.similarity import SemanticSimilarity
 
 __all__ = [
@@ -37,5 +38,6 @@
     "JudgeCriteria",
     "RELEVANCY_CRITERIA",
     "ROUGE",
+    "SchemaConformance",
     "SemanticSimilarity",
 ]

openagent_eval/metrics/generation/schema_conformance.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from jsonschema import Draft202012Validator
+
+from openagent_eval.metrics.base import BaseMetric, MetricResult
+
+
+class SchemaConformance(BaseMetric):
+    """Measure whether a generated JSON answer conforms to a JSON Schema."""
+
+    name = "schema_conformance"
+    description = "Measures JSON Schema conformance of a generated answer."
+
+    def evaluate(self, **kwargs: Any) -> MetricResult:
+        """Evaluate a generated answer against an expected JSON Schema."""
+        answer = kwargs.get("answer", "")
+        metadata = kwargs.get("metadata", {})
+
+        schema = metadata.get("json_schema") if isinstance(metadata, dict) else None
+
+        if schema is None:
+            return MetricResult(
+                score=0.0,
+                reason="No JSON schema provided",
+                metadata={"valid": False, "json_parseable": False},
+            )
+
+        try:
+            data = json.loads(answer)
+        except (json.JSONDecodeError, TypeError):
+            return MetricResult(
+                score=0.0,
+                reason="Answer is not valid JSON",
+                metadata={"valid": False, "json_parseable": False},
+            )
+
+        validator = Draft202012Validator(schema)
+        errors = list(validator.iter_errors(data))
+
+        if not errors:
+            return MetricResult(
+                score=1.0,
+                reason="Answer conforms to JSON schema",
+                metadata={
+                    "valid": True,
+                    "json_parseable": True,
+                },
+            )
+
+        # For object schemas, provide deterministic top-level
+        # field-level partial scoring.
+        if (
+            isinstance(data, dict)
+            and schema.get("type") == "object"
+            and isinstance(schema.get("properties"), dict)
+        ):
+            properties = schema["properties"]
+            required = set(schema.get("required", []))
+
+            applicable_properties = [
+                name for name in properties if name in data or name in required
+            ]
+
+            if applicable_properties:
+                valid_properties = 0
+
+                for name in applicable_properties:
+                    if name not in data:
+                        continue
+
+                    property_schema = properties[name]
+
+                    if Draft202012Validator(property_schema).is_valid(data[name]):
+                        valid_properties += 1
+
+                score = valid_properties / len(applicable_properties)
+
+                if score == 1.0 and errors:
+                    score = 0.0
+
+                return MetricResult(
+                    score=score,
+                    reason="Answer partially conforms to JSON schema",
+                    metadata={
+                        "valid": False,
+                        "json_parseable": True,
+                        "valid_properties": valid_properties,
+                        "total_properties": len(applicable_properties),
+                    },
+                )
+
+        return MetricResult(
+            score=0.0,
+            reason="Answer does not conform to JSON schema",
+            metadata={
+                "valid": False,
+                "json_parseable": True,
+            },
+        )

pyproject.toml

@@ -31,6 +31,7 @@ dependencies = [
     "loguru>=0.7.0",
     "jinja2>=3.1.0",
     "httpx>=0.27.0",
+    "jsonschema>=4.26.0",
 ]
 
 [project.urls]

tests/unit/test_metrics/test_generation.py

@@ -15,6 +15,7 @@
     F1Score,
     Faithfulness,
     HallucinationDetection,
+    SchemaConformance,
     SemanticSimilarity,
 )
 
@@ -194,6 +195,90 @@ def _force_fallback(*args, **kwargs):
         assert partial.metadata["method"] == "simple_recall"
 
 
+class TestSchemaConformance:
+    """Tests for SchemaConformance metric."""
+
+    def setup_method(self):
+        self.metric = SchemaConformance()
+        self.schema = {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string"},
+                "age": {"type": "integer"},
+            },
+            "required": ["name", "age"],
+        }
+
+    def test_valid_output(self):
+        """Fully conformant JSON receives a perfect score."""
+        result = self.metric.evaluate(
+            answer='{"name": "Alice", "age": 25}',
+            metadata={"json_schema": self.schema},
+        )
+
+        assert result.score == 1.0
+        assert result.metadata["valid"] is True
+        assert result.metadata["json_parseable"] is True
+
+    def test_invalid_json(self):
+        """Malformed JSON receives zero score."""
+        result = self.metric.evaluate(
+            answer="not valid json",
+            metadata={"json_schema": self.schema},
+        )
+
+        assert result.score == 0.0
+        assert result.metadata["valid"] is False
+        assert result.metadata["json_parseable"] is False
+
+    def test_partial_schema_violation(self):
+        """Valid JSON with one invalid field receives a partial score."""
+        result = self.metric.evaluate(
+            answer='{"name": "Alice", "age": "twenty"}',
+            metadata={"json_schema": self.schema},
+        )
+
+        assert result.score == 0.5
+        assert result.metadata["valid"] is False
+        assert result.metadata["json_parseable"] is True
+
+    def test_missing_schema(self):
+        """Missing schema receives zero score."""
+        result = self.metric.evaluate(
+            answer='{"name": "Alice", "age": 25}',
+            metadata={},
+        )
+
+        assert result.score == 0.0
+        assert result.metadata["valid"] is False
+
+    def test_empty_schema_accepts_valid_json(self):
+        """An empty JSON Schema accepts any valid JSON value."""
+        result = self.metric.evaluate(
+            answer='{"name": "Alice"}',
+            metadata={"json_schema": {}},
+        )
+
+        assert result.score == 1.0
+        assert result.metadata["valid"] is True
+        assert result.metadata["json_parseable"] is True
+
+    def test_extra_property_is_not_fully_conformant(self):
+        """Extra properties prevent full schema conformance."""
+        schema = {
+            **self.schema,
+            "additionalProperties": False,
+        }
+
+        result = self.metric.evaluate(
+            answer='{"name": "Alice", "age": 25, "extra": "value"}',
+            metadata={"json_schema": schema},
+        )
+
+        assert result.score < 1.0
+        assert result.metadata["valid"] is False
+
+
 class TestSemanticSimilarity:
     """Tests for SemanticSimilarity metric."""
 

tests/unit/test_metrics/test_registry.py

@@ -31,6 +31,7 @@ def test_registry_has_all_metrics(self):
             "semantic_similarity",
             "exact_match",
             "f1_score",
+            "schema_conformance",
             "bleu",
             "rouge",
             "bertscore",
@@ -59,6 +60,10 @@ def test_get_metric_existing(self):
         assert metric_class is not None
         assert issubclass(metric_class, BaseMetric)
 
+        schema_metric_class = get_metric("schema_conformance")
+        assert schema_metric_class is not None
+        assert issubclass(schema_metric_class, BaseMetric)
+
     def test_get_metric_nonexistent(self):
         """get_metric raises KeyError for nonexistent metric."""
         with pytest.raises(KeyError, match="not found"):