diff --git a/langfuse/api/unstable/__init__.py b/langfuse/api/unstable/__init__.py index 8fe7fdc80..8ac8c3c43 100644 --- a/langfuse/api/unstable/__init__.py +++ b/langfuse/api/unstable/__init__.py @@ -53,6 +53,7 @@ EvaluationRuleNullFilterOperator, EvaluationRuleNumberFilterOperator, EvaluationRuleOptionsFilterOperator, + EvaluationRuleReadMapping, EvaluationRuleStatus, EvaluationRuleStringFilterOperator, EvaluationRuleTarget, @@ -63,7 +64,6 @@ EvaluatorOutputDefinition_Categorical, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, - EvaluatorScope, EvaluatorType, NullEvaluationRuleFilter, NumberEvaluationRuleFilter, @@ -76,6 +76,7 @@ PublicEvaluatorOutputDefinition_Categorical, PublicEvaluatorOutputDefinition_Numeric, PublicNumericEvaluatorOutputDefinition, + PublicNumericEvaluatorOutputScoreDefinition, StringEvaluationRuleFilter, StringObjectEvaluationRuleFilter, StringOptionsEvaluationRuleFilter, @@ -121,17 +122,22 @@ from .evaluation_rules import ( CodeEvaluationRuleEvaluatorReference, CreateCodeEvaluationRuleRequest, + CreateEvaluationRuleEvaluatorAssignment, + CreateEvaluationRuleEvaluatorReference, CreateEvaluationRuleRequest, + CreateEvaluationRuleWithEvaluatorsRequest, CreateLlmAsJudgeEvaluationRuleRequest, DeleteEvaluationRuleResponse, EvaluationRule, EvaluationRuleBase, EvaluationRuleEvaluator, + EvaluationRuleEvaluatorAssignment, EvaluationRuleEvaluatorReference, EvaluationRuleTimeScope, EvaluationRules, LegacyEvaluationObject, LegacyEvaluationRule, + LegacyEvaluationRuleEvaluatorAssignment, LegacyEvaluationRuleMapping, LegacyEvaluationRuleTarget, LlmAsJudgeEvaluationRuleEvaluatorReference, @@ -171,7 +177,10 @@ "CreateDashboardPlacementRequest_Widget": ".dashboards", "CreateDashboardRequest": ".dashboards", "CreateDashboardWidgetRequest": ".dashboard_widgets", + "CreateEvaluationRuleEvaluatorAssignment": ".evaluation_rules", + "CreateEvaluationRuleEvaluatorReference": ".evaluation_rules", "CreateEvaluationRuleRequest": ".evaluation_rules", + "CreateEvaluationRuleWithEvaluatorsRequest": ".evaluation_rules", "CreateEvaluatorRequest": ".evaluators", "CreateEvaluatorRequest_Code": ".evaluators", "CreateEvaluatorRequest_LlmAsJudge": ".evaluators", @@ -209,6 +218,7 @@ "EvaluationRuleBase": ".evaluation_rules", "EvaluationRuleBooleanFilterOperator": ".commons", "EvaluationRuleEvaluator": ".evaluation_rules", + "EvaluationRuleEvaluatorAssignment": ".evaluation_rules", "EvaluationRuleEvaluatorReference": ".evaluation_rules", "EvaluationRuleFilter": ".commons", "EvaluationRuleFilter_ArrayOptions": ".commons", @@ -226,6 +236,7 @@ "EvaluationRuleNullFilterOperator": ".commons", "EvaluationRuleNumberFilterOperator": ".commons", "EvaluationRuleOptionsFilterOperator": ".commons", + "EvaluationRuleReadMapping": ".commons", "EvaluationRuleStatus": ".commons", "EvaluationRuleStringFilterOperator": ".commons", "EvaluationRuleTarget": ".commons", @@ -240,7 +251,6 @@ "EvaluatorOutputDefinition_Categorical": ".commons", "EvaluatorOutputDefinition_Numeric": ".commons", "EvaluatorOutputFieldDefinition": ".commons", - "EvaluatorScope": ".commons", "EvaluatorType": ".commons", "Evaluator_Code": ".evaluators", "Evaluator_LlmAsJudge": ".evaluators", @@ -248,6 +258,7 @@ "InternalServerError": ".errors", "LegacyEvaluationObject": ".evaluation_rules", "LegacyEvaluationRule": ".evaluation_rules", + "LegacyEvaluationRuleEvaluatorAssignment": ".evaluation_rules", "LegacyEvaluationRuleMapping": ".evaluation_rules", "LegacyEvaluationRuleTarget": ".evaluation_rules", "LlmAsJudgeEvaluationRuleEvaluatorReference": ".evaluation_rules", @@ -271,6 +282,7 @@ "PublicEvaluatorOutputDefinition_Categorical": ".commons", "PublicEvaluatorOutputDefinition_Numeric": ".commons", "PublicNumericEvaluatorOutputDefinition": ".commons", + "PublicNumericEvaluatorOutputScoreDefinition": ".commons", "ReadableEvaluationRule": ".evaluation_rules", "StringEvaluationRuleFilter": ".commons", "StringObjectEvaluationRuleFilter": ".commons", @@ -336,7 +348,10 @@ def __dir__(): "CreateDashboardPlacementRequest_Widget", "CreateDashboardRequest", "CreateDashboardWidgetRequest", + "CreateEvaluationRuleEvaluatorAssignment", + "CreateEvaluationRuleEvaluatorReference", "CreateEvaluationRuleRequest", + "CreateEvaluationRuleWithEvaluatorsRequest", "CreateEvaluatorRequest", "CreateEvaluatorRequest_Code", "CreateEvaluatorRequest_LlmAsJudge", @@ -374,6 +389,7 @@ def __dir__(): "EvaluationRuleBase", "EvaluationRuleBooleanFilterOperator", "EvaluationRuleEvaluator", + "EvaluationRuleEvaluatorAssignment", "EvaluationRuleEvaluatorReference", "EvaluationRuleFilter", "EvaluationRuleFilter_ArrayOptions", @@ -391,6 +407,7 @@ def __dir__(): "EvaluationRuleNullFilterOperator", "EvaluationRuleNumberFilterOperator", "EvaluationRuleOptionsFilterOperator", + "EvaluationRuleReadMapping", "EvaluationRuleStatus", "EvaluationRuleStringFilterOperator", "EvaluationRuleTarget", @@ -405,7 +422,6 @@ def __dir__(): "EvaluatorOutputDefinition_Categorical", "EvaluatorOutputDefinition_Numeric", "EvaluatorOutputFieldDefinition", - "EvaluatorScope", "EvaluatorType", "Evaluator_Code", "Evaluator_LlmAsJudge", @@ -413,6 +429,7 @@ def __dir__(): "InternalServerError", "LegacyEvaluationObject", "LegacyEvaluationRule", + "LegacyEvaluationRuleEvaluatorAssignment", "LegacyEvaluationRuleMapping", "LegacyEvaluationRuleTarget", "LlmAsJudgeEvaluationRuleEvaluatorReference", @@ -436,6 +453,7 @@ def __dir__(): "PublicEvaluatorOutputDefinition_Categorical", "PublicEvaluatorOutputDefinition_Numeric", "PublicNumericEvaluatorOutputDefinition", + "PublicNumericEvaluatorOutputScoreDefinition", "ReadableEvaluationRule", "StringEvaluationRuleFilter", "StringObjectEvaluationRuleFilter", diff --git a/langfuse/api/unstable/commons/__init__.py b/langfuse/api/unstable/commons/__init__.py index c617b53c7..748c1d8cd 100644 --- a/langfuse/api/unstable/commons/__init__.py +++ b/langfuse/api/unstable/commons/__init__.py @@ -30,6 +30,7 @@ EvaluationRuleNullFilterOperator, EvaluationRuleNumberFilterOperator, EvaluationRuleOptionsFilterOperator, + EvaluationRuleReadMapping, EvaluationRuleStatus, EvaluationRuleStringFilterOperator, EvaluationRuleTarget, @@ -40,7 +41,6 @@ EvaluatorOutputDefinition_Categorical, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, - EvaluatorScope, EvaluatorType, NullEvaluationRuleFilter, NumberEvaluationRuleFilter, @@ -53,6 +53,7 @@ PublicEvaluatorOutputDefinition_Categorical, PublicEvaluatorOutputDefinition_Numeric, PublicNumericEvaluatorOutputDefinition, + PublicNumericEvaluatorOutputScoreDefinition, StringEvaluationRuleFilter, StringObjectEvaluationRuleFilter, StringOptionsEvaluationRuleFilter, @@ -81,6 +82,7 @@ "EvaluationRuleNullFilterOperator": ".types", "EvaluationRuleNumberFilterOperator": ".types", "EvaluationRuleOptionsFilterOperator": ".types", + "EvaluationRuleReadMapping": ".types", "EvaluationRuleStatus": ".types", "EvaluationRuleStringFilterOperator": ".types", "EvaluationRuleTarget": ".types", @@ -91,7 +93,6 @@ "EvaluatorOutputDefinition_Categorical": ".types", "EvaluatorOutputDefinition_Numeric": ".types", "EvaluatorOutputFieldDefinition": ".types", - "EvaluatorScope": ".types", "EvaluatorType": ".types", "NullEvaluationRuleFilter": ".types", "NumberEvaluationRuleFilter": ".types", @@ -104,6 +105,7 @@ "PublicEvaluatorOutputDefinition_Categorical": ".types", "PublicEvaluatorOutputDefinition_Numeric": ".types", "PublicNumericEvaluatorOutputDefinition": ".types", + "PublicNumericEvaluatorOutputScoreDefinition": ".types", "StringEvaluationRuleFilter": ".types", "StringObjectEvaluationRuleFilter": ".types", "StringOptionsEvaluationRuleFilter": ".types", @@ -161,6 +163,7 @@ def __dir__(): "EvaluationRuleNullFilterOperator", "EvaluationRuleNumberFilterOperator", "EvaluationRuleOptionsFilterOperator", + "EvaluationRuleReadMapping", "EvaluationRuleStatus", "EvaluationRuleStringFilterOperator", "EvaluationRuleTarget", @@ -171,7 +174,6 @@ def __dir__(): "EvaluatorOutputDefinition_Categorical", "EvaluatorOutputDefinition_Numeric", "EvaluatorOutputFieldDefinition", - "EvaluatorScope", "EvaluatorType", "NullEvaluationRuleFilter", "NumberEvaluationRuleFilter", @@ -184,6 +186,7 @@ def __dir__(): "PublicEvaluatorOutputDefinition_Categorical", "PublicEvaluatorOutputDefinition_Numeric", "PublicNumericEvaluatorOutputDefinition", + "PublicNumericEvaluatorOutputScoreDefinition", "StringEvaluationRuleFilter", "StringObjectEvaluationRuleFilter", "StringOptionsEvaluationRuleFilter", diff --git a/langfuse/api/unstable/commons/types/__init__.py b/langfuse/api/unstable/commons/types/__init__.py index 487480da4..1b60a87c1 100644 --- a/langfuse/api/unstable/commons/types/__init__.py +++ b/langfuse/api/unstable/commons/types/__init__.py @@ -41,6 +41,7 @@ from .evaluation_rule_options_filter_operator import ( EvaluationRuleOptionsFilterOperator, ) + from .evaluation_rule_read_mapping import EvaluationRuleReadMapping from .evaluation_rule_status import EvaluationRuleStatus from .evaluation_rule_string_filter_operator import ( EvaluationRuleStringFilterOperator, @@ -55,7 +56,6 @@ EvaluatorOutputDefinition_Numeric, ) from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition - from .evaluator_scope import EvaluatorScope from .evaluator_type import EvaluatorType from .null_evaluation_rule_filter import NullEvaluationRuleFilter from .number_evaluation_rule_filter import NumberEvaluationRuleFilter @@ -78,6 +78,9 @@ from .public_numeric_evaluator_output_definition import ( PublicNumericEvaluatorOutputDefinition, ) + from .public_numeric_evaluator_output_score_definition import ( + PublicNumericEvaluatorOutputScoreDefinition, + ) from .string_evaluation_rule_filter import StringEvaluationRuleFilter from .string_object_evaluation_rule_filter import StringObjectEvaluationRuleFilter from .string_options_evaluation_rule_filter import StringOptionsEvaluationRuleFilter @@ -105,6 +108,7 @@ "EvaluationRuleNullFilterOperator": ".evaluation_rule_null_filter_operator", "EvaluationRuleNumberFilterOperator": ".evaluation_rule_number_filter_operator", "EvaluationRuleOptionsFilterOperator": ".evaluation_rule_options_filter_operator", + "EvaluationRuleReadMapping": ".evaluation_rule_read_mapping", "EvaluationRuleStatus": ".evaluation_rule_status", "EvaluationRuleStringFilterOperator": ".evaluation_rule_string_filter_operator", "EvaluationRuleTarget": ".evaluation_rule_target", @@ -115,7 +119,6 @@ "EvaluatorOutputDefinition_Categorical": ".evaluator_output_definition", "EvaluatorOutputDefinition_Numeric": ".evaluator_output_definition", "EvaluatorOutputFieldDefinition": ".evaluator_output_field_definition", - "EvaluatorScope": ".evaluator_scope", "EvaluatorType": ".evaluator_type", "NullEvaluationRuleFilter": ".null_evaluation_rule_filter", "NumberEvaluationRuleFilter": ".number_evaluation_rule_filter", @@ -128,6 +131,7 @@ "PublicEvaluatorOutputDefinition_Categorical": ".public_evaluator_output_definition", "PublicEvaluatorOutputDefinition_Numeric": ".public_evaluator_output_definition", "PublicNumericEvaluatorOutputDefinition": ".public_numeric_evaluator_output_definition", + "PublicNumericEvaluatorOutputScoreDefinition": ".public_numeric_evaluator_output_score_definition", "StringEvaluationRuleFilter": ".string_evaluation_rule_filter", "StringObjectEvaluationRuleFilter": ".string_object_evaluation_rule_filter", "StringOptionsEvaluationRuleFilter": ".string_options_evaluation_rule_filter", @@ -185,6 +189,7 @@ def __dir__(): "EvaluationRuleNullFilterOperator", "EvaluationRuleNumberFilterOperator", "EvaluationRuleOptionsFilterOperator", + "EvaluationRuleReadMapping", "EvaluationRuleStatus", "EvaluationRuleStringFilterOperator", "EvaluationRuleTarget", @@ -195,7 +200,6 @@ def __dir__(): "EvaluatorOutputDefinition_Categorical", "EvaluatorOutputDefinition_Numeric", "EvaluatorOutputFieldDefinition", - "EvaluatorScope", "EvaluatorType", "NullEvaluationRuleFilter", "NumberEvaluationRuleFilter", @@ -208,6 +212,7 @@ def __dir__(): "PublicEvaluatorOutputDefinition_Categorical", "PublicEvaluatorOutputDefinition_Numeric", "PublicNumericEvaluatorOutputDefinition", + "PublicNumericEvaluatorOutputScoreDefinition", "StringEvaluationRuleFilter", "StringObjectEvaluationRuleFilter", "StringOptionsEvaluationRuleFilter", diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_read_mapping.py b/langfuse/api/unstable/commons/types/evaluation_rule_read_mapping.py new file mode 100644 index 000000000..20cb403a7 --- /dev/null +++ b/langfuse/api/unstable/commons/types/evaluation_rule_read_mapping.py @@ -0,0 +1,56 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ....core.pydantic_utilities import UniversalBaseModel +from ....core.serialization import FieldMetadata +from .evaluation_rule_mapping_source import EvaluationRuleMappingSource + + +class EvaluationRuleReadMapping(UniversalBaseModel): + """ + Maps an evaluator variable to target data, as returned by read endpoints. + + Reads are more permissive than writes in two ways, so that data configured before a + migration stays readable and repairable: + - `source` is `null` when a mapping was never fully configured. Set a concrete `source` to fix it. + - every `source` value is allowed regardless of the rule `target`, because an evaluator's + default mapping is stored independently of the rules that use it. + + Requests always require a concrete `source` that is valid for the selected `target`. + + Examples + -------- + from langfuse.unstable.commons import ( + EvaluationRuleMappingSource, + EvaluationRuleReadMapping, + ) + + EvaluationRuleReadMapping( + variable="input", + source=EvaluationRuleMappingSource.INPUT, + ) + """ + + variable: str = pydantic.Field() + """ + Prompt variable name without braces. + """ + + source: typing.Optional[EvaluationRuleMappingSource] = pydantic.Field(default=None) + """ + Source field populating the variable, or `null` when the mapping is incomplete. + """ + + json_path: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="jsonPath") + ] = pydantic.Field(default=None) + """ + Optional JSONPath selector applied to the selected source. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/commons/types/evaluator_output_definition.py b/langfuse/api/unstable/commons/types/evaluator_output_definition.py index f545a19a8..661d142bb 100644 --- a/langfuse/api/unstable/commons/types/evaluator_output_definition.py +++ b/langfuse/api/unstable/commons/types/evaluator_output_definition.py @@ -12,6 +12,9 @@ from .public_categorical_evaluator_output_score_definition import ( PublicCategoricalEvaluatorOutputScoreDefinition, ) +from .public_numeric_evaluator_output_score_definition import ( + PublicNumericEvaluatorOutputScoreDefinition, +) class EvaluatorOutputDefinition_Numeric(UniversalBaseModel): @@ -22,6 +25,7 @@ class EvaluatorOutputDefinition_Numeric(UniversalBaseModel): - `dataType` is required. - Do not send `version`; that is an internal storage detail and is not part of the public request contract. - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and `score.description`. + - For `NUMERIC`, `score.minValue` and `score.maxValue` optionally define inclusive bounds. If both are set, `minValue` must not exceed `maxValue`. - For `CATEGORICAL`, also provide `score.categories` and `score.shouldAllowMultipleMatches`. Examples @@ -30,6 +34,7 @@ class EvaluatorOutputDefinition_Numeric(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PublicNumericEvaluatorOutputScoreDefinition, ) EvaluatorOutputDefinition_Numeric( @@ -37,8 +42,10 @@ class EvaluatorOutputDefinition_Numeric(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ @@ -47,7 +54,7 @@ class EvaluatorOutputDefinition_Numeric(UniversalBaseModel): typing.Literal["NUMERIC"], FieldMetadata(alias="dataType") ] = "NUMERIC" reasoning: EvaluatorOutputFieldDefinition - score: EvaluatorOutputFieldDefinition + score: PublicNumericEvaluatorOutputScoreDefinition model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True @@ -62,6 +69,7 @@ class EvaluatorOutputDefinition_Boolean(UniversalBaseModel): - `dataType` is required. - Do not send `version`; that is an internal storage detail and is not part of the public request contract. - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and `score.description`. + - For `NUMERIC`, `score.minValue` and `score.maxValue` optionally define inclusive bounds. If both are set, `minValue` must not exceed `maxValue`. - For `CATEGORICAL`, also provide `score.categories` and `score.shouldAllowMultipleMatches`. Examples @@ -70,6 +78,7 @@ class EvaluatorOutputDefinition_Boolean(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PublicNumericEvaluatorOutputScoreDefinition, ) EvaluatorOutputDefinition_Numeric( @@ -77,8 +86,10 @@ class EvaluatorOutputDefinition_Boolean(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ @@ -102,6 +113,7 @@ class EvaluatorOutputDefinition_Categorical(UniversalBaseModel): - `dataType` is required. - Do not send `version`; that is an internal storage detail and is not part of the public request contract. - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and `score.description`. + - For `NUMERIC`, `score.minValue` and `score.maxValue` optionally define inclusive bounds. If both are set, `minValue` must not exceed `maxValue`. - For `CATEGORICAL`, also provide `score.categories` and `score.shouldAllowMultipleMatches`. Examples @@ -110,6 +122,7 @@ class EvaluatorOutputDefinition_Categorical(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PublicNumericEvaluatorOutputScoreDefinition, ) EvaluatorOutputDefinition_Numeric( @@ -117,8 +130,10 @@ class EvaluatorOutputDefinition_Categorical(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ @@ -139,6 +154,7 @@ class EvaluatorOutputDefinition_Categorical(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PublicNumericEvaluatorOutputScoreDefinition, ) EvaluatorOutputDefinition_Numeric( @@ -146,8 +162,10 @@ class EvaluatorOutputDefinition_Categorical(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ diff --git a/langfuse/api/unstable/commons/types/evaluator_scope.py b/langfuse/api/unstable/commons/types/evaluator_scope.py deleted file mode 100644 index 7ce796418..000000000 --- a/langfuse/api/unstable/commons/types/evaluator_scope.py +++ /dev/null @@ -1,29 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluatorScope(enum.StrEnum): - """ - Where an evaluator comes from. - - - `project`: created in your project - - `managed`: provided by Langfuse - """ - - PROJECT = "project" - MANAGED = "managed" - - def visit( - self, - project: typing.Callable[[], T_Result], - managed: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluatorScope.PROJECT: - return project() - if self is EvaluatorScope.MANAGED: - return managed() diff --git a/langfuse/api/unstable/commons/types/public_evaluator_output_definition.py b/langfuse/api/unstable/commons/types/public_evaluator_output_definition.py index 43c7aa9ba..52331e6c1 100644 --- a/langfuse/api/unstable/commons/types/public_evaluator_output_definition.py +++ b/langfuse/api/unstable/commons/types/public_evaluator_output_definition.py @@ -12,6 +12,9 @@ from .public_categorical_evaluator_output_score_definition import ( PublicCategoricalEvaluatorOutputScoreDefinition, ) +from .public_numeric_evaluator_output_score_definition import ( + PublicNumericEvaluatorOutputScoreDefinition, +) class PublicEvaluatorOutputDefinition_Numeric(UniversalBaseModel): @@ -32,6 +35,7 @@ class PublicEvaluatorOutputDefinition_Numeric(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputFieldDefinition, PublicEvaluatorOutputDefinition_Numeric, + PublicNumericEvaluatorOutputScoreDefinition, ) PublicEvaluatorOutputDefinition_Numeric( @@ -39,8 +43,10 @@ class PublicEvaluatorOutputDefinition_Numeric(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ @@ -49,7 +55,7 @@ class PublicEvaluatorOutputDefinition_Numeric(UniversalBaseModel): typing.Literal["NUMERIC"], FieldMetadata(alias="dataType") ] = "NUMERIC" reasoning: EvaluatorOutputFieldDefinition - score: EvaluatorOutputFieldDefinition + score: PublicNumericEvaluatorOutputScoreDefinition model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True @@ -74,6 +80,7 @@ class PublicEvaluatorOutputDefinition_Boolean(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputFieldDefinition, PublicEvaluatorOutputDefinition_Numeric, + PublicNumericEvaluatorOutputScoreDefinition, ) PublicEvaluatorOutputDefinition_Numeric( @@ -81,8 +88,10 @@ class PublicEvaluatorOutputDefinition_Boolean(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ @@ -116,6 +125,7 @@ class PublicEvaluatorOutputDefinition_Categorical(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputFieldDefinition, PublicEvaluatorOutputDefinition_Numeric, + PublicNumericEvaluatorOutputScoreDefinition, ) PublicEvaluatorOutputDefinition_Numeric( @@ -123,8 +133,10 @@ class PublicEvaluatorOutputDefinition_Categorical(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ @@ -145,6 +157,7 @@ class PublicEvaluatorOutputDefinition_Categorical(UniversalBaseModel): EvaluatorOutputDataType, EvaluatorOutputFieldDefinition, PublicEvaluatorOutputDefinition_Numeric, + PublicNumericEvaluatorOutputScoreDefinition, ) PublicEvaluatorOutputDefinition_Numeric( @@ -152,8 +165,10 @@ class PublicEvaluatorOutputDefinition_Categorical(UniversalBaseModel): reasoning=EvaluatorOutputFieldDefinition( description="Explain why the answer is correct or incorrect.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Return a score between 0 and 1.", + min_value=0.0, + max_value=1.0, ), ) """ diff --git a/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_definition.py b/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_definition.py index 68987d2ff..db83d0121 100644 --- a/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_definition.py +++ b/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_definition.py @@ -8,6 +8,9 @@ from ....core.serialization import FieldMetadata from .evaluator_output_data_type import EvaluatorOutputDataType from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition +from .public_numeric_evaluator_output_score_definition import ( + PublicNumericEvaluatorOutputScoreDefinition, +) class PublicNumericEvaluatorOutputDefinition(UniversalBaseModel): @@ -19,7 +22,7 @@ class PublicNumericEvaluatorOutputDefinition(UniversalBaseModel): """ reasoning: EvaluatorOutputFieldDefinition - score: EvaluatorOutputFieldDefinition + score: PublicNumericEvaluatorOutputScoreDefinition model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_score_definition.py b/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_score_definition.py new file mode 100644 index 000000000..7f8c39597 --- /dev/null +++ b/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_score_definition.py @@ -0,0 +1,29 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ....core.pydantic_utilities import UniversalBaseModel +from ....core.serialization import FieldMetadata + + +class PublicNumericEvaluatorOutputScoreDefinition(UniversalBaseModel): + description: str + min_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="minValue") + ] = pydantic.Field(default=None) + """ + Inclusive minimum value for numeric evaluator scores. + """ + + max_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="maxValue") + ] = pydantic.Field(default=None) + """ + Inclusive maximum value for numeric evaluator scores. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/evaluation_rules/__init__.py b/langfuse/api/unstable/evaluation_rules/__init__.py index c8031798c..f10039b83 100644 --- a/langfuse/api/unstable/evaluation_rules/__init__.py +++ b/langfuse/api/unstable/evaluation_rules/__init__.py @@ -9,17 +9,22 @@ from .types import ( CodeEvaluationRuleEvaluatorReference, CreateCodeEvaluationRuleRequest, + CreateEvaluationRuleEvaluatorAssignment, + CreateEvaluationRuleEvaluatorReference, CreateEvaluationRuleRequest, + CreateEvaluationRuleWithEvaluatorsRequest, CreateLlmAsJudgeEvaluationRuleRequest, DeleteEvaluationRuleResponse, EvaluationRule, EvaluationRuleBase, EvaluationRuleEvaluator, + EvaluationRuleEvaluatorAssignment, EvaluationRuleEvaluatorReference, EvaluationRuleTimeScope, EvaluationRules, LegacyEvaluationObject, LegacyEvaluationRule, + LegacyEvaluationRuleEvaluatorAssignment, LegacyEvaluationRuleMapping, LegacyEvaluationRuleTarget, LlmAsJudgeEvaluationRuleEvaluatorReference, @@ -30,17 +35,22 @@ _dynamic_imports: typing.Dict[str, str] = { "CodeEvaluationRuleEvaluatorReference": ".types", "CreateCodeEvaluationRuleRequest": ".types", + "CreateEvaluationRuleEvaluatorAssignment": ".types", + "CreateEvaluationRuleEvaluatorReference": ".types", "CreateEvaluationRuleRequest": ".types", + "CreateEvaluationRuleWithEvaluatorsRequest": ".types", "CreateLlmAsJudgeEvaluationRuleRequest": ".types", "DeleteEvaluationRuleResponse": ".types", "EvaluationRule": ".types", "EvaluationRuleBase": ".types", "EvaluationRuleEvaluator": ".types", + "EvaluationRuleEvaluatorAssignment": ".types", "EvaluationRuleEvaluatorReference": ".types", "EvaluationRuleTimeScope": ".types", "EvaluationRules": ".types", "LegacyEvaluationObject": ".types", "LegacyEvaluationRule": ".types", + "LegacyEvaluationRuleEvaluatorAssignment": ".types", "LegacyEvaluationRuleMapping": ".types", "LegacyEvaluationRuleTarget": ".types", "LlmAsJudgeEvaluationRuleEvaluatorReference": ".types", @@ -80,17 +90,22 @@ def __dir__(): __all__ = [ "CodeEvaluationRuleEvaluatorReference", "CreateCodeEvaluationRuleRequest", + "CreateEvaluationRuleEvaluatorAssignment", + "CreateEvaluationRuleEvaluatorReference", "CreateEvaluationRuleRequest", + "CreateEvaluationRuleWithEvaluatorsRequest", "CreateLlmAsJudgeEvaluationRuleRequest", "DeleteEvaluationRuleResponse", "EvaluationRule", "EvaluationRuleBase", "EvaluationRuleEvaluator", + "EvaluationRuleEvaluatorAssignment", "EvaluationRuleEvaluatorReference", "EvaluationRuleTimeScope", "EvaluationRules", "LegacyEvaluationObject", "LegacyEvaluationRule", + "LegacyEvaluationRuleEvaluatorAssignment", "LegacyEvaluationRuleMapping", "LegacyEvaluationRuleTarget", "LlmAsJudgeEvaluationRuleEvaluatorReference", diff --git a/langfuse/api/unstable/evaluation_rules/client.py b/langfuse/api/unstable/evaluation_rules/client.py index a1206fbb4..598481a40 100644 --- a/langfuse/api/unstable/evaluation_rules/client.py +++ b/langfuse/api/unstable/evaluation_rules/client.py @@ -8,6 +8,9 @@ from ..commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ..commons.types.evaluation_rule_target import EvaluationRuleTarget from .raw_client import AsyncRawEvaluationRulesClient, RawEvaluationRulesClient +from .types.create_evaluation_rule_evaluator_assignment import ( + CreateEvaluationRuleEvaluatorAssignment, +) from .types.create_evaluation_rule_request import CreateEvaluationRuleRequest from .types.delete_evaluation_rule_response import DeleteEvaluationRuleResponse from .types.evaluation_rule import EvaluationRule @@ -50,19 +53,21 @@ def create( Key rules: - `name` must be unique within the project for public evaluation rules - `target` must be `observation` or `experiment` - - `evaluator.name` + `evaluator.scope` must identify an existing evaluator family returned by the evaluator endpoints + - provide either the compatibility `evaluator` field or the new `evaluators` array, never both + - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - Langfuse resolves that family to its latest version before saving the evaluation rule - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - for `llm_as_judge` evaluators, every evaluator prompt variable must be mapped exactly once + - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping + - the effective mapping must map every evaluator prompt variable exactly once - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 50 evaluation rules can be effectively active in one project at the same time + - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) If an evaluation rule with the same `name` already exists in the project, the API returns `409`. In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - If enabling this resource would exceed the 50-active limit, the API also returns `409`. + If enabling this resource would exceed the 500-active limit, the API also returns `409`. In that case, disable or pause another active evaluation rule before enabling a new one. Current scope: @@ -98,7 +103,6 @@ def create( EvaluationRuleMappingSource, EvaluationRuleOptionsFilterOperator, EvaluationRuleTarget, - EvaluatorScope, ) from langfuse.unstable.evaluation_rules import ( CreateLlmAsJudgeEvaluationRuleRequest, @@ -119,7 +123,6 @@ def create( name="answer-correctness-live", evaluator=LlmAsJudgeEvaluationRuleEvaluatorReference( name="answer-correctness", - scope=EvaluatorScope.PROJECT, type=LlmAsJudgeEvaluatorType.LLM_AS_JUDGE, ), target=EvaluationRuleTarget.OBSERVATION, @@ -245,6 +248,9 @@ def update( evaluation_rule_id: str, *, name: typing.Optional[str] = OMIT, + evaluators: typing.Optional[ + typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] + ] = OMIT, evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, target: typing.Optional[EvaluationRuleTarget] = OMIT, enabled: typing.Optional[bool] = OMIT, @@ -270,7 +276,7 @@ def update( - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 50 active evaluation rules, the API returns `409` + - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` Recovery guidance: - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` @@ -284,11 +290,17 @@ def update( name : typing.Optional[str] Updated deployment name. + evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] + Full replacement of the rule's evaluator assignments: entries that are + not listed are detached. + + Mutually exclusive with the deprecated `evaluator` and `mapping` fields. + evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Updated evaluator family. + Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` and `scope` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. + A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. target : typing.Optional[EvaluationRuleTarget] Updated target object type. @@ -335,6 +347,7 @@ def update( _response = self._raw_client.update( evaluation_rule_id, name=name, + evaluators=evaluators, evaluator=evaluator, target=target, enabled=enabled, @@ -421,19 +434,21 @@ async def create( Key rules: - `name` must be unique within the project for public evaluation rules - `target` must be `observation` or `experiment` - - `evaluator.name` + `evaluator.scope` must identify an existing evaluator family returned by the evaluator endpoints + - provide either the compatibility `evaluator` field or the new `evaluators` array, never both + - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - Langfuse resolves that family to its latest version before saving the evaluation rule - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - for `llm_as_judge` evaluators, every evaluator prompt variable must be mapped exactly once + - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping + - the effective mapping must map every evaluator prompt variable exactly once - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 50 evaluation rules can be effectively active in one project at the same time + - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) If an evaluation rule with the same `name` already exists in the project, the API returns `409`. In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - If enabling this resource would exceed the 50-active limit, the API also returns `409`. + If enabling this resource would exceed the 500-active limit, the API also returns `409`. In that case, disable or pause another active evaluation rule before enabling a new one. Current scope: @@ -471,7 +486,6 @@ async def create( EvaluationRuleMappingSource, EvaluationRuleOptionsFilterOperator, EvaluationRuleTarget, - EvaluatorScope, ) from langfuse.unstable.evaluation_rules import ( CreateLlmAsJudgeEvaluationRuleRequest, @@ -495,7 +509,6 @@ async def main() -> None: name="answer-correctness-live", evaluator=LlmAsJudgeEvaluationRuleEvaluatorReference( name="answer-correctness", - scope=EvaluatorScope.PROJECT, type=LlmAsJudgeEvaluatorType.LLM_AS_JUDGE, ), target=EvaluationRuleTarget.OBSERVATION, @@ -640,6 +653,9 @@ async def update( evaluation_rule_id: str, *, name: typing.Optional[str] = OMIT, + evaluators: typing.Optional[ + typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] + ] = OMIT, evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, target: typing.Optional[EvaluationRuleTarget] = OMIT, enabled: typing.Optional[bool] = OMIT, @@ -665,7 +681,7 @@ async def update( - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 50 active evaluation rules, the API returns `409` + - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` Recovery guidance: - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` @@ -679,11 +695,17 @@ async def update( name : typing.Optional[str] Updated deployment name. + evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] + Full replacement of the rule's evaluator assignments: entries that are + not listed are detached. + + Mutually exclusive with the deprecated `evaluator` and `mapping` fields. + evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Updated evaluator family. + Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` and `scope` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. + A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. target : typing.Optional[EvaluationRuleTarget] Updated target object type. @@ -738,6 +760,7 @@ async def main() -> None: _response = await self._raw_client.update( evaluation_rule_id, name=name, + evaluators=evaluators, evaluator=evaluator, target=target, enabled=enabled, diff --git a/langfuse/api/unstable/evaluation_rules/raw_client.py b/langfuse/api/unstable/evaluation_rules/raw_client.py index 803f76d63..607f87f3c 100644 --- a/langfuse/api/unstable/evaluation_rules/raw_client.py +++ b/langfuse/api/unstable/evaluation_rules/raw_client.py @@ -44,6 +44,9 @@ ) from ..errors.errors.unprocessable_content_error import UnprocessableContentError from ..errors.types.public_api_error import PublicApiError +from .types.create_evaluation_rule_evaluator_assignment import ( + CreateEvaluationRuleEvaluatorAssignment, +) from .types.create_evaluation_rule_request import CreateEvaluationRuleRequest from .types.delete_evaluation_rule_response import DeleteEvaluationRuleResponse from .types.evaluation_rule import EvaluationRule @@ -75,19 +78,21 @@ def create( Key rules: - `name` must be unique within the project for public evaluation rules - `target` must be `observation` or `experiment` - - `evaluator.name` + `evaluator.scope` must identify an existing evaluator family returned by the evaluator endpoints + - provide either the compatibility `evaluator` field or the new `evaluators` array, never both + - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - Langfuse resolves that family to its latest version before saving the evaluation rule - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - for `llm_as_judge` evaluators, every evaluator prompt variable must be mapped exactly once + - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping + - the effective mapping must map every evaluator prompt variable exactly once - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 50 evaluation rules can be effectively active in one project at the same time + - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) If an evaluation rule with the same `name` already exists in the project, the API returns `409`. In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - If enabling this resource would exceed the 50-active limit, the API also returns `409`. + If enabling this resource would exceed the 500-active limit, the API also returns `409`. In that case, disable or pause another active evaluation rule before enabling a new one. Current scope: @@ -670,6 +675,9 @@ def update( evaluation_rule_id: str, *, name: typing.Optional[str] = OMIT, + evaluators: typing.Optional[ + typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] + ] = OMIT, evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, target: typing.Optional[EvaluationRuleTarget] = OMIT, enabled: typing.Optional[bool] = OMIT, @@ -695,7 +703,7 @@ def update( - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 50 active evaluation rules, the API returns `409` + - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` Recovery guidance: - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` @@ -709,11 +717,17 @@ def update( name : typing.Optional[str] Updated deployment name. + evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] + Full replacement of the rule's evaluator assignments: entries that are + not listed are detached. + + Mutually exclusive with the deprecated `evaluator` and `mapping` fields. + evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Updated evaluator family. + Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` and `scope` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. + A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. target : typing.Optional[EvaluationRuleTarget] Updated target object type. @@ -746,6 +760,11 @@ def update( method="PATCH", json={ "name": name, + "evaluators": convert_and_respect_annotation_metadata( + object_=evaluators, + annotation=typing.Sequence[CreateEvaluationRuleEvaluatorAssignment], + direction="write", + ), "evaluator": convert_and_respect_annotation_metadata( object_=evaluator, annotation=EvaluationRuleEvaluatorReference, @@ -1138,19 +1157,21 @@ async def create( Key rules: - `name` must be unique within the project for public evaluation rules - `target` must be `observation` or `experiment` - - `evaluator.name` + `evaluator.scope` must identify an existing evaluator family returned by the evaluator endpoints + - provide either the compatibility `evaluator` field or the new `evaluators` array, never both + - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - Langfuse resolves that family to its latest version before saving the evaluation rule - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - for `llm_as_judge` evaluators, every evaluator prompt variable must be mapped exactly once + - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping + - the effective mapping must map every evaluator prompt variable exactly once - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 50 evaluation rules can be effectively active in one project at the same time + - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) If an evaluation rule with the same `name` already exists in the project, the API returns `409`. In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - If enabling this resource would exceed the 50-active limit, the API also returns `409`. + If enabling this resource would exceed the 500-active limit, the API also returns `409`. In that case, disable or pause another active evaluation rule before enabling a new one. Current scope: @@ -1733,6 +1754,9 @@ async def update( evaluation_rule_id: str, *, name: typing.Optional[str] = OMIT, + evaluators: typing.Optional[ + typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] + ] = OMIT, evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, target: typing.Optional[EvaluationRuleTarget] = OMIT, enabled: typing.Optional[bool] = OMIT, @@ -1758,7 +1782,7 @@ async def update( - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 50 active evaluation rules, the API returns `409` + - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` Recovery guidance: - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` @@ -1772,11 +1796,17 @@ async def update( name : typing.Optional[str] Updated deployment name. + evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] + Full replacement of the rule's evaluator assignments: entries that are + not listed are detached. + + Mutually exclusive with the deprecated `evaluator` and `mapping` fields. + evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Updated evaluator family. + Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` and `scope` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. + A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. target : typing.Optional[EvaluationRuleTarget] Updated target object type. @@ -1809,6 +1839,11 @@ async def update( method="PATCH", json={ "name": name, + "evaluators": convert_and_respect_annotation_metadata( + object_=evaluators, + annotation=typing.Sequence[CreateEvaluationRuleEvaluatorAssignment], + direction="write", + ), "evaluator": convert_and_respect_annotation_metadata( object_=evaluator, annotation=EvaluationRuleEvaluatorReference, diff --git a/langfuse/api/unstable/evaluation_rules/types/__init__.py b/langfuse/api/unstable/evaluation_rules/types/__init__.py index cd5d4a832..c8198e458 100644 --- a/langfuse/api/unstable/evaluation_rules/types/__init__.py +++ b/langfuse/api/unstable/evaluation_rules/types/__init__.py @@ -10,7 +10,16 @@ CodeEvaluationRuleEvaluatorReference, ) from .create_code_evaluation_rule_request import CreateCodeEvaluationRuleRequest + from .create_evaluation_rule_evaluator_assignment import ( + CreateEvaluationRuleEvaluatorAssignment, + ) + from .create_evaluation_rule_evaluator_reference import ( + CreateEvaluationRuleEvaluatorReference, + ) from .create_evaluation_rule_request import CreateEvaluationRuleRequest + from .create_evaluation_rule_with_evaluators_request import ( + CreateEvaluationRuleWithEvaluatorsRequest, + ) from .create_llm_as_judge_evaluation_rule_request import ( CreateLlmAsJudgeEvaluationRuleRequest, ) @@ -18,11 +27,15 @@ from .evaluation_rule import EvaluationRule from .evaluation_rule_base import EvaluationRuleBase from .evaluation_rule_evaluator import EvaluationRuleEvaluator + from .evaluation_rule_evaluator_assignment import EvaluationRuleEvaluatorAssignment from .evaluation_rule_evaluator_reference import EvaluationRuleEvaluatorReference from .evaluation_rule_time_scope import EvaluationRuleTimeScope from .evaluation_rules import EvaluationRules from .legacy_evaluation_object import LegacyEvaluationObject from .legacy_evaluation_rule import LegacyEvaluationRule + from .legacy_evaluation_rule_evaluator_assignment import ( + LegacyEvaluationRuleEvaluatorAssignment, + ) from .legacy_evaluation_rule_mapping import LegacyEvaluationRuleMapping from .legacy_evaluation_rule_target import LegacyEvaluationRuleTarget from .llm_as_judge_evaluation_rule_evaluator_reference import ( @@ -34,17 +47,22 @@ _dynamic_imports: typing.Dict[str, str] = { "CodeEvaluationRuleEvaluatorReference": ".code_evaluation_rule_evaluator_reference", "CreateCodeEvaluationRuleRequest": ".create_code_evaluation_rule_request", + "CreateEvaluationRuleEvaluatorAssignment": ".create_evaluation_rule_evaluator_assignment", + "CreateEvaluationRuleEvaluatorReference": ".create_evaluation_rule_evaluator_reference", "CreateEvaluationRuleRequest": ".create_evaluation_rule_request", + "CreateEvaluationRuleWithEvaluatorsRequest": ".create_evaluation_rule_with_evaluators_request", "CreateLlmAsJudgeEvaluationRuleRequest": ".create_llm_as_judge_evaluation_rule_request", "DeleteEvaluationRuleResponse": ".delete_evaluation_rule_response", "EvaluationRule": ".evaluation_rule", "EvaluationRuleBase": ".evaluation_rule_base", "EvaluationRuleEvaluator": ".evaluation_rule_evaluator", + "EvaluationRuleEvaluatorAssignment": ".evaluation_rule_evaluator_assignment", "EvaluationRuleEvaluatorReference": ".evaluation_rule_evaluator_reference", "EvaluationRuleTimeScope": ".evaluation_rule_time_scope", "EvaluationRules": ".evaluation_rules", "LegacyEvaluationObject": ".legacy_evaluation_object", "LegacyEvaluationRule": ".legacy_evaluation_rule", + "LegacyEvaluationRuleEvaluatorAssignment": ".legacy_evaluation_rule_evaluator_assignment", "LegacyEvaluationRuleMapping": ".legacy_evaluation_rule_mapping", "LegacyEvaluationRuleTarget": ".legacy_evaluation_rule_target", "LlmAsJudgeEvaluationRuleEvaluatorReference": ".llm_as_judge_evaluation_rule_evaluator_reference", @@ -84,17 +102,22 @@ def __dir__(): __all__ = [ "CodeEvaluationRuleEvaluatorReference", "CreateCodeEvaluationRuleRequest", + "CreateEvaluationRuleEvaluatorAssignment", + "CreateEvaluationRuleEvaluatorReference", "CreateEvaluationRuleRequest", + "CreateEvaluationRuleWithEvaluatorsRequest", "CreateLlmAsJudgeEvaluationRuleRequest", "DeleteEvaluationRuleResponse", "EvaluationRule", "EvaluationRuleBase", "EvaluationRuleEvaluator", + "EvaluationRuleEvaluatorAssignment", "EvaluationRuleEvaluatorReference", "EvaluationRuleTimeScope", "EvaluationRules", "LegacyEvaluationObject", "LegacyEvaluationRule", + "LegacyEvaluationRuleEvaluatorAssignment", "LegacyEvaluationRuleMapping", "LegacyEvaluationRuleTarget", "LlmAsJudgeEvaluationRuleEvaluatorReference", diff --git a/langfuse/api/unstable/evaluation_rules/types/code_evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/code_evaluation_rule_evaluator_reference.py index 1c259bab8..813c065b0 100644 --- a/langfuse/api/unstable/evaluation_rules/types/code_evaluation_rule_evaluator_reference.py +++ b/langfuse/api/unstable/evaluation_rules/types/code_evaluation_rule_evaluator_reference.py @@ -4,7 +4,6 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluator_scope import EvaluatorScope class CodeEvaluationRuleEvaluatorReference(UniversalBaseModel): @@ -17,11 +16,6 @@ class CodeEvaluationRuleEvaluatorReference(UniversalBaseModel): Evaluator family name. """ - scope: EvaluatorScope = pydantic.Field() - """ - Whether the evaluator family is project-owned or Langfuse-managed. - """ - type: typing.Literal["code"] = pydantic.Field(default="code") """ Must be `code`. diff --git a/langfuse/api/unstable/evaluation_rules/types/create_code_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/create_code_evaluation_rule_request.py index 08df1f78a..fe3edb606 100644 --- a/langfuse/api/unstable/evaluation_rules/types/create_code_evaluation_rule_request.py +++ b/langfuse/api/unstable/evaluation_rules/types/create_code_evaluation_rule_request.py @@ -21,7 +21,7 @@ class CreateCodeEvaluationRuleRequest(UniversalBaseModel): """ Code evaluator family to use. - Use `name`, `scope`, and `type` from the evaluator endpoints. + Use `name` and `type` from the evaluator endpoints. Langfuse resolves that family to its latest version before saving the rule. """ diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py new file mode 100644 index 000000000..a48961404 --- /dev/null +++ b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ....core.pydantic_utilities import UniversalBaseModel +from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping +from .create_evaluation_rule_evaluator_reference import ( + CreateEvaluationRuleEvaluatorReference, +) + + +class CreateEvaluationRuleEvaluatorAssignment(UniversalBaseModel): + evaluator: CreateEvaluationRuleEvaluatorReference + mapping: typing.Optional[typing.List[EvaluationRuleMapping]] = pydantic.Field( + default=None + ) + """ + Optional rule-specific override. Omit to inherit the evaluator version's default mapping. Omit for code evaluators. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_reference.py new file mode 100644 index 000000000..37a16c62d --- /dev/null +++ b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_reference.py @@ -0,0 +1,19 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ....core.pydantic_utilities import UniversalBaseModel +from ...commons.types.evaluator_type import EvaluatorType + + +class CreateEvaluationRuleEvaluatorReference(UniversalBaseModel): + name: str + type: typing.Optional[EvaluatorType] = pydantic.Field(default=None) + """ + Defaults to `llm_as_judge`. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_request.py index a6504934d..4e77864cd 100644 --- a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_request.py +++ b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_request.py @@ -3,10 +3,15 @@ import typing from .create_code_evaluation_rule_request import CreateCodeEvaluationRuleRequest +from .create_evaluation_rule_with_evaluators_request import ( + CreateEvaluationRuleWithEvaluatorsRequest, +) from .create_llm_as_judge_evaluation_rule_request import ( CreateLlmAsJudgeEvaluationRuleRequest, ) CreateEvaluationRuleRequest = typing.Union[ - CreateLlmAsJudgeEvaluationRuleRequest, CreateCodeEvaluationRuleRequest + CreateLlmAsJudgeEvaluationRuleRequest, + CreateCodeEvaluationRuleRequest, + CreateEvaluationRuleWithEvaluatorsRequest, ] diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_with_evaluators_request.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_with_evaluators_request.py new file mode 100644 index 000000000..1594bd9b3 --- /dev/null +++ b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_with_evaluators_request.py @@ -0,0 +1,34 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ....core.pydantic_utilities import UniversalBaseModel +from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter +from ...commons.types.evaluation_rule_target import EvaluationRuleTarget +from .create_evaluation_rule_evaluator_assignment import ( + CreateEvaluationRuleEvaluatorAssignment, +) + + +class CreateEvaluationRuleWithEvaluatorsRequest(UniversalBaseModel): + name: str = pydantic.Field() + """ + Human-readable deployment name. + """ + + evaluators: typing.List[CreateEvaluationRuleEvaluatorAssignment] = pydantic.Field() + """ + One or more evaluator assignments. Providing the deprecated top-level `evaluator` or `mapping` fields alongside this is rejected with `400`. + + Multiple assignments are supported on writable targets. + """ + + target: EvaluationRuleTarget + enabled: bool + sampling: typing.Optional[float] = None + filter: typing.Optional[typing.List[EvaluationRuleFilter]] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py index b511b4353..715df7765 100644 --- a/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py +++ b/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py @@ -22,7 +22,7 @@ class CreateLlmAsJudgeEvaluationRuleRequest(UniversalBaseModel): """ LLM-as-judge evaluator family to use. - Use `name`, `scope`, and `type` from the evaluator endpoints. If `type` is omitted, Langfuse defaults it to `llm_as_judge` for backwards compatibility. + Use `name` and `type` from the evaluator endpoints. If `type` is omitted, Langfuse defaults it to `llm_as_judge` for backwards compatibility. Langfuse resolves that family to its latest version before saving the rule. """ diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py index e7a098fb0..6fabadd4e 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py @@ -4,9 +4,10 @@ import pydantic from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping +from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping from ...commons.types.evaluation_rule_target import EvaluationRuleTarget from .evaluation_rule_base import EvaluationRuleBase +from .evaluation_rule_evaluator_assignment import EvaluationRuleEvaluatorAssignment class EvaluationRule(EvaluationRuleBase): @@ -17,17 +18,17 @@ class EvaluationRule(EvaluationRuleBase): from langfuse.unstable.commons import ( EvaluationRuleFilter_StringOptions, - EvaluationRuleMapping, EvaluationRuleMappingSource, EvaluationRuleOptionsFilterOperator, + EvaluationRuleReadMapping, EvaluationRuleStatus, EvaluationRuleTarget, - EvaluatorScope, EvaluatorType, ) from langfuse.unstable.evaluation_rules import ( EvaluationRule, EvaluationRuleEvaluator, + EvaluationRuleEvaluatorAssignment, ) EvaluationRule( @@ -36,9 +37,27 @@ class EvaluationRule(EvaluationRuleBase): evaluator=EvaluationRuleEvaluator( id="evaltmpl_123", name="answer-correctness", - scope=EvaluatorScope.PROJECT, type=EvaluatorType.LLM_AS_JUDGE, ), + evaluators=[ + EvaluationRuleEvaluatorAssignment( + evaluator=EvaluationRuleEvaluator( + id="evaltmpl_123", + name="answer-correctness", + type=EvaluatorType.LLM_AS_JUDGE, + ), + mapping=[ + EvaluationRuleReadMapping( + variable="input", + source=EvaluationRuleMappingSource.INPUT, + ), + EvaluationRuleReadMapping( + variable="output", + source=EvaluationRuleMappingSource.OUTPUT, + ), + ], + ) + ], target=EvaluationRuleTarget.OBSERVATION, enabled=True, status=EvaluationRuleStatus.ACTIVE, @@ -51,11 +70,11 @@ class EvaluationRule(EvaluationRuleBase): ) ], mapping=[ - EvaluationRuleMapping( + EvaluationRuleReadMapping( variable="input", source=EvaluationRuleMappingSource.INPUT, ), - EvaluationRuleMapping( + EvaluationRuleReadMapping( variable="output", source=EvaluationRuleMappingSource.OUTPUT, ), @@ -69,6 +88,11 @@ class EvaluationRule(EvaluationRuleBase): ) """ + evaluators: typing.List[EvaluationRuleEvaluatorAssignment] = pydantic.Field() + """ + Evaluators attached to this rule in deterministic assignment order. A `null` mapping inherits the evaluator version's default mapping. + """ + target: EvaluationRuleTarget = pydantic.Field() """ Target object type that should trigger scoring. @@ -79,9 +103,9 @@ class EvaluationRule(EvaluationRuleBase): List of filter conditions used to decide whether a target should be evaluated. """ - mapping: typing.List[EvaluationRuleMapping] = pydantic.Field() + mapping: typing.List[EvaluationRuleReadMapping] = pydantic.Field() """ - Variable mappings used to populate evaluator runtime variables from the live target object. + Deprecated compatibility alias containing the effective mapping for `evaluators[0]`. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py index e504f250b..95ec471c9 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py @@ -38,13 +38,13 @@ class EvaluationRuleBase(UniversalBaseModel): Human-readable deployment name. This is independent from the evaluator name. """ - evaluator: EvaluationRuleEvaluator = pydantic.Field() + evaluator: typing.Optional[EvaluationRuleEvaluator] = pydantic.Field(default=None) """ - Evaluator currently used by this rule. + Deprecated compatibility alias for `evaluators[0].evaluator`, or `null` + when the rule currently has no evaluator assignments. - `name` and `scope` identify the evaluator family conceptually. - `id` is the currently active evaluator version in that family. - If you create a newer project version with the same evaluator name later, existing evaluation rules are moved to it automatically. + `id` identifies the evaluator family. The rule automatically uses the + latest available version of that evaluator. """ enabled: bool = pydantic.Field() diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py index c27497c9d..a2f36d885 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py @@ -4,7 +4,6 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluator_scope import EvaluatorScope from ...commons.types.evaluator_type import EvaluatorType @@ -12,8 +11,8 @@ class EvaluationRuleEvaluator(UniversalBaseModel): """ Resolved evaluator currently used by the evaluation rule. - `id` is the exact active evaluator version. - `name`, `scope`, and `type` identify the evaluator family conceptually. + `id` identifies the evaluator family. Evaluation runs automatically use + the latest available evaluator version. """ id: str = pydantic.Field() @@ -26,11 +25,6 @@ class EvaluationRuleEvaluator(UniversalBaseModel): Evaluator family name. """ - scope: EvaluatorScope = pydantic.Field() - """ - Whether the evaluator family is project-owned or Langfuse-managed. - """ - type: EvaluatorType = pydantic.Field() """ Evaluator engine type. diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py new file mode 100644 index 000000000..2062dcbe3 --- /dev/null +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py @@ -0,0 +1,22 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ....core.pydantic_utilities import UniversalBaseModel +from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping +from .evaluation_rule_evaluator import EvaluationRuleEvaluator + + +class EvaluationRuleEvaluatorAssignment(UniversalBaseModel): + evaluator: EvaluationRuleEvaluator + mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = pydantic.Field( + default=None + ) + """ + Rule-specific override, or `null` to inherit the evaluator version's default mapping. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_reference.py index a2a38723d..367ae6bc5 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_reference.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_reference.py @@ -4,14 +4,13 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluator_scope import EvaluatorScope class EvaluationRuleEvaluatorReference(UniversalBaseModel): """ Evaluator family reference used when updating an evaluation rule. - `name` and `scope` identify the evaluator family in the authenticated project context. + `name` identifies the evaluator family in the authenticated project context. A rule's evaluator type cannot be changed, so this reference does not accept a `type`; the family must match the rule's current evaluator type. """ @@ -20,11 +19,6 @@ class EvaluationRuleEvaluatorReference(UniversalBaseModel): Evaluator family name. """ - scope: EvaluatorScope = pydantic.Field() - """ - Whether the evaluator family is project-owned or Langfuse-managed. - """ - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True ) diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py index b9738f8b7..c2aeb3a3d 100644 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py @@ -8,6 +8,9 @@ from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter from .evaluation_rule_base import EvaluationRuleBase from .evaluation_rule_time_scope import EvaluationRuleTimeScope +from .legacy_evaluation_rule_evaluator_assignment import ( + LegacyEvaluationRuleEvaluatorAssignment, +) from .legacy_evaluation_rule_mapping import LegacyEvaluationRuleMapping from .legacy_evaluation_rule_target import LegacyEvaluationRuleTarget @@ -16,7 +19,12 @@ class LegacyEvaluationRule(EvaluationRuleBase): """ Legacy trace- or dataset-level evaluation rule returned by list and get for migration. - This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation that each evaluator variable previously read from. Its filters use the persisted legacy filter format so migration clients can read the configuration without losing information. + This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each evaluator variable. + """ + + evaluators: typing.List[LegacyEvaluationRuleEvaluatorAssignment] = pydantic.Field() + """ + Evaluators attached to this rule in deterministic assignment order. """ target: LegacyEvaluationRuleTarget diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py new file mode 100644 index 000000000..146f49f7c --- /dev/null +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py @@ -0,0 +1,17 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ....core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_evaluator import EvaluationRuleEvaluator +from .legacy_evaluation_rule_mapping import LegacyEvaluationRuleMapping + + +class LegacyEvaluationRuleEvaluatorAssignment(UniversalBaseModel): + evaluator: EvaluationRuleEvaluator + mapping: typing.Optional[typing.List[LegacyEvaluationRuleMapping]] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_mapping.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_mapping.py index 79a8856ca..b740b0e29 100644 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_mapping.py +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_mapping.py @@ -14,18 +14,10 @@ class LegacyEvaluationRuleMapping(UniversalBaseModel): Maps one evaluator variable to a trace, dataset item, or field on a named observation in a legacy rule. """ - variable: str = pydantic.Field() - """ - Evaluator prompt variable populated by this mapping. - """ - + variable: str langfuse_object: typing_extensions.Annotated[ LegacyEvaluationObject, FieldMetadata(alias="langfuseObject") - ] = pydantic.Field() - """ - Trace, dataset item, or observation type from which the value is read. - """ - + ] object_name: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="objectName") ] = pydantic.Field(default=None) @@ -33,17 +25,10 @@ class LegacyEvaluationRuleMapping(UniversalBaseModel): Observation name to match, or `null` when `langfuseObject` is `trace` or `dataset_item`. """ - source: str = pydantic.Field() - """ - Stored field selected from the trace, dataset item, or observation. - """ - + source: str json_path: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="jsonPath") - ] = pydantic.Field(default=None) - """ - Optional JSONPath selector applied to the selected field. - """ + ] = None model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py index ca57fe517..7747b7b10 100644 --- a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py +++ b/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py @@ -4,7 +4,6 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluator_scope import EvaluatorScope from .llm_as_judge_evaluator_type import LlmAsJudgeEvaluatorType @@ -18,11 +17,6 @@ class LlmAsJudgeEvaluationRuleEvaluatorReference(UniversalBaseModel): Evaluator family name. """ - scope: EvaluatorScope = pydantic.Field() - """ - Whether the evaluator family is project-owned or Langfuse-managed. - """ - type: typing.Optional[LlmAsJudgeEvaluatorType] = pydantic.Field(default=None) """ Evaluator engine type. Defaults to `llm_as_judge` when omitted. diff --git a/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py index 40e5043a6..e81c52c99 100644 --- a/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py +++ b/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py @@ -7,6 +7,9 @@ from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ...commons.types.evaluation_rule_target import EvaluationRuleTarget +from .create_evaluation_rule_evaluator_assignment import ( + CreateEvaluationRuleEvaluatorAssignment, +) from .evaluation_rule_evaluator_reference import EvaluationRuleEvaluatorReference @@ -19,6 +22,8 @@ class UpdateEvaluationRuleRequest(UniversalBaseModel): Practical guidance: - If you only want to rename the rule or change sampling, send just those fields. + - To add, remove, or remap evaluators, send `evaluators`. It replaces the whole assignment set, so include every evaluator the rule should keep. + - `evaluators` cannot be combined with the deprecated `evaluator`/`mapping` pair, which only ever addressed the first assignment. - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the evaluator variables. - If you change `target` for an LLM-as-judge rule, usually send both `filter` and `mapping` in the same request. - For code evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically. @@ -30,14 +35,24 @@ class UpdateEvaluationRuleRequest(UniversalBaseModel): Updated deployment name. """ + evaluators: typing.Optional[ + typing.List[CreateEvaluationRuleEvaluatorAssignment] + ] = pydantic.Field(default=None) + """ + Full replacement of the rule's evaluator assignments: entries that are + not listed are detached. + + Mutually exclusive with the deprecated `evaluator` and `mapping` fields. + """ + evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = pydantic.Field( default=None ) """ - Updated evaluator family. + Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` and `scope` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. + A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. """ target: typing.Optional[EvaluationRuleTarget] = pydantic.Field(default=None) diff --git a/langfuse/api/unstable/evaluators/client.py b/langfuse/api/unstable/evaluators/client.py index 5a5a03944..272792a5e 100644 --- a/langfuse/api/unstable/evaluators/client.py +++ b/langfuse/api/unstable/evaluators/client.py @@ -45,13 +45,14 @@ def create( Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. + - The evaluator `id` remains stable across versions. + - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. + 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. Code evaluator validation: - At creation, Langfuse only validates the request shape @@ -81,10 +82,13 @@ def create( -------- from langfuse import LangfuseAPI from langfuse.unstable.commons import ( + EvaluationRuleMapping, + EvaluationRuleMappingSource, EvaluatorModelConfig, EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PublicNumericEvaluatorOutputScoreDefinition, ) from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge @@ -105,7 +109,7 @@ def create( reasoning=EvaluatorOutputFieldDefinition( description="Explain why the score was assigned.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Correctness score between 0 and 1.", ), ), @@ -113,6 +117,16 @@ def create( provider="openai", model="gpt-4.1-mini", ), + mapping=[ + EvaluationRuleMapping( + variable="input", + source=EvaluationRuleMappingSource.INPUT, + ), + EvaluationRuleMapping( + variable="output", + source=EvaluationRuleMappingSource.OUTPUT, + ), + ], ), ) """ @@ -133,8 +147,7 @@ def list( Important behavior: - This endpoint returns the latest version of each available evaluator. - - Results can include evaluators from your project and Langfuse-managed evaluators. - - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. + - Every evaluator is owned by the authenticated project. Parameters ---------- @@ -179,7 +192,7 @@ def get( """ Get one evaluator by `id`. - Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. + This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- @@ -222,9 +235,8 @@ def delete( Delete an evaluator. Important behavior: - - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. + - This deletes the evaluator including all of its stored versions. + - Evaluation rule assignments referencing the evaluator are also deleted. - Scores already produced by the evaluator are not deleted. Parameters @@ -292,13 +304,14 @@ async def create( Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. + - The evaluator `id` remains stable across versions. + - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. + 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. Code evaluator validation: - At creation, Langfuse only validates the request shape @@ -330,10 +343,13 @@ async def create( from langfuse import AsyncLangfuseAPI from langfuse.unstable.commons import ( + EvaluationRuleMapping, + EvaluationRuleMappingSource, EvaluatorModelConfig, EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PublicNumericEvaluatorOutputScoreDefinition, ) from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge @@ -357,7 +373,7 @@ async def main() -> None: reasoning=EvaluatorOutputFieldDefinition( description="Explain why the score was assigned.", ), - score=EvaluatorOutputFieldDefinition( + score=PublicNumericEvaluatorOutputScoreDefinition( description="Correctness score between 0 and 1.", ), ), @@ -365,6 +381,16 @@ async def main() -> None: provider="openai", model="gpt-4.1-mini", ), + mapping=[ + EvaluationRuleMapping( + variable="input", + source=EvaluationRuleMappingSource.INPUT, + ), + EvaluationRuleMapping( + variable="output", + source=EvaluationRuleMappingSource.OUTPUT, + ), + ], ), ) @@ -388,8 +414,7 @@ async def list( Important behavior: - This endpoint returns the latest version of each available evaluator. - - Results can include evaluators from your project and Langfuse-managed evaluators. - - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. + - Every evaluator is owned by the authenticated project. Parameters ---------- @@ -442,7 +467,7 @@ async def get( """ Get one evaluator by `id`. - Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. + This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- @@ -495,9 +520,8 @@ async def delete( Delete an evaluator. Important behavior: - - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. + - This deletes the evaluator including all of its stored versions. + - Evaluation rule assignments referencing the evaluator are also deleted. - Scores already produced by the evaluator are not deleted. Parameters diff --git a/langfuse/api/unstable/evaluators/raw_client.py b/langfuse/api/unstable/evaluators/raw_client.py index ab6f2fd10..8e08ee8e4 100644 --- a/langfuse/api/unstable/evaluators/raw_client.py +++ b/langfuse/api/unstable/evaluators/raw_client.py @@ -70,13 +70,14 @@ def create( Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. + - The evaluator `id` remains stable across versions. + - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. + 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. Code evaluator validation: - At creation, Langfuse only validates the request shape @@ -289,8 +290,7 @@ def list( Important behavior: - This endpoint returns the latest version of each available evaluator. - - Results can include evaluators from your project and Langfuse-managed evaluators. - - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. + - Every evaluator is owned by the authenticated project. Parameters ---------- @@ -469,7 +469,7 @@ def get( """ Get one evaluator by `id`. - Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. + This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- @@ -653,9 +653,8 @@ def delete( Delete an evaluator. Important behavior: - - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. + - This deletes the evaluator including all of its stored versions. + - Evaluation rule assignments referencing the evaluator are also deleted. - Scores already produced by the evaluator are not deleted. Parameters @@ -740,17 +739,6 @@ def delete( ), ), ) - if _response.status_code == 409: - raise ConflictError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), @@ -862,13 +850,14 @@ async def create( Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. + - The evaluator `id` remains stable across versions. + - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. + 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. Code evaluator validation: - At creation, Langfuse only validates the request shape @@ -1081,8 +1070,7 @@ async def list( Important behavior: - This endpoint returns the latest version of each available evaluator. - - Results can include evaluators from your project and Langfuse-managed evaluators. - - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. + - Every evaluator is owned by the authenticated project. Parameters ---------- @@ -1261,7 +1249,7 @@ async def get( """ Get one evaluator by `id`. - Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. + This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- @@ -1445,9 +1433,8 @@ async def delete( Delete an evaluator. Important behavior: - - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. + - This deletes the evaluator including all of its stored versions. + - Evaluation rule assignments referencing the evaluator are also deleted. - Scores already produced by the evaluator are not deleted. Parameters @@ -1532,17 +1519,6 @@ async def delete( ), ), ) - if _response.status_code == 409: - raise ConflictError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), diff --git a/langfuse/api/unstable/evaluators/types/create_evaluator_request.py b/langfuse/api/unstable/evaluators/types/create_evaluator_request.py index a866aa4c5..94b7ad620 100644 --- a/langfuse/api/unstable/evaluators/types/create_evaluator_request.py +++ b/langfuse/api/unstable/evaluators/types/create_evaluator_request.py @@ -11,6 +11,7 @@ from ...commons.types.code_evaluator_source_code_language import ( CodeEvaluatorSourceCodeLanguage, ) +from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ...commons.types.evaluator_model_config import EvaluatorModelConfig from ...commons.types.evaluator_output_definition import EvaluatorOutputDefinition @@ -20,7 +21,7 @@ class CreateEvaluatorRequest_LlmAsJudge(UniversalBaseModel): Request body for creating an evaluator. If the same `name` already exists in your project, Langfuse creates the next version and returns it. - Existing evaluation rules in the same project are then moved to that new latest version automatically. + Existing evaluation rules automatically use the latest evaluator version. If `type` is omitted, Langfuse defaults it to `llm_as_judge` for backwards compatibility. """ @@ -33,6 +34,7 @@ class CreateEvaluatorRequest_LlmAsJudge(UniversalBaseModel): model_config_: typing_extensions.Annotated[ typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") ] = None + mapping: typing.Optional[typing.List[EvaluationRuleMapping]] = None model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True @@ -44,7 +46,7 @@ class CreateEvaluatorRequest_Code(UniversalBaseModel): Request body for creating an evaluator. If the same `name` already exists in your project, Langfuse creates the next version and returns it. - Existing evaluation rules in the same project are then moved to that new latest version automatically. + Existing evaluation rules automatically use the latest evaluator version. If `type` is omitted, Langfuse defaults it to `llm_as_judge` for backwards compatibility. """ diff --git a/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py b/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py index 09e121b1b..456a142f9 100644 --- a/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py +++ b/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py @@ -6,6 +6,7 @@ import typing_extensions from ....core.pydantic_utilities import UniversalBaseModel from ....core.serialization import FieldMetadata +from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ...commons.types.evaluator_model_config import EvaluatorModelConfig from ...commons.types.evaluator_output_definition import EvaluatorOutputDefinition @@ -38,6 +39,13 @@ class CreateLlmAsJudgeEvaluatorRequest(UniversalBaseModel): Optional explicit model configuration. Omit or set to `null` to use the project default evaluation model. """ + mapping: typing.Optional[typing.List[EvaluationRuleMapping]] = pydantic.Field( + default=None + ) + """ + Optional default variable mapping inherited by rule assignments that do not provide an override. + """ + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True ) diff --git a/langfuse/api/unstable/evaluators/types/evaluator.py b/langfuse/api/unstable/evaluators/types/evaluator.py index 69295e0fd..61995dada 100644 --- a/langfuse/api/unstable/evaluators/types/evaluator.py +++ b/langfuse/api/unstable/evaluators/types/evaluator.py @@ -12,8 +12,8 @@ from ...commons.types.code_evaluator_source_code_language import ( CodeEvaluatorSourceCodeLanguage, ) +from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping from ...commons.types.evaluator_model_config import EvaluatorModelConfig -from ...commons.types.evaluator_scope import EvaluatorScope from ...commons.types.public_evaluator_output_definition import ( PublicEvaluatorOutputDefinition, ) @@ -33,7 +33,8 @@ class Evaluator_LlmAsJudge(UniversalBaseModel): Versioning behavior: - `GET /evaluators` returns the latest version of each available evaluator. - - `GET /evaluators/{id}` can return an older version. + - `GET /evaluators/{id}` returns the latest version. + - `id` identifies the evaluator family and remains stable when a new version is created. - Evaluation rules always run against the latest version for the selected evaluator name within the same source (`project` or `managed`). """ @@ -48,8 +49,8 @@ class Evaluator_LlmAsJudge(UniversalBaseModel): id: str name: str version: int - scope: EvaluatorScope variables: typing.List[str] + mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = None evaluation_rule_count: typing_extensions.Annotated[ int, FieldMetadata(alias="evaluationRuleCount") ] @@ -79,7 +80,8 @@ class Evaluator_Code(UniversalBaseModel): Versioning behavior: - `GET /evaluators` returns the latest version of each available evaluator. - - `GET /evaluators/{id}` can return an older version. + - `GET /evaluators/{id}` returns the latest version. + - `id` identifies the evaluator family and remains stable when a new version is created. - Evaluation rules always run against the latest version for the selected evaluator name within the same source (`project` or `managed`). """ @@ -91,8 +93,8 @@ class Evaluator_Code(UniversalBaseModel): id: str name: str version: int - scope: EvaluatorScope variables: typing.List[str] + mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = None evaluation_rule_count: typing_extensions.Annotated[ int, FieldMetadata(alias="evaluationRuleCount") ] diff --git a/langfuse/api/unstable/evaluators/types/evaluator_base.py b/langfuse/api/unstable/evaluators/types/evaluator_base.py index 7a8362657..00ff20a11 100644 --- a/langfuse/api/unstable/evaluators/types/evaluator_base.py +++ b/langfuse/api/unstable/evaluators/types/evaluator_base.py @@ -7,13 +7,13 @@ import typing_extensions from ....core.pydantic_utilities import UniversalBaseModel from ....core.serialization import FieldMetadata -from ...commons.types.evaluator_scope import EvaluatorScope +from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping class EvaluatorBase(UniversalBaseModel): id: str = pydantic.Field() """ - Identifier of this evaluator. + Stable identifier of this evaluator across all versions. """ name: str = pydantic.Field() @@ -26,11 +26,6 @@ class EvaluatorBase(UniversalBaseModel): Version number of this evaluator. """ - scope: EvaluatorScope = pydantic.Field() - """ - Where this evaluator comes from: your project or Langfuse-managed defaults. - """ - variables: typing.List[str] = pydantic.Field() """ Variables that can be mapped when creating an evaluation rule. @@ -38,11 +33,22 @@ class EvaluatorBase(UniversalBaseModel): LLM evaluators require every variable to be mapped exactly once. Code evaluators always expose the fixed runtime payload fields and Langfuse maps them automatically. """ + mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = pydantic.Field( + default=None + ) + """ + Default variable mapping for this evaluator version, or `null` when no default is configured. + + An entry's `source` is `null` when that variable was never fully configured, and sources + are not restricted by rule `target` here, because the default is stored on the evaluator + rather than on any one rule. + """ + evaluation_rule_count: typing_extensions.Annotated[ int, FieldMetadata(alias="evaluationRuleCount") ] = pydantic.Field() """ - Number of evaluation rules in the project that currently use this evaluator version. + Number of evaluation rules in the project that currently use this evaluator. """ created_at: typing_extensions.Annotated[