Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions backend/modules/evaluation/application/eval_openapi_app.go
Original file line number Diff line number Diff line change
Expand Up @@ -1680,14 +1680,12 @@ func (e *EvalOpenAPIApplication) GetEvalTargetOutputFieldContentOApi(ctx context
return nil, errorx.NewByCode(errno.ResourceNotFoundCode, errorx.WithExtraMsg("eval target record not found for the given experiment result"))
}

// ★ 跨空间共享: 评测对象执行记录随执行落在冻结的来源空间(TargetSpaceID>0), 按 space_id 严格过滤;
// 用调用方空间读会查不到 record → target 输出/session 丢失。按来源空间读。
recordSpaceID := req.GetWorkspaceID()
if exptDetail, gErr := e.manager.GetDetail(ctx, req.GetExperimentID(), req.GetWorkspaceID(), entity.NewSession(ctx)); gErr == nil && exptDetail != nil && exptDetail.TargetSpaceID > 0 {
recordSpaceID = exptDetail.TargetSpaceID
}

record, err := e.targetSvc.GetRecordByID(ctx, recordSpaceID, targetRecordID)
record, err := service.GetExperimentTargetRecord(ctx, e.targetSvc, req.GetWorkspaceID(), recordSpaceID, targetRecordID)
if err != nil {
return nil, err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package service

import (
"context"
"errors"
"fmt"
"strconv"
"strings"
Expand Down Expand Up @@ -1379,7 +1380,7 @@ func (e *ExptMangerImpl) listItemTurnIDs(ctx context.Context, spaceID, exptID in
// 传入的 itemIDs 在 Terminated 分支是被终止的 item、在其它终态分支是未完成的 item。
//
// targetSourceSpaceID: 跨空间共享场景下评测对象的来源空间 (Expt.TargetSpaceID, 0=同消费方空间);
// EvalTargetRecord 落库时 SpaceID = 来源空间, 用消费方 spaceID 查会漏, 沙箱清不掉。
// Both consumer-owned records and legacy source-owned attempts must be reclaimed.
func (e *ExptMangerImpl) terminateSandboxExecutesForCancelledItems(ctx context.Context, spaceID, targetSourceSpaceID, exptID int64, exptRunID *int64, terminatedItemIDs []int64, waitForDestroy ...bool) error {
if e.evalTargetService == nil || exptRunID == nil || *exptRunID <= 0 || len(terminatedItemIDs) == 0 {
if len(terminatedItemIDs) > 0 && len(waitForDestroy) > 0 && waitForDestroy[0] {
Expand All @@ -1390,12 +1391,16 @@ func (e *ExptMangerImpl) terminateSandboxExecutesForCancelledItems(ctx context.C
if lifecycle, ok := e.evalTargetService.(interface {
TerminateSandboxAttemptsForRunItems(context.Context, int64, int64, []int64, ...bool) error
}); ok {
if err := lifecycle.TerminateSandboxAttemptsForRunItems(ctx, resolveLoadSpaceID(spaceID, targetSourceSpaceID), *exptRunID, terminatedItemIDs, waitForDestroy...); err != nil {
logs.CtxWarn(ctx, "[SandboxDestroy] discover pre-dispatch attempts failed, run_id=%d, err=%v", *exptRunID, err)
if len(waitForDestroy) > 0 && waitForDestroy[0] {
return err
var cleanupErr error
for _, recordSpaceID := range ExperimentTargetRecordSpaces(spaceID, targetSourceSpaceID) {
if err := lifecycle.TerminateSandboxAttemptsForRunItems(ctx, recordSpaceID, *exptRunID, terminatedItemIDs, waitForDestroy...); err != nil {
logs.CtxWarn(ctx, "[SandboxDestroy] discover pre-dispatch attempts failed, run_id=%d, err=%v", *exptRunID, err)
cleanupErr = errors.Join(cleanupErr, err)
}
}
if cleanupErr != nil && len(waitForDestroy) > 0 && waitForDestroy[0] {
return cleanupErr
}
}
turnRunLogs, err := e.turnResultRepo.MGetItemTurnRunLogs(ctx, exptID, *exptRunID, terminatedItemIDs, spaceID)
if err != nil {
Expand All @@ -1416,12 +1421,9 @@ func (e *ExptMangerImpl) terminateSandboxExecutesForCancelledItems(ctx context.C
for id := range recordIDSet {
recordIDs = append(recordIDs, id)
}
// 跨空间: EvalTargetRecord.SpaceID = 来源空间, TerminateAsyncRecordsAndDestroySandbox 的
// List/Version/Destroy 全链路都用它,否则 space 过滤把 record 过滤空、Destroy RPC 发不出。
targetSpaceID := resolveLoadSpaceID(spaceID, targetSourceSpaceID)
return e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox(
ctx,
targetSpaceID,

return terminateExperimentTargetRecords(
ctx, e.evalTargetService, spaceID, targetSourceSpaceID,
recordIDs,
int32(errno.AsyncEvalTargetTerminatedCode),
"async eval target terminated: experiment cancelled",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3892,7 +3892,7 @@ func TestExptMangerImpl_emitSandboxAgentExperimentFinished(t *testing.T) {

// TestExptMangerImpl_terminateSandboxExecutesForCancelledItems_CrossSpace 覆盖实验取消 / 终态时
// 沙箱销毁传给 TerminateAsyncRecordsAndDestroySandbox 的 spaceID 解析:
// - targetSourceSpaceID > 0 → 用来源空间 (跨空间共享评测对象)
// - targetSourceSpaceID > 0 → 消费方记录与旧来源空间记录均需回收
// - targetSourceSpaceID = 0 → 回退到消费方 spaceID (同空间/老数据)
// - 早退分支: evalTargetService = nil / exptRunID = nil or <=0 / len(itemIDs) = 0 均不发 RPC
// - record 全空 / MGet 出错 亦不发 RPC
Expand All @@ -3903,14 +3903,19 @@ func TestExptMangerImpl_terminateSandboxExecutesForCancelledItems_CrossSpace(t *
consumerSpaceID := int64(3)
itemIDs := []int64{10}

t.Run("targetSourceSpaceID>0 → 用来源空间 (99)", func(t *testing.T) {
t.Run("shared target → clean consumer and legacy source records", func(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mgr := newTestExptManager(ctrl)

mgr.turnResultRepo.(*repoMocks.MockIExptTurnResultRepo).
EXPECT().MGetItemTurnRunLogs(ctx, exptID, *exptRunID, itemIDs, consumerSpaceID).
Return([]*entity.ExptTurnResultRunLog{{ItemID: 10, TargetResultID: 500}}, nil)
mgr.evalTargetService.(*svcMocks.MockIEvalTargetService).
EXPECT().TerminateAsyncRecordsAndDestroySandbox(
gomock.Any(), consumerSpaceID, []int64{500}, int32(errno.AsyncEvalTargetTerminatedCode),
gomock.Any(), false,
).Times(1)
mgr.evalTargetService.(*svcMocks.MockIEvalTargetService).
EXPECT().TerminateAsyncRecordsAndDestroySandbox(
gomock.Any(), int64(99), []int64{500}, int32(errno.AsyncEvalTargetTerminatedCode),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,13 +161,8 @@ func (e *ExptAggrResultServiceImpl) computeEvaluatorAggrGroup(ctx context.Contex
func (e *ExptAggrResultServiceImpl) buildExptTargetMtrAggregatorGroup(ctx context.Context, spaceID, exptID int64) (*targetMtrAggrGroup, error) {
const queryInterval = time.Millisecond * 30

// ★ 跨空间共享: 评测对象执行记录随执行落来源空间(冻结 TargetSpaceID), 聚合读 record 也需按来源空间,
// 否则跨空间实验 target latency/token 聚合取不到 record。载入实验取冻结来源空间。
// 此处 fail-closed: 载入失败必须返回错误让上层重试, 不可退回调用方空间继续 ——
// 跨空间实验(TargetSpaceID>0)下带错空间继续会让 BatchGetRecordByIDs 全查不到,
// latency/token 被静默聚合成 0 并落库, 事后无法与真实 0 区分且不会自愈。
// 例外: 实验已被软删 / spaceID 不匹配时 GetByID 返回 ResourceNotFound —— 这是确定性结果,
// 重试不会好转, fail-closed 会让 MQ 永久重投且锁只能靠 TTL 过期; 此类按调用方空间降级继续。
// Resolve the frozen source workspace for legacy execution records.

exptDO, exptErr := e.experimentRepo.GetByID(ctx, exptID, spaceID)
if exptErr != nil {
if statusErr, ok := errorx.FromStatusError(exptErr); !ok || statusErr.Code() != errno.ResourceNotFoundCode {
Expand Down Expand Up @@ -213,7 +208,7 @@ func (e *ExptAggrResultServiceImpl) buildExptTargetMtrAggregatorGroup(ctx contex
}

for _, resIDs := range gslice.Chunk(targetResultIDs, 50) {
records, err := e.evalTargetSvc.BatchGetRecordByIDs(ctx, recordSpaceID, resIDs)
records, err := BatchGetExperimentTargetRecords(ctx, e.evalTargetSvc, spaceID, recordSpaceID, resIDs)
if err != nil {
return nil, err
}
Expand Down
6 changes: 2 additions & 4 deletions backend/modules/evaluation/domain/service/expt_result_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -2443,10 +2443,8 @@ func (e *ExptResultBuilder) buildTargetOutput(ctx context.Context) error {
targetResultIDs = append(targetResultIDs, turnResult.TargetResultID)
targetResultID2turnResultID[turnResult.TargetResultID] = turnResult.ID
}
// ★ 跨空间共享: 评测对象执行记录(eval_target_record)随执行落在来源空间(冻结 TargetSpaceID),
// 按 space_id 严格过滤; 用调用方空间读会查不到 → target 输出(含 Ext[sandbox_session_id])整体丢失,
// 前端 case 旁不显示 aiosession。按来源空间读 (0=同调用方空间)。
targetRecords, err := e.evalTargetService.BatchGetRecordByIDs(ctx, resolveLoadSpaceID(e.SpaceID, e.exptDO.TargetSpaceID), targetResultIDs)

targetRecords, err := BatchGetExperimentTargetRecords(ctx, e.evalTargetService, e.SpaceID, e.exptDO.TargetSpaceID, targetResultIDs)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Get/BatchGet 兼容旧来源空间,但运行次数与重试日志聚合仍只查实验空间

这里通过 BatchGetExperimentTargetRecords 保留旧 A 记录的读取;同文件 :1438 的 CountItemRunsByRun 与 :1455 的 ListItemRetryRecordsByRun 仍只传 builder.SpaceID=E,两个 DAO 都按 space_id 等值过滤。

旧共享实验的 attempts 在 A,会继续显示 total_runs=0、retry_records 为空;同一 run 跨升级时也只统计 E 中的新 attempts,漏掉 A 中的失败尝试和日志。这是已有共享功能的历史兼容缺口,不是新实验的执行故障。

建议用授权实验的 E 和冻结 A 联合读取、按 record ID 去重。Count 保持原聚合口径:先合并每个 item/turn 的次数,再取 item 下最大的 turn 次数,不能直接相加两个空间各自的 MAX。覆盖旧 A、纯新 E、同 run 混存 A/E 三类数据。

if err != nil {
return err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ func runRetryGenerationPreEval(t *testing.T, f *retryGenerationFixture) ([]*enti
turnRepo.EXPECT().GetItemTurnResults(gomock.Any(), int64(1), int64(10), int64(3)).Return(f.turns, nil).Times(repeats)
turnRepo.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(3), turnIDs).Return(refs, nil).Times(2 * repeats)
idgen.EXPECT().GenMultiIDs(gomock.Any(), len(f.turns)).Return(turnIDs, nil).Times(repeats)
if f.targetSpace != 3 {
targetSvc.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(nil, nil).AnyTimes()
}
targetSvc.EXPECT().GetRecordByID(gomock.Any(), f.targetSpace, int64(201)).DoAndReturn(func(ctx context.Context, _, _ int64) (*entity.EvalTargetRecord, error) {
assert.True(t, contexts.CtxWriteDB(ctx))
return &entity.EvalTargetRecord{ID: 201, ExperimentRunID: 101, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil
Expand Down Expand Up @@ -496,7 +499,7 @@ func TestRetryFailure_SourceLogMembershipPreservesNewFormatIdentity(t *testing.T
}, nil)
turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(101), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{nil, origin}, nil)
sources := &retryEvaluatorSourceLogs{repo: turnRepo, event: &entity.ExptItemEvalEvent{ExptID: 1, EvalSetItemID: 10, SpaceID: 3}, logsByRun: make(map[int64]map[int64]*entity.ExptTurnResultRunLog)}
targetID, got, selectErr := failRetrySelectTurnRunLogRefs(context.Background(), 3, true, tr, targetSvc, recordSvc, []*entity.ExptTurnEvaluatorResultRef{{ExptTurnResultID: tr.ID, EvaluatorVersionID: 401, Alias: "judge-a", EvaluatorResultID: 301}, {ExptTurnResultID: tr.ID, InlineKey: "inline-a", SourceType: int32(entity.EvaluatorRecordSourceTypeInline), EvaluatorResultID: 302}}, sources)
targetID, got, selectErr := failRetrySelectTurnRunLogRefs(context.Background(), 3, 0, true, tr, targetSvc, recordSvc, []*entity.ExptTurnEvaluatorResultRef{{ExptTurnResultID: tr.ID, EvaluatorVersionID: 401, Alias: "judge-a", EvaluatorResultID: 301}, {ExptTurnResultID: tr.ID, InlineKey: "inline-a", SourceType: int32(entity.EvaluatorRecordSourceTypeInline), EvaluatorResultID: 302}}, sources)
require.NoError(t, selectErr)
assert.Equal(t, int64(201), targetID)
if !tt.registered && !tt.inline {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func TestFailRetrySelectTurnRunLogRefs_TargetRequirementControlsEvaluatorReuse(t
}

targetID, evaluatorResults, selectErr := failRetrySelectTurnRunLogRefs(
context.Background(), 1, tt.targetRequired,
context.Background(), 1, 0, tt.targetRequired,
&entity.ExptTurnResult{
EvaluatorResults: &entity.EvaluatorResults{
EvalVerIDToResID: map[int64]int64{10: evaluatorRecordID},
Expand Down Expand Up @@ -425,6 +425,8 @@ func TestExptFailRetryExec_BuildPagePlan_MultiSetGroupsTargetReadsBySourceSpace(
{ItemID: 12, ItemConfig: &entity.ExptItemConfig{TargetSourceSpaceID: rowTargetSpace}},
}, nil)
success := entity.EvalTargetRunStatusSuccess
targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), []int64{110}).Return(nil, nil)
targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), []int64{100, 120}).Return(nil, nil)
targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), topTargetSpace, []int64{110}).Return([]*entity.EvalTargetRecord{{ID: 110, Status: &success}}, nil)
targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), rowTargetSpace, []int64{100, 120}).Return([]*entity.EvalTargetRecord{
{ID: 100, Status: &success},
Expand Down Expand Up @@ -483,6 +485,7 @@ func TestExptFailRetryExec_BuildPagePlan_MultiSetMissingItemRefFallsBackToExperi
ItemID: itemID, ItemConfig: nil,
}}, nil)
success := entity.EvalTargetRunStatusSuccess
targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), []int64{30}).Return(nil, nil)
targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(99), []int64{30}).Return([]*entity.EvalTargetRecord{{ID: 30, Status: &success}}, nil)
idgen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{900}, nil)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1298,7 +1298,7 @@ func (e *ExptRecordEvalModeSubmit) PostEval(ctx context.Context, eiec *entity.Ex
// failRetrySelectTurnRunLogRefs keeps successful stages that belong to the reused target.
func failRetrySelectTurnRunLogRefs(
ctx context.Context,
spaceID int64,
spaceID, sourceSpaceID int64,
targetRequired bool,
tr *entity.ExptTurnResult,
evalTarget IEvalTargetService,
Expand All @@ -1322,7 +1322,7 @@ func failRetrySelectTurnRunLogRefs(
if evalTarget == nil {
return 0, nil, errors.New("eval target service is nil")
}
targetRec, err := evalTarget.GetRecordByID(ctx, spaceID, tr.TargetResultID)
targetRec, err := GetExperimentTargetRecord(ctx, evalTarget, spaceID, sourceSpaceID, tr.TargetResultID)
if err != nil {
return 0, nil, err
}
Expand Down Expand Up @@ -1571,9 +1571,7 @@ func (e *ExptRecordEvalModeFailRetry) PreEval(ctx context.Context, eiec *entity.
runLog.Status = entity.TurnRunState_Processing
runLog.ExptRunID = eiec.Event.ExptRunID
runLog.ErrMsg = ""
// 跨空间共享: Target 记录随执行落来源空间(冻结 TargetSpaceID), 失败重试选引用时须按来源空间读;
// 用调用方空间读会得 nil → 误判 Target 非 Success → 清零 target_result_id 触发无谓重跑 Target。
targetID, evalIDs, err := failRetrySelectTurnRunLogRefs(snapshotCtx, resolveLoadSpaceID(eiec.Event.SpaceID, eiec.TargetSourceSpaceID()), !shouldSkipTargetNode(eiec.Expt), tr, e.evalTargetService, e.evaluatorRecordSvc, refsByTurnResultID[tr.ID], sourceLogs)
targetID, evalIDs, err := failRetrySelectTurnRunLogRefs(snapshotCtx, eiec.Event.SpaceID, eiec.TargetSourceSpaceID(), !shouldSkipTargetNode(eiec.Expt), tr, e.evalTargetService, e.evaluatorRecordSvc, refsByTurnResultID[tr.ID], sourceLogs)
if err != nil {
return fmt.Errorf("%w: %w", errRetryStartDependencyFailure, err)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1678,7 +1678,7 @@ func Test_failRetrySelectTurnRunLogRefs(t *testing.T) {
}}, nil)
}
gotTargetID, gotEvalResults, selectErr := failRetrySelectTurnRunLogRefs(
context.Background(), tt.spaceID, targetRequired, tt.tr, evalTarget, evalRecord, refs, sourceLogs,
context.Background(), tt.spaceID, 0, targetRequired, tt.tr, evalTarget, evalRecord, refs, sourceLogs,
)
if tt.name == "TargetResultID > 0, evalTarget is nil -> dependency error" || tt.name == "TargetResultID > 0, GetRecordByID returns error -> returns 0, nil" {
require.Error(t, selectErr)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -443,9 +443,7 @@ func (e *ExptItemEvalCtxExecutor) buildExptTurnEvalCtx(ctx context.Context, turn
}

if tid := existTurnRunResult.TargetResultID; tid > 0 {
// ★ 跨空间共享: 评测对象执行记录随执行落来源空间(冻结 TargetSpaceID), 按来源空间读;
// 用调用方空间读会得 nil → 异步回调 validateEvalTargetCtx 报 "target result must not be nil"。
targetRecord, err := e.evalTargetService.GetRecordByID(recordCtx, resolveLoadSpaceID(spaceID, eiec.TargetSourceSpaceID()), tid)
targetRecord, err := GetExperimentTargetRecord(recordCtx, e.evalTargetService, spaceID, eiec.TargetSourceSpaceID(), tid)
if err != nil {
return nil, err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -935,13 +935,11 @@ func (e *ExptSchedulerImpl) sweepTerminatedSandboxItems(ctx context.Context, eve
return items, nil, nil
}

// 跨空间共享: EvalTargetRecord.SpaceID 是来源空间, 用消费方 event.SpaceID 查会漏。
var targetSourceSpaceID int64
if expt != nil {
targetSourceSpaceID = expt.TargetSpaceID
}
targetSpaceID := resolveLoadSpaceID(event.SpaceID, targetSourceSpaceID)
terminatedRecordIDs, statusMap := e.evalTargetService.CheckSandboxTerminated(ctx, targetSpaceID, recordIDs)
terminatedRecordIDs, statusMap := checkExperimentSandboxTerminated(ctx, e.evalTargetService, event.SpaceID, targetSourceSpaceID, recordIDs)
if len(terminatedRecordIDs) == 0 {
return items, nil, nil
}
Expand Down Expand Up @@ -983,10 +981,8 @@ func (e *ExptSchedulerImpl) sweepTerminatedSandboxItems(ctx context.Context, eve
event.ExptID, event.ExptRunID, terminatedItemIDs, firstStatus)

// 沙箱已经是终态,Destroy 不需要再发 EndCmd,zombieTimeout=false。
// targetSpaceID 复用上面 CheckSandboxTerminated 用的来源空间, 保持一致。
_ = e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox(
ctx,
targetSpaceID,
_ = terminateExperimentTargetRecords(
ctx, e.evalTargetService, event.SpaceID, targetSourceSpaceID,
terminatedRecordIDs,
int32(errno.SandboxTerminatedBeforeReportCode),
fmt.Sprintf("sandbox execute reached terminal state (%s) before result was reported", firstStatus),
Expand Down Expand Up @@ -1110,17 +1106,12 @@ func (e *ExptSchedulerImpl) terminateZombieEvalTargetRecords(ctx context.Context
return nil
}

// 跨空间共享: EvalTargetRecord 落库时 SpaceID = 评测对象来源空间 (resolveLoadSpaceID 结果),
// 若这里仍用 event.SpaceID (消费方) 去查, DAO 的 SpaceID.Eq 过滤会返回空 → Destroy RPC 根本发不出,
// 导致沙箱在来源空间空跑到 session TTL 兜底才回收。与 expt_run_item_turn_impl.go:130 保持同一套解析。
var targetSourceSpaceID int64
if expt != nil {
targetSourceSpaceID = expt.TargetSpaceID
}
targetSpaceID := resolveLoadSpaceID(event.SpaceID, targetSourceSpaceID)
_ = e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox(
ctx,
targetSpaceID,
_ = terminateExperimentTargetRecords(
ctx, e.evalTargetService, event.SpaceID, targetSourceSpaceID,
recordIDs,
int32(errno.AsyncEvalTargetZombieTimeoutCode),
"async eval target terminated: experiment item exceeded zombie timeout",
Expand Down
Loading
Loading