diff --git a/backend/modules/evaluation/application/eval_openapi_app.go b/backend/modules/evaluation/application/eval_openapi_app.go index 26ccd43ea..0df430ff5 100644 --- a/backend/modules/evaluation/application/eval_openapi_app.go +++ b/backend/modules/evaluation/application/eval_openapi_app.go @@ -1680,14 +1680,12 @@ func (e *EvalOpenAPIApplication) GetEvalTargetOutputFieldContentOApi(ctx context return nil, errorx.NewByCode(errno.ResourceNotFoundCode, errorx.WithExtraMsg("eval target record not found for the given experiment result")) } - // ★ 跨空间共享: 评测对象执行记录随执行落在冻结的来源空间(TargetSpaceID>0), 按 space_id 严格过滤; - // 用调用方空间读会查不到 record → target 输出/session 丢失。按来源空间读。 recordSpaceID := req.GetWorkspaceID() if exptDetail, gErr := e.manager.GetDetail(ctx, req.GetExperimentID(), req.GetWorkspaceID(), entity.NewSession(ctx)); gErr == nil && exptDetail != nil && exptDetail.TargetSpaceID > 0 { recordSpaceID = exptDetail.TargetSpaceID } - record, err := e.targetSvc.GetRecordByID(ctx, recordSpaceID, targetRecordID) + record, err := service.GetExperimentTargetRecord(ctx, e.targetSvc, req.GetWorkspaceID(), recordSpaceID, targetRecordID) if err != nil { return nil, err } diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go index a99201cd5..cafa5bcdc 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go @@ -5,6 +5,7 @@ package service import ( "context" + "errors" "fmt" "strconv" "strings" @@ -1379,7 +1380,7 @@ func (e *ExptMangerImpl) listItemTurnIDs(ctx context.Context, spaceID, exptID in // 传入的 itemIDs 在 Terminated 分支是被终止的 item、在其它终态分支是未完成的 item。 // // targetSourceSpaceID: 跨空间共享场景下评测对象的来源空间 (Expt.TargetSpaceID, 0=同消费方空间); -// EvalTargetRecord 落库时 SpaceID = 来源空间, 用消费方 spaceID 查会漏, 沙箱清不掉。 +// Both consumer-owned records and legacy source-owned attempts must be reclaimed. func (e *ExptMangerImpl) terminateSandboxExecutesForCancelledItems(ctx context.Context, spaceID, targetSourceSpaceID, exptID int64, exptRunID *int64, terminatedItemIDs []int64, waitForDestroy ...bool) error { if e.evalTargetService == nil || exptRunID == nil || *exptRunID <= 0 || len(terminatedItemIDs) == 0 { if len(terminatedItemIDs) > 0 && len(waitForDestroy) > 0 && waitForDestroy[0] { @@ -1390,12 +1391,16 @@ func (e *ExptMangerImpl) terminateSandboxExecutesForCancelledItems(ctx context.C if lifecycle, ok := e.evalTargetService.(interface { TerminateSandboxAttemptsForRunItems(context.Context, int64, int64, []int64, ...bool) error }); ok { - if err := lifecycle.TerminateSandboxAttemptsForRunItems(ctx, resolveLoadSpaceID(spaceID, targetSourceSpaceID), *exptRunID, terminatedItemIDs, waitForDestroy...); err != nil { - logs.CtxWarn(ctx, "[SandboxDestroy] discover pre-dispatch attempts failed, run_id=%d, err=%v", *exptRunID, err) - if len(waitForDestroy) > 0 && waitForDestroy[0] { - return err + var cleanupErr error + for _, recordSpaceID := range ExperimentTargetRecordSpaces(spaceID, targetSourceSpaceID) { + if err := lifecycle.TerminateSandboxAttemptsForRunItems(ctx, recordSpaceID, *exptRunID, terminatedItemIDs, waitForDestroy...); err != nil { + logs.CtxWarn(ctx, "[SandboxDestroy] discover pre-dispatch attempts failed, run_id=%d, err=%v", *exptRunID, err) + cleanupErr = errors.Join(cleanupErr, err) } } + if cleanupErr != nil && len(waitForDestroy) > 0 && waitForDestroy[0] { + return cleanupErr + } } turnRunLogs, err := e.turnResultRepo.MGetItemTurnRunLogs(ctx, exptID, *exptRunID, terminatedItemIDs, spaceID) if err != nil { @@ -1416,12 +1421,9 @@ func (e *ExptMangerImpl) terminateSandboxExecutesForCancelledItems(ctx context.C for id := range recordIDSet { recordIDs = append(recordIDs, id) } - // 跨空间: EvalTargetRecord.SpaceID = 来源空间, TerminateAsyncRecordsAndDestroySandbox 的 - // List/Version/Destroy 全链路都用它,否则 space 过滤把 record 过滤空、Destroy RPC 发不出。 - targetSpaceID := resolveLoadSpaceID(spaceID, targetSourceSpaceID) - return e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox( - ctx, - targetSpaceID, + + return terminateExperimentTargetRecords( + ctx, e.evalTargetService, spaceID, targetSourceSpaceID, recordIDs, int32(errno.AsyncEvalTargetTerminatedCode), "async eval target terminated: experiment cancelled", diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index 759882497..6b5a82723 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -3892,7 +3892,7 @@ func TestExptMangerImpl_emitSandboxAgentExperimentFinished(t *testing.T) { // TestExptMangerImpl_terminateSandboxExecutesForCancelledItems_CrossSpace 覆盖实验取消 / 终态时 // 沙箱销毁传给 TerminateAsyncRecordsAndDestroySandbox 的 spaceID 解析: -// - targetSourceSpaceID > 0 → 用来源空间 (跨空间共享评测对象) +// - targetSourceSpaceID > 0 → 消费方记录与旧来源空间记录均需回收 // - targetSourceSpaceID = 0 → 回退到消费方 spaceID (同空间/老数据) // - 早退分支: evalTargetService = nil / exptRunID = nil or <=0 / len(itemIDs) = 0 均不发 RPC // - record 全空 / MGet 出错 亦不发 RPC @@ -3903,7 +3903,7 @@ func TestExptMangerImpl_terminateSandboxExecutesForCancelledItems_CrossSpace(t * consumerSpaceID := int64(3) itemIDs := []int64{10} - t.Run("targetSourceSpaceID>0 → 用来源空间 (99)", func(t *testing.T) { + t.Run("shared target → clean consumer and legacy source records", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() mgr := newTestExptManager(ctrl) @@ -3911,6 +3911,11 @@ func TestExptMangerImpl_terminateSandboxExecutesForCancelledItems_CrossSpace(t * mgr.turnResultRepo.(*repoMocks.MockIExptTurnResultRepo). EXPECT().MGetItemTurnRunLogs(ctx, exptID, *exptRunID, itemIDs, consumerSpaceID). Return([]*entity.ExptTurnResultRunLog{{ItemID: 10, TargetResultID: 500}}, nil) + mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). + EXPECT().TerminateAsyncRecordsAndDestroySandbox( + gomock.Any(), consumerSpaceID, []int64{500}, int32(errno.AsyncEvalTargetTerminatedCode), + gomock.Any(), false, + ).Times(1) mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). EXPECT().TerminateAsyncRecordsAndDestroySandbox( gomock.Any(), int64(99), []int64{500}, int32(errno.AsyncEvalTargetTerminatedCode), diff --git a/backend/modules/evaluation/domain/service/expt_result_aggr_impl.go b/backend/modules/evaluation/domain/service/expt_result_aggr_impl.go index 17c05ba33..1475cfcb6 100644 --- a/backend/modules/evaluation/domain/service/expt_result_aggr_impl.go +++ b/backend/modules/evaluation/domain/service/expt_result_aggr_impl.go @@ -161,13 +161,8 @@ func (e *ExptAggrResultServiceImpl) computeEvaluatorAggrGroup(ctx context.Contex func (e *ExptAggrResultServiceImpl) buildExptTargetMtrAggregatorGroup(ctx context.Context, spaceID, exptID int64) (*targetMtrAggrGroup, error) { const queryInterval = time.Millisecond * 30 - // ★ 跨空间共享: 评测对象执行记录随执行落来源空间(冻结 TargetSpaceID), 聚合读 record 也需按来源空间, - // 否则跨空间实验 target latency/token 聚合取不到 record。载入实验取冻结来源空间。 - // 此处 fail-closed: 载入失败必须返回错误让上层重试, 不可退回调用方空间继续 —— - // 跨空间实验(TargetSpaceID>0)下带错空间继续会让 BatchGetRecordByIDs 全查不到, - // latency/token 被静默聚合成 0 并落库, 事后无法与真实 0 区分且不会自愈。 - // 例外: 实验已被软删 / spaceID 不匹配时 GetByID 返回 ResourceNotFound —— 这是确定性结果, - // 重试不会好转, fail-closed 会让 MQ 永久重投且锁只能靠 TTL 过期; 此类按调用方空间降级继续。 + // Resolve the frozen source workspace for legacy execution records. + exptDO, exptErr := e.experimentRepo.GetByID(ctx, exptID, spaceID) if exptErr != nil { if statusErr, ok := errorx.FromStatusError(exptErr); !ok || statusErr.Code() != errno.ResourceNotFoundCode { @@ -213,7 +208,7 @@ func (e *ExptAggrResultServiceImpl) buildExptTargetMtrAggregatorGroup(ctx contex } for _, resIDs := range gslice.Chunk(targetResultIDs, 50) { - records, err := e.evalTargetSvc.BatchGetRecordByIDs(ctx, recordSpaceID, resIDs) + records, err := BatchGetExperimentTargetRecords(ctx, e.evalTargetSvc, spaceID, recordSpaceID, resIDs) if err != nil { return nil, err } diff --git a/backend/modules/evaluation/domain/service/expt_result_impl.go b/backend/modules/evaluation/domain/service/expt_result_impl.go index 20f1ad21b..f7a51871b 100644 --- a/backend/modules/evaluation/domain/service/expt_result_impl.go +++ b/backend/modules/evaluation/domain/service/expt_result_impl.go @@ -2443,10 +2443,8 @@ func (e *ExptResultBuilder) buildTargetOutput(ctx context.Context) error { targetResultIDs = append(targetResultIDs, turnResult.TargetResultID) targetResultID2turnResultID[turnResult.TargetResultID] = turnResult.ID } - // ★ 跨空间共享: 评测对象执行记录(eval_target_record)随执行落在来源空间(冻结 TargetSpaceID), - // 按 space_id 严格过滤; 用调用方空间读会查不到 → target 输出(含 Ext[sandbox_session_id])整体丢失, - // 前端 case 旁不显示 aiosession。按来源空间读 (0=同调用方空间)。 - targetRecords, err := e.evalTargetService.BatchGetRecordByIDs(ctx, resolveLoadSpaceID(e.SpaceID, e.exptDO.TargetSpaceID), targetResultIDs) + + targetRecords, err := BatchGetExperimentTargetRecords(ctx, e.evalTargetService, e.SpaceID, e.exptDO.TargetSpaceID, targetResultIDs) if err != nil { return err } diff --git a/backend/modules/evaluation/domain/service/expt_retry_failure_generation_test.go b/backend/modules/evaluation/domain/service/expt_retry_failure_generation_test.go index 5bbbf4d76..70439f223 100644 --- a/backend/modules/evaluation/domain/service/expt_retry_failure_generation_test.go +++ b/backend/modules/evaluation/domain/service/expt_retry_failure_generation_test.go @@ -83,6 +83,9 @@ func runRetryGenerationPreEval(t *testing.T, f *retryGenerationFixture) ([]*enti turnRepo.EXPECT().GetItemTurnResults(gomock.Any(), int64(1), int64(10), int64(3)).Return(f.turns, nil).Times(repeats) turnRepo.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(3), turnIDs).Return(refs, nil).Times(2 * repeats) idgen.EXPECT().GenMultiIDs(gomock.Any(), len(f.turns)).Return(turnIDs, nil).Times(repeats) + if f.targetSpace != 3 { + targetSvc.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(nil, nil).AnyTimes() + } targetSvc.EXPECT().GetRecordByID(gomock.Any(), f.targetSpace, int64(201)).DoAndReturn(func(ctx context.Context, _, _ int64) (*entity.EvalTargetRecord, error) { assert.True(t, contexts.CtxWriteDB(ctx)) return &entity.EvalTargetRecord{ID: 201, ExperimentRunID: 101, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil @@ -496,7 +499,7 @@ func TestRetryFailure_SourceLogMembershipPreservesNewFormatIdentity(t *testing.T }, nil) turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(101), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{nil, origin}, nil) sources := &retryEvaluatorSourceLogs{repo: turnRepo, event: &entity.ExptItemEvalEvent{ExptID: 1, EvalSetItemID: 10, SpaceID: 3}, logsByRun: make(map[int64]map[int64]*entity.ExptTurnResultRunLog)} - targetID, got, selectErr := failRetrySelectTurnRunLogRefs(context.Background(), 3, true, tr, targetSvc, recordSvc, []*entity.ExptTurnEvaluatorResultRef{{ExptTurnResultID: tr.ID, EvaluatorVersionID: 401, Alias: "judge-a", EvaluatorResultID: 301}, {ExptTurnResultID: tr.ID, InlineKey: "inline-a", SourceType: int32(entity.EvaluatorRecordSourceTypeInline), EvaluatorResultID: 302}}, sources) + targetID, got, selectErr := failRetrySelectTurnRunLogRefs(context.Background(), 3, 0, true, tr, targetSvc, recordSvc, []*entity.ExptTurnEvaluatorResultRef{{ExptTurnResultID: tr.ID, EvaluatorVersionID: 401, Alias: "judge-a", EvaluatorResultID: 301}, {ExptTurnResultID: tr.ID, InlineKey: "inline-a", SourceType: int32(entity.EvaluatorRecordSourceTypeInline), EvaluatorResultID: 302}}, sources) require.NoError(t, selectErr) assert.Equal(t, int64(201), targetID) if !tt.registered && !tt.inline { diff --git a/backend/modules/evaluation/domain/service/expt_retry_failure_stage_resume_test.go b/backend/modules/evaluation/domain/service/expt_retry_failure_stage_resume_test.go index 20cae28e5..a8c51868a 100644 --- a/backend/modules/evaluation/domain/service/expt_retry_failure_stage_resume_test.go +++ b/backend/modules/evaluation/domain/service/expt_retry_failure_stage_resume_test.go @@ -91,7 +91,7 @@ func TestFailRetrySelectTurnRunLogRefs_TargetRequirementControlsEvaluatorReuse(t } targetID, evaluatorResults, selectErr := failRetrySelectTurnRunLogRefs( - context.Background(), 1, tt.targetRequired, + context.Background(), 1, 0, tt.targetRequired, &entity.ExptTurnResult{ EvaluatorResults: &entity.EvaluatorResults{ EvalVerIDToResID: map[int64]int64{10: evaluatorRecordID}, @@ -425,6 +425,8 @@ func TestExptFailRetryExec_BuildPagePlan_MultiSetGroupsTargetReadsBySourceSpace( {ItemID: 12, ItemConfig: &entity.ExptItemConfig{TargetSourceSpaceID: rowTargetSpace}}, }, nil) success := entity.EvalTargetRunStatusSuccess + targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), []int64{110}).Return(nil, nil) + targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), []int64{100, 120}).Return(nil, nil) targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), topTargetSpace, []int64{110}).Return([]*entity.EvalTargetRecord{{ID: 110, Status: &success}}, nil) targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), rowTargetSpace, []int64{100, 120}).Return([]*entity.EvalTargetRecord{ {ID: 100, Status: &success}, @@ -483,6 +485,7 @@ func TestExptFailRetryExec_BuildPagePlan_MultiSetMissingItemRefFallsBackToExperi ItemID: itemID, ItemConfig: nil, }}, nil) success := entity.EvalTargetRunStatusSuccess + targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), []int64{30}).Return(nil, nil) targetSvc.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(99), []int64{30}).Return([]*entity.EvalTargetRecord{{ID: 30, Status: &success}}, nil) idgen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{900}, nil) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go index 8d1b187bb..82007d048 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go @@ -1298,7 +1298,7 @@ func (e *ExptRecordEvalModeSubmit) PostEval(ctx context.Context, eiec *entity.Ex // failRetrySelectTurnRunLogRefs keeps successful stages that belong to the reused target. func failRetrySelectTurnRunLogRefs( ctx context.Context, - spaceID int64, + spaceID, sourceSpaceID int64, targetRequired bool, tr *entity.ExptTurnResult, evalTarget IEvalTargetService, @@ -1322,7 +1322,7 @@ func failRetrySelectTurnRunLogRefs( if evalTarget == nil { return 0, nil, errors.New("eval target service is nil") } - targetRec, err := evalTarget.GetRecordByID(ctx, spaceID, tr.TargetResultID) + targetRec, err := GetExperimentTargetRecord(ctx, evalTarget, spaceID, sourceSpaceID, tr.TargetResultID) if err != nil { return 0, nil, err } @@ -1571,9 +1571,7 @@ func (e *ExptRecordEvalModeFailRetry) PreEval(ctx context.Context, eiec *entity. runLog.Status = entity.TurnRunState_Processing runLog.ExptRunID = eiec.Event.ExptRunID runLog.ErrMsg = "" - // 跨空间共享: Target 记录随执行落来源空间(冻结 TargetSpaceID), 失败重试选引用时须按来源空间读; - // 用调用方空间读会得 nil → 误判 Target 非 Success → 清零 target_result_id 触发无谓重跑 Target。 - targetID, evalIDs, err := failRetrySelectTurnRunLogRefs(snapshotCtx, resolveLoadSpaceID(eiec.Event.SpaceID, eiec.TargetSourceSpaceID()), !shouldSkipTargetNode(eiec.Expt), tr, e.evalTargetService, e.evaluatorRecordSvc, refsByTurnResultID[tr.ID], sourceLogs) + targetID, evalIDs, err := failRetrySelectTurnRunLogRefs(snapshotCtx, eiec.Event.SpaceID, eiec.TargetSourceSpaceID(), !shouldSkipTargetNode(eiec.Expt), tr, e.evalTargetService, e.evaluatorRecordSvc, refsByTurnResultID[tr.ID], sourceLogs) if err != nil { return fmt.Errorf("%w: %w", errRetryStartDependencyFailure, err) } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go index 80f232424..499e22d60 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go @@ -1678,7 +1678,7 @@ func Test_failRetrySelectTurnRunLogRefs(t *testing.T) { }}, nil) } gotTargetID, gotEvalResults, selectErr := failRetrySelectTurnRunLogRefs( - context.Background(), tt.spaceID, targetRequired, tt.tr, evalTarget, evalRecord, refs, sourceLogs, + context.Background(), tt.spaceID, 0, targetRequired, tt.tr, evalTarget, evalRecord, refs, sourceLogs, ) if tt.name == "TargetResultID > 0, evalTarget is nil -> dependency error" || tt.name == "TargetResultID > 0, GetRecordByID returns error -> returns 0, nil" { require.Error(t, selectErr) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index 973f29c20..66b434792 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -443,9 +443,7 @@ func (e *ExptItemEvalCtxExecutor) buildExptTurnEvalCtx(ctx context.Context, turn } if tid := existTurnRunResult.TargetResultID; tid > 0 { - // ★ 跨空间共享: 评测对象执行记录随执行落来源空间(冻结 TargetSpaceID), 按来源空间读; - // 用调用方空间读会得 nil → 异步回调 validateEvalTargetCtx 报 "target result must not be nil"。 - targetRecord, err := e.evalTargetService.GetRecordByID(recordCtx, resolveLoadSpaceID(spaceID, eiec.TargetSourceSpaceID()), tid) + targetRecord, err := GetExperimentTargetRecord(recordCtx, e.evalTargetService, spaceID, eiec.TargetSourceSpaceID(), tid) if err != nil { return nil, err } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index 4316c9b7c..d0b051771 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -935,13 +935,11 @@ func (e *ExptSchedulerImpl) sweepTerminatedSandboxItems(ctx context.Context, eve return items, nil, nil } - // 跨空间共享: EvalTargetRecord.SpaceID 是来源空间, 用消费方 event.SpaceID 查会漏。 var targetSourceSpaceID int64 if expt != nil { targetSourceSpaceID = expt.TargetSpaceID } - targetSpaceID := resolveLoadSpaceID(event.SpaceID, targetSourceSpaceID) - terminatedRecordIDs, statusMap := e.evalTargetService.CheckSandboxTerminated(ctx, targetSpaceID, recordIDs) + terminatedRecordIDs, statusMap := checkExperimentSandboxTerminated(ctx, e.evalTargetService, event.SpaceID, targetSourceSpaceID, recordIDs) if len(terminatedRecordIDs) == 0 { return items, nil, nil } @@ -983,10 +981,8 @@ func (e *ExptSchedulerImpl) sweepTerminatedSandboxItems(ctx context.Context, eve event.ExptID, event.ExptRunID, terminatedItemIDs, firstStatus) // 沙箱已经是终态,Destroy 不需要再发 EndCmd,zombieTimeout=false。 - // targetSpaceID 复用上面 CheckSandboxTerminated 用的来源空间, 保持一致。 - _ = e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox( - ctx, - targetSpaceID, + _ = terminateExperimentTargetRecords( + ctx, e.evalTargetService, event.SpaceID, targetSourceSpaceID, terminatedRecordIDs, int32(errno.SandboxTerminatedBeforeReportCode), fmt.Sprintf("sandbox execute reached terminal state (%s) before result was reported", firstStatus), @@ -1110,17 +1106,12 @@ func (e *ExptSchedulerImpl) terminateZombieEvalTargetRecords(ctx context.Context return nil } - // 跨空间共享: EvalTargetRecord 落库时 SpaceID = 评测对象来源空间 (resolveLoadSpaceID 结果), - // 若这里仍用 event.SpaceID (消费方) 去查, DAO 的 SpaceID.Eq 过滤会返回空 → Destroy RPC 根本发不出, - // 导致沙箱在来源空间空跑到 session TTL 兜底才回收。与 expt_run_item_turn_impl.go:130 保持同一套解析。 var targetSourceSpaceID int64 if expt != nil { targetSourceSpaceID = expt.TargetSpaceID } - targetSpaceID := resolveLoadSpaceID(event.SpaceID, targetSourceSpaceID) - _ = e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox( - ctx, - targetSpaceID, + _ = terminateExperimentTargetRecords( + ctx, e.evalTargetService, event.SpaceID, targetSourceSpaceID, recordIDs, int32(errno.AsyncEvalTargetZombieTimeoutCode), "async eval target terminated: experiment item exceeded zombie timeout", diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go index 391a15e25..3cc83e956 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go @@ -2407,7 +2407,7 @@ func Test_recordEvalItemRunLogs_publishFailInterrupts(t *testing.T) { // TestExptSchedulerImpl_terminateZombieEvalTargetRecords_CrossSpace 覆盖 zombie 兜底销毁沙箱时 // 传入 TerminateAsyncRecordsAndDestroySandbox 的 spaceID 解析: -// - Expt.TargetSpaceID > 0 → 用来源空间 (跨空间共享评测对象), 避免 DAO SpaceID.Eq 过滤把 record 过滤空 +// - Expt.TargetSpaceID > 0 → 回收实验空间及历史来源空间记录 // - Expt.TargetSpaceID = 0 → 回退到 event.SpaceID (消费方 = 来源方) // - expt = nil → 回退到 event.SpaceID (向后兼容) func TestExptSchedulerImpl_terminateZombieEvalTargetRecords_CrossSpace(t *testing.T) { @@ -2422,7 +2422,7 @@ func TestExptSchedulerImpl_terminateZombieEvalTargetRecords_CrossSpace(t *testin }, mockTurnRepo, mockTargetSvc } - t.Run("TargetSpaceID>0 → 用来源空间 (99) 而不是 event.SpaceID (3)", func(t *testing.T) { + t.Run("shared target → clean consumer and legacy source records", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() svc, mockTurnRepo, mockTargetSvc := newSvc(ctrl) @@ -2436,6 +2436,10 @@ func TestExptSchedulerImpl_terminateZombieEvalTargetRecords_CrossSpace(t *testin } mockTurnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)). Return([]*entity.ExptTurnResultRunLog{{ItemID: 10, TargetResultID: 500}}, nil) + mockTargetSvc.EXPECT().TerminateAsyncRecordsAndDestroySandbox( + gomock.Any(), int64(3), []int64{500}, int32(errno.AsyncEvalTargetZombieTimeoutCode), + gomock.Any(), true, + ).Times(1) mockTargetSvc.EXPECT().TerminateAsyncRecordsAndDestroySandbox( gomock.Any(), int64(99), []int64{500}, int32(errno.AsyncEvalTargetZombieTimeoutCode), gomock.Any(), true, @@ -2487,7 +2491,7 @@ func TestExptSchedulerImpl_terminateZombieEvalTargetRecords_CrossSpace(t *testin // TestExptSchedulerImpl_sweepTerminatedSandboxItems_CrossSpace 覆盖 sweep 分支中 // CheckSandboxTerminated 与 TerminateAsyncRecordsAndDestroySandbox 传入的 spaceID 解析: -// - Expt.TargetSpaceID > 0 → 都用来源空间 +// - Expt.TargetSpaceID > 0 → 覆盖实验空间及历史来源空间记录 // - Expt.TargetSpaceID = 0 → 都用 event.SpaceID func TestExptSchedulerImpl_sweepTerminatedSandboxItems_CrossSpace(t *testing.T) { baseEvent := &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} @@ -2506,7 +2510,7 @@ func TestExptSchedulerImpl_sweepTerminatedSandboxItems_CrossSpace(t *testing.T) }, mockTurnRepo, mockItemRepo, mockTargetSvc } - t.Run("TargetSpaceID>0 → Check/Terminate 都用来源空间 (99)", func(t *testing.T) { + t.Run("shared target → check and clean both record spaces", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() svc, mockTurnRepo, mockItemRepo, mockTargetSvc := newSvc(ctrl) @@ -2515,9 +2519,14 @@ func TestExptSchedulerImpl_sweepTerminatedSandboxItems_CrossSpace(t *testing.T) mockTurnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)). Return([]*entity.ExptTurnResultRunLog{{ItemID: 10, TargetResultID: 500}}, nil) // 关键: CheckSandboxTerminated 用 targetSpaceID=99 + mockTargetSvc.EXPECT().CheckSandboxTerminated(gomock.Any(), int64(3), []int64{500}).Return(nil, nil) mockTargetSvc.EXPECT().CheckSandboxTerminated(gomock.Any(), int64(99), []int64{500}). Return([]int64{500}, map[int64]string{500: "Failed"}) // 关键: Terminate 也用 targetSpaceID=99, 与 Check 侧保持一致 + mockTargetSvc.EXPECT().TerminateAsyncRecordsAndDestroySandbox( + gomock.Any(), int64(3), []int64{500}, int32(errno.SandboxTerminatedBeforeReportCode), + gomock.Any(), false, + ).Times(1) mockTargetSvc.EXPECT().TerminateAsyncRecordsAndDestroySandbox( gomock.Any(), int64(99), []int64{500}, int32(errno.SandboxTerminatedBeforeReportCode), gomock.Any(), false, diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index 465c93dbd..73dce7807 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -922,7 +922,7 @@ func (e *ExptFailRetryExec) buildPagePlan(ctx context.Context, event *entity.Exp } for targetSpaceID, recordIDSet := range targetIDsBySpace { recordIDs := sortedInt64Set(recordIDSet) - records, err := e.evalTargetService.BatchGetRecordByIDs(currentRunCtx, targetSpaceID, recordIDs) + records, err := BatchGetExperimentTargetRecords(currentRunCtx, e.evalTargetService, event.SpaceID, targetSpaceID, recordIDs) if err != nil { return nil, e.retryStartDependencyError(ctx, event, "eval_target_record", err) } diff --git a/backend/modules/evaluation/domain/service/sandbox_invocation.go b/backend/modules/evaluation/domain/service/sandbox_invocation.go index a54069ae0..d48605c06 100644 --- a/backend/modules/evaluation/domain/service/sandbox_invocation.go +++ b/backend/modules/evaluation/domain/service/sandbox_invocation.go @@ -240,7 +240,7 @@ func (l *sandboxInvocationLifecycle) abort(ctx context.Context, cause error) { logs.CtxWarn(cleanupCtx, "sandbox initialization failure persistence failed, record_id=%d, err=%v", l.record.ID, err) } } - _ = l.service.destroySandboxExecute(cleanupCtx, strconv.FormatInt(gptr.Indirect(l.param.ExperimentID), 10), l.schedulerSpaceID(), l.spaceID, sandboxExecuteIDsOf(cleanupCtx, l.record), false) + _ = l.service.destroySandboxExecute(cleanupCtx, strconv.FormatInt(gptr.Indirect(l.param.ExperimentID), 10), l.schedulerSpaceID(), resolveLoadSpaceID(l.spaceID, l.target.SpaceID), sandboxExecuteIDsOf(cleanupCtx, l.record), false) } func sandboxRecordSchedulerSpaceID(r *entity.EvalTargetRecord) int64 { diff --git a/backend/modules/evaluation/domain/service/target_impl.go b/backend/modules/evaluation/domain/service/target_impl.go index f0f43fdd9..bc223cac7 100644 --- a/backend/modules/evaluation/domain/service/target_impl.go +++ b/backend/modules/evaluation/domain/service/target_impl.go @@ -584,7 +584,11 @@ func (e *EvalTargetServiceImpl) AsyncExecuteTarget(ctx context.Context, spaceID, func (e *EvalTargetServiceImpl) asyncExecuteTarget(ctx context.Context, spaceID int64, target *entity.EvalTarget, param *entity.ExecuteTargetCtx, inputData *entity.EvalTargetInputData, ) (record *entity.EvalTargetRecord, callee string, err error) { - defer func(st time.Time) { e.metric.EmitRun(spaceID, err, st) }(time.Now()) // todo(@liushengyang): mtr + recordSpaceID := spaceID + if target.EvalTargetType == entity.EvalTargetTypeSandboxAgent && param.ExptSpaceID > 0 { + recordSpaceID = param.ExptSpaceID + } + defer func(st time.Time) { e.metric.EmitRun(recordSpaceID, err, st) }(time.Now()) // todo(@liushengyang): mtr defer goroutine.Recovery(ctx) targetID := target.ID @@ -606,11 +610,11 @@ func (e *EvalTargetServiceImpl) asyncExecuteTarget(ctx context.Context, spaceID TimeConsumingMS: gptr.Of(int64(0)), } - ctx, span := looptracer.GetTracer().StartSpan(ctx, "EvalTarget", "eval_target", looptracer.WithStartNewTrace(), looptracer.WithSpanWorkspaceID(strconv.FormatInt(spaceID, 10))) + ctx, span := looptracer.GetTracer().StartSpan(ctx, "EvalTarget", "eval_target", looptracer.WithStartNewTrace(), looptracer.WithSpanWorkspaceID(strconv.FormatInt(recordSpaceID, 10))) span.SetCallType("EvalTarget") ctx = looptracer.GetTracer().Inject(ctx) - lifecycle := &sandboxInvocationLifecycle{service: e, spaceID: spaceID, target: target, param: param, input: inputData, traceID: span.GetTraceID()} + lifecycle := &sandboxInvocationLifecycle{service: e, spaceID: recordSpaceID, target: target, param: param, input: inputData, traceID: span.GetTraceID()} invokeID, callee, ext, execErr := operator.AsyncExecute(ctx, spaceID, &entity.ExecuteEvalTargetParam{ PrepareSandboxExecution: lifecycle.prepare, @@ -662,7 +666,7 @@ func (e *EvalTargetServiceImpl) asyncExecuteTarget(ctx context.Context, spaceID userID := session.UserIDInCtxOrEmpty(ctx) record = &entity.EvalTargetRecord{ ID: invokeID, - SpaceID: spaceID, + SpaceID: recordSpaceID, TargetID: targetID, TargetVersionID: targetVersionID, ExperimentRunID: gptr.Indirect(param.ExperimentRunID), @@ -858,7 +862,7 @@ func (e *EvalTargetServiceImpl) destroySandboxExecuteIfNeeded(ctx context.Contex return } // 仅 SandboxAgent 评测对象需要销毁沙箱执行 - targetVersion, err := e.evalTargetRepo.GetEvalTargetVersion(ctx, record.SpaceID, record.TargetVersionID) + targetVersion, err := e.recordTargetVersion(ctx, record) if err != nil { logs.CtxWarn(ctx, "[SandboxDestroy] get eval target version fail, space_id=%d, version_id=%d, err=%v", record.SpaceID, record.TargetVersionID, err) @@ -880,7 +884,7 @@ func (e *EvalTargetServiceImpl) destroySandboxExecuteIfNeeded(ctx context.Contex executeIDs := sandboxExecuteIDsOf(ctx, record) logs.CtxInfo(ctx, "[SandboxDestroy] destroying sandbox executes, record_id=%d, task_id=%s, expt_run_id=%d, execute_ids=%v", record.ID, taskID, record.ExperimentRunID, executeIDs) - _ = e.destroySandboxExecute(ctx, taskID, sandboxWorkspaceIDOf(record), record.SpaceID, executeIDs, false) + _ = e.destroySandboxExecute(ctx, taskID, sandboxWorkspaceIDOf(record), resolveLoadSpaceID(record.SpaceID, targetVersion.SpaceID), executeIDs, false) } // sandboxWorkspaceIDOf 保留旧任务的来源空间归属,新任务使用 operator 记录的调度空间。 @@ -1116,18 +1120,18 @@ func (e *EvalTargetServiceImpl) TerminateAsyncRecordsAndDestroySandbox(ctx conte for id := range versionIDSet { versionIDs = append(versionIDs, id) } - versions, err := e.evalTargetRepo.BatchGetEvalTargetVersion(ctx, spaceID, versionIDs) + versions, err := e.recordTargetVersions(ctx, spaceID, versionIDs, records) if err != nil { logs.CtxWarn(ctx, "[SandboxDestroy] batch get eval target versions fail, space_id=%d, err=%v", spaceID, err) return err } - sandboxVersionIDs := make(map[int64]struct{}) + sandboxVersionIDs := make(map[int64]int64) for _, v := range versions { if v == nil || v.EvalTargetVersion == nil { continue } if v.EvalTargetType == entity.EvalTargetTypeSandboxAgent { - sandboxVersionIDs[v.EvalTargetVersion.ID] = struct{}{} + sandboxVersionIDs[v.EvalTargetVersion.ID] = v.SpaceID } } if len(sandboxVersionIDs) == 0 { @@ -1168,14 +1172,14 @@ func (e *EvalTargetServiceImpl) TerminateAsyncRecordsAndDestroySandbox(ctx conte if id == extraExecuteID { continue } - if err := e.destroySandboxExecute(ctx, taskID, schedulerSpaceID, r.SpaceID, []string{id}, zombieTimeout, wait); err != nil { + if err := e.destroySandboxExecute(ctx, taskID, schedulerSpaceID, resolveLoadSpaceID(r.SpaceID, sandboxVersionIDs[r.TargetVersionID]), []string{id}, zombieTimeout, wait); err != nil { return err } } // 双沙箱:额外销毁从沙箱 execute,语义对齐 destroySandboxExecuteIfNeeded (success 分支)。 // 从沙箱 zombieTimeout 恒为 false,不需要下发 SandboxAgent EndCmd(EndCmd 只对主沙箱有意义)。 if extraExecuteID != "" { - if err := e.destroySandboxExtraExecute(ctx, taskID, schedulerSpaceID, r.SpaceID, extraExecuteID, wait); err != nil { + if err := e.destroySandboxExtraExecute(ctx, taskID, schedulerSpaceID, resolveLoadSpaceID(r.SpaceID, sandboxVersionIDs[r.TargetVersionID]), extraExecuteID, wait); err != nil { return err } } @@ -1260,18 +1264,18 @@ func (e *EvalTargetServiceImpl) CheckSandboxTerminated(ctx context.Context, spac for id := range versionIDSet { versionIDs = append(versionIDs, id) } - versions, err := e.evalTargetRepo.BatchGetEvalTargetVersion(ctx, spaceID, versionIDs) + versions, err := e.recordTargetVersions(ctx, spaceID, versionIDs, records) if err != nil { logs.CtxWarn(ctx, "[SandboxStatusCheck] batch get eval target versions fail, space_id=%d, err=%v", spaceID, err) return nil, nil } - sandboxVersionIDs := make(map[int64]struct{}) + sandboxVersionIDs := make(map[int64]int64) for _, v := range versions { if v == nil || v.EvalTargetVersion == nil { continue } if v.EvalTargetType == entity.EvalTargetTypeSandboxAgent { - sandboxVersionIDs[v.EvalTargetVersion.ID] = struct{}{} + sandboxVersionIDs[v.EvalTargetVersion.ID] = v.SpaceID } } if len(sandboxVersionIDs) == 0 { @@ -1561,7 +1565,7 @@ func (e *EvalTargetServiceImpl) emitTargetTrace(ctx context.Context, span looptr setSpanInputOutput(ctx, spanParam, record.EvalTargetInputData, record.EvalTargetOutputData) if record.TargetVersionID > 0 { - evalTargetDO, err := e.GetEvalTargetVersion(ctx, record.SpaceID, record.TargetVersionID, false) + evalTargetDO, err := e.recordTargetVersion(ctx, record) if err != nil { return "", err } diff --git a/backend/modules/evaluation/domain/service/target_impl_sandbox_workspace_test.go b/backend/modules/evaluation/domain/service/target_impl_sandbox_workspace_test.go index 8e8f1ea47..f960c920e 100644 --- a/backend/modules/evaluation/domain/service/target_impl_sandbox_workspace_test.go +++ b/backend/modules/evaluation/domain/service/target_impl_sandbox_workspace_test.go @@ -8,6 +8,9 @@ import ( "testing" "time" + errno "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + errorx "github.com/coze-dev/coze-loop/backend/pkg/errorx" + "github.com/bytedance/gg/gptr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -97,6 +100,10 @@ func TestSandboxRecordSchedulerWorkspace(t *testing.T) { func TestAsyncExecuteTarget_CreateRecordFail_ExperimentWorkspace(t *testing.T) { ctrl := gomock.NewController(t) svc, target, input, got, mu, done := createRecordFailFixture(t, ctrl, entity.EvalTargetTypeSandboxAgent, nil) + store := svc.evalTargetRepo.(*repomocks.MockIEvalTargetRepo) + store.EXPECT().GetEvalTargetVersion(gomock.Any(), int64(2), target.EvalTargetVersion.ID).Return(nil, errorx.NewByCode(errno.ResourceNotFoundCode)) + store.EXPECT().GetEvalTarget(gomock.Any(), target.ID).Return(target, nil) + _, _, err := svc.asyncExecuteTarget(context.Background(), target.SpaceID, target, &entity.ExecuteTargetCtx{ ExperimentID: gptr.Of(int64(40)), ExptSpaceID: 2, }, input) diff --git a/backend/modules/evaluation/domain/service/target_record_space.go b/backend/modules/evaluation/domain/service/target_record_space.go new file mode 100644 index 000000000..668c8b742 --- /dev/null +++ b/backend/modules/evaluation/domain/service/target_record_space.go @@ -0,0 +1,142 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "fmt" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/errorx" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +// ExperimentTargetRecordSpaces keeps legacy source-owned records reachable. Callers +// must obtain record IDs and the frozen source workspace from an authorized experiment. +func ExperimentTargetRecordSpaces(spaceID, sourceSpaceID int64) []int64 { + spaces := []int64{spaceID} + if sourceSpaceID > 0 && sourceSpaceID != spaceID { + spaces = append(spaces, sourceSpaceID) + } + return spaces +} + +func GetExperimentTargetRecord(ctx context.Context, svc IEvalTargetService, spaceID, sourceSpaceID, recordID int64) (*entity.EvalTargetRecord, error) { + for _, recordSpaceID := range ExperimentTargetRecordSpaces(spaceID, sourceSpaceID) { + record, err := svc.GetRecordByID(ctx, recordSpaceID, recordID) + if err != nil || record != nil { + return record, err + } + } + return nil, nil +} + +func BatchGetExperimentTargetRecords(ctx context.Context, svc IEvalTargetService, spaceID, sourceSpaceID int64, recordIDs []int64) ([]*entity.EvalTargetRecord, error) { + missing := recordIDs + var records []*entity.EvalTargetRecord + for _, recordSpaceID := range ExperimentTargetRecordSpaces(spaceID, sourceSpaceID) { + if len(missing) == 0 { + break + } + batch, err := svc.BatchGetRecordByIDs(ctx, recordSpaceID, missing) + if err != nil { + return nil, err + } + records = append(records, batch...) + found := make(map[int64]struct{}, len(batch)) + for _, record := range batch { + if record != nil { + found[record.ID] = struct{}{} + } + } + next := make([]int64, 0, len(missing)) + for _, id := range missing { + if _, ok := found[id]; !ok { + next = append(next, id) + } + } + missing = next + } + return records, nil +} + +// The record is already scoped to an authorized execution. Its definition can +// belong to a shared source workspace; never use that workspace to read records. +func (e *EvalTargetServiceImpl) recordTargetVersion(ctx context.Context, record *entity.EvalTargetRecord) (*entity.EvalTarget, error) { + version, err := e.evalTargetRepo.GetEvalTargetVersion(ctx, record.SpaceID, record.TargetVersionID) + if err == nil { + return version, nil + } + if status, ok := errorx.FromStatusError(err); !ok || status.Code() != errno.ResourceNotFoundCode { + return nil, err + } + if record.TargetID <= 0 { + return nil, err + } + target, lookupErr := e.evalTargetRepo.GetEvalTarget(ctx, record.TargetID) + if lookupErr != nil { + return nil, lookupErr + } + if target == nil || target.SpaceID <= 0 || target.SpaceID == record.SpaceID { + return nil, err + } + version, err = e.evalTargetRepo.GetEvalTargetVersion(ctx, target.SpaceID, record.TargetVersionID) + if err != nil { + return nil, err + } + if version == nil || version.ID != record.TargetID { + return nil, fmt.Errorf("target record definition mismatch: record_id=%d", record.ID) + } + return version, nil +} + +func (e *EvalTargetServiceImpl) recordTargetVersions(ctx context.Context, spaceID int64, versionIDs []int64, records []*entity.EvalTargetRecord) ([]*entity.EvalTarget, error) { + versions, err := e.evalTargetRepo.BatchGetEvalTargetVersion(ctx, spaceID, versionIDs) + if err != nil { + return nil, err + } + found := make(map[int64]bool, len(versions)) + for _, v := range versions { + if v != nil && v.EvalTargetVersion != nil { + found[v.EvalTargetVersion.ID] = true + } + } + for _, record := range records { + if record == nil || record.TargetID <= 0 || record.TargetVersionID <= 0 || found[record.TargetVersionID] { + continue + } + v, err := e.recordTargetVersion(ctx, record) + if err != nil { + return nil, err + } + if v != nil { + versions = append(versions, v) + } + found[record.TargetVersionID] = true + } + return versions, nil +} + +func terminateExperimentTargetRecords(ctx context.Context, svc IEvalTargetService, spaceID, sourceSpaceID int64, ids []int64, code int32, message string, zombie bool, wait ...bool) error { + var result error + for _, recordSpaceID := range ExperimentTargetRecordSpaces(spaceID, sourceSpaceID) { + result = errors.Join(result, svc.TerminateAsyncRecordsAndDestroySandbox(ctx, recordSpaceID, ids, code, message, zombie, wait...)) + } + return result +} + +func checkExperimentSandboxTerminated(ctx context.Context, svc IEvalTargetService, spaceID, sourceSpaceID int64, ids []int64) ([]int64, map[int64]string) { + var terminated []int64 + statuses := make(map[int64]string) + for _, recordSpaceID := range ExperimentTargetRecordSpaces(spaceID, sourceSpaceID) { + found, status := svc.CheckSandboxTerminated(ctx, recordSpaceID, ids) + terminated = append(terminated, found...) + for id, value := range status { + statuses[id] = value + } + } + return terminated, statuses +} diff --git a/backend/modules/evaluation/domain/service/target_record_space_test.go b/backend/modules/evaluation/domain/service/target_record_space_test.go new file mode 100644 index 000000000..1ac0acb9a --- /dev/null +++ b/backend/modules/evaluation/domain/service/target_record_space_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "testing" + + "github.com/bytedance/gg/gptr" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + servicemocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/errorx" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestExperimentTargetRecordOwnership(t *testing.T) { + ctx := context.Background() + t.Run("consumer record does not read source", func(t *testing.T) { + svc := servicemocks.NewMockIEvalTargetService(gomock.NewController(t)) + record := &entity.EvalTargetRecord{ID: 30, SpaceID: 10} + svc.EXPECT().GetRecordByID(ctx, int64(10), int64(30)).Return(record, nil) + got, err := GetExperimentTargetRecord(ctx, svc, 10, 20, 30) + require.NoError(t, err) + require.Same(t, record, got) + }) + t.Run("legacy record remains readable", func(t *testing.T) { + svc := servicemocks.NewMockIEvalTargetService(gomock.NewController(t)) + record := &entity.EvalTargetRecord{ID: 30, SpaceID: 20} + gomock.InOrder(svc.EXPECT().GetRecordByID(ctx, int64(10), int64(30)).Return(nil, nil), svc.EXPECT().GetRecordByID(ctx, int64(20), int64(30)).Return(record, nil)) + got, err := GetExperimentTargetRecord(ctx, svc, 10, 20, 30) + require.NoError(t, err) + require.Same(t, record, got) + }) + t.Run("query failures do not trigger cross-space fallback", func(t *testing.T) { + svc := servicemocks.NewMockIEvalTargetService(gomock.NewController(t)) + failure := errors.New("read unavailable") + svc.EXPECT().GetRecordByID(ctx, int64(10), int64(30)).Return(nil, failure) + _, err := GetExperimentTargetRecord(ctx, svc, 10, 20, 30) + require.ErrorIs(t, err, failure) + }) + t.Run("mixed generations fetch only missing source records", func(t *testing.T) { + svc := servicemocks.NewMockIEvalTargetService(gomock.NewController(t)) + current := &entity.EvalTargetRecord{ID: 30, SpaceID: 10} + legacy := &entity.EvalTargetRecord{ID: 31, SpaceID: 20} + gomock.InOrder(svc.EXPECT().BatchGetRecordByIDs(ctx, int64(10), []int64{30, 31}).Return([]*entity.EvalTargetRecord{current}, nil), svc.EXPECT().BatchGetRecordByIDs(ctx, int64(20), []int64{31}).Return([]*entity.EvalTargetRecord{legacy}, nil)) + got, err := BatchGetExperimentTargetRecords(ctx, svc, 10, 20, []int64{30, 31}) + require.NoError(t, err) + require.Equal(t, []*entity.EvalTargetRecord{current, legacy}, got) + }) +} + +func TestSandboxRecordDefinitionRemainsInSourceWorkspace(t *testing.T) { + ctrl := gomock.NewController(t) + store := repomocks.NewMockIEvalTargetRepo(ctrl) + svc := &EvalTargetServiceImpl{evalTargetRepo: store} + record := &entity.EvalTargetRecord{ID: 30, SpaceID: 10, TargetID: 40, TargetVersionID: 50} + target := &entity.EvalTarget{ID: 40, SpaceID: 20, EvalTargetType: entity.EvalTargetTypeSandboxAgent, EvalTargetVersion: &entity.EvalTargetVersion{ID: 50}} + gomock.InOrder( + store.EXPECT().GetEvalTargetVersion(gomock.Any(), int64(10), int64(50)).Return(nil, errorx.NewByCode(errno.ResourceNotFoundCode)), + store.EXPECT().GetEvalTarget(gomock.Any(), int64(40)).Return(target, nil), + store.EXPECT().GetEvalTargetVersion(gomock.Any(), int64(20), int64(50)).Return(target, nil), + ) + got, err := svc.recordTargetVersion(context.Background(), record) + require.NoError(t, err) + require.Same(t, target, got) +} + +func TestSandboxAsyncRecordUsesExperimentWorkspace(t *testing.T) { + for _, tc := range []struct { + name string + experimentSpace, expectedSpace int64 + }{{"shared", 10, 10}, {"same-space", 20, 20}, {"debug", 0, 20}} { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := repomocks.NewMockIEvalTargetRepo(ctrl) + operator := servicemocks.NewMockISourceEvalTargetOperateService(ctrl) + metric := metricsmocks.NewMockEvalTargetMetrics(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + target := &entity.EvalTarget{ID: 40, SpaceID: 20, SourceTargetID: "shared-agent", EvalTargetType: entity.EvalTargetTypeSandboxAgent, EvalTargetVersion: &entity.EvalTargetVersion{ID: 50, SourceTargetVersion: "v1"}} + input := &entity.EvalTargetInputData{} + operator.EXPECT().ValidateInput(gomock.Any(), int64(20), gomock.Any(), input).Return(nil) + operator.EXPECT().AsyncExecute(gomock.Any(), int64(20), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, p *entity.ExecuteEvalTargetParam) (int64, string, map[string]string, error) { + require.Equal(t, tc.experimentSpace, p.ExptSpaceID) + require.Same(t, target, p.EvalTarget) + return 30, "sandbox_agent", nil, nil + }) + metric.EXPECT().EmitRun(tc.expectedSpace, gomock.Any(), gomock.Any()) + config.EXPECT().BuildEvalExt(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + if tc.expectedSpace != 20 { + store.EXPECT().GetEvalTargetVersion(gomock.Any(), tc.expectedSpace, int64(50)).Return(nil, errorx.NewByCode(errno.ResourceNotFoundCode)).AnyTimes() + store.EXPECT().GetEvalTarget(gomock.Any(), int64(40)).Return(target, nil).AnyTimes() + } + store.EXPECT().GetEvalTargetVersion(gomock.Any(), int64(20), int64(50)).Return(target, nil).AnyTimes() + store.EXPECT().CreateEvalTargetRecord(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.EvalTargetRecord, _ *bool) (int64, error) { + require.Equal(t, tc.expectedSpace, r.SpaceID) + return r.ID, nil + }) + svc := &EvalTargetServiceImpl{evalTargetRepo: store, typedOperators: map[entity.EvalTargetType]ISourceEvalTargetOperateService{entity.EvalTargetTypeSandboxAgent: operator}, metric: metric, configer: config} + record, _, err := svc.asyncExecuteTarget(context.Background(), 20, target, &entity.ExecuteTargetCtx{ExptSpaceID: tc.experimentSpace, ExperimentID: gptr.Of(int64(60))}, input) + require.NoError(t, err) + require.Equal(t, tc.expectedSpace, record.SpaceID) + }) + } +} + +func TestExperimentSandboxCleanupIncludesLegacyRecords(t *testing.T) { + ctx := context.Background() + svc := servicemocks.NewMockIEvalTargetService(gomock.NewController(t)) + failure := errors.New("consumer cleanup unavailable") + gomock.InOrder( + svc.EXPECT().TerminateAsyncRecordsAndDestroySandbox(ctx, int64(10), []int64{30, 31}, int32(1), "cancelled", false, true).Return(failure), + svc.EXPECT().TerminateAsyncRecordsAndDestroySandbox(ctx, int64(20), []int64{30, 31}, int32(1), "cancelled", false, true).Return(nil), + ) + require.ErrorIs(t, terminateExperimentTargetRecords(ctx, svc, 10, 20, []int64{30, 31}, 1, "cancelled", false, true), failure) +} + +func TestExperimentSandboxStatusIncludesBothGenerations(t *testing.T) { + ctx := context.Background() + svc := servicemocks.NewMockIEvalTargetService(gomock.NewController(t)) + svc.EXPECT().CheckSandboxTerminated(ctx, int64(10), []int64{30, 31}).Return([]int64{30}, map[int64]string{30: "Failed"}) + svc.EXPECT().CheckSandboxTerminated(ctx, int64(20), []int64{30, 31}).Return([]int64{31}, map[int64]string{31: "Canceled"}) + ids, status := checkExperimentSandboxTerminated(ctx, svc, 10, 20, []int64{30, 31}) + require.Equal(t, []int64{30, 31}, ids) + require.Equal(t, map[int64]string{30: "Failed", 31: "Canceled"}, status) +}