diff --git a/.github/workflows/backend-ci.yaml b/.github/workflows/backend-ci.yaml index 0b3a485d55..1ec05fa283 100644 --- a/.github/workflows/backend-ci.yaml +++ b/.github/workflows/backend-ci.yaml @@ -51,6 +51,30 @@ jobs: defaults: run: working-directory: ./backend + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: cozeloop + MYSQL_PASSWORD: cozeloop + MYSQL_DATABASE: expt_running_retry_test + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -ucozeloop -pcozeloop" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + redis: + image: redis:8.2.0 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 steps: - name: Checkout code uses: actions/checkout@v4 @@ -60,12 +84,33 @@ jobs: with: go-version: "1.24.0" + # Service containers start before checkout, so the init SQL (which lives + # in the repository) is applied here instead of via initdb.d. + - name: Initialize MySQL schema + run: | + set -euo pipefail + loaded=0 + for f in ../release/deployment/docker-compose/bootstrap/mysql-init/init-sql/*.sql; do + docker exec -i -e MYSQL_PWD=cozeloop "${{ job.services.mysql.id }}" \ + mysql --protocol=TCP -h 127.0.0.1 -ucozeloop expt_running_retry_test < "$f" + loaded=$((loaded + 1)) + done + echo "initialized ${loaded} schema files in expt_running_retry_test" + # Lock-observation tests read performance_schema.data_locks, which the + # application user cannot see without an explicit read grant. + docker exec -i -e MYSQL_PWD=root "${{ job.services.mysql.id }}" \ + mysql --protocol=TCP -h 127.0.0.1 -uroot \ + -e "GRANT SELECT ON performance_schema.* TO 'cozeloop'@'%'; FLUSH PRIVILEGES;" + - name: Build run: go build -v ./... - name: Run Unit Tests + env: + EXPT_RETRY_MYSQL_DSN: "cozeloop:cozeloop@tcp(127.0.0.1:3306)/expt_running_retry_test?charset=utf8mb4&parseTime=True&loc=Local" + EXPT_RETRY_REAL_REDIS_ADDR: "127.0.0.1:6379" run: | - go test -gcflags="all=-N -l" -race -v -coverprofile=coverage.out -coverpkg=./... ./... + go test -gcflags="all=-N -l" -race -v -timeout 45m -coverprofile=coverage.out -coverpkg=./... ./... - name: Upload Coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go b/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go index cac204cc38..c1786e3be3 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go @@ -4503,6 +4503,7 @@ type Experiment struct { ExpectedQuotaConsumption *ExpectedQuotaConsumption `thrift:"expected_quota_consumption,118,optional" frugal:"118,optional,ExpectedQuotaConsumption" form:"expected_quota_consumption" json:"expected_quota_consumption,omitempty" query:"expected_quota_consumption"` // 从 experiment.eval_conf 回显；缺省表示普通评测。 VerificationConfig *VerificationConfig `thrift:"verification_config,119,optional" frugal:"119,optional,VerificationConfig" form:"verification_config" json:"verification_config,omitempty" query:"verification_config"` + LatestRunID *int64 `thrift:"latest_run_id,120,optional" frugal:"120,optional,i64" json:"latest_run_id,omitempty" form:"latest_run_id" query:"latest_run_id"` } func NewExperiment() *Experiment { @@ -5123,6 +5124,18 @@ func (p *Experiment) GetVerificationConfig() (v *VerificationConfig) { } return p.VerificationConfig } + +var Experiment_LatestRunID_DEFAULT int64 + +func (p *Experiment) GetLatestRunID() (v int64) { + if p == nil { + return + } + if !p.IsSetLatestRunID() { + return Experiment_LatestRunID_DEFAULT + } + return *p.LatestRunID +} func (p *Experiment) SetID(val *int64) { p.ID = val } @@ -5276,6 +5289,9 @@ func (p *Experiment) SetExpectedQuotaConsumption(val *ExpectedQuotaConsumption) func (p *Experiment) SetVerificationConfig(val *VerificationConfig) { p.VerificationConfig = val } +func (p *Experiment) SetLatestRunID(val *int64) { + p.LatestRunID = val +} var fieldIDToName_Experiment = map[int16]string{ 1: "id", @@ -5329,6 +5345,7 @@ var fieldIDToName_Experiment = map[int16]string{ 117: "scheduler_mode", 118: "expected_quota_consumption", 119: "verification_config", + 120: "latest_run_id", } func (p *Experiment) IsSetID() bool { @@ -5535,6 +5552,10 @@ func (p *Experiment) IsSetVerificationConfig() bool { return p.VerificationConfig != nil } +func (p *Experiment) IsSetLatestRunID() bool { + return p.LatestRunID != nil +} + func (p *Experiment) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -5961,6 +5982,14 @@ func (p *Experiment) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 120: + if fieldTypeId == thrift.I64 { + if err = p.ReadField120(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } default: if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError @@ -6607,6 +6636,17 @@ func (p *Experiment) ReadField119(iprot thrift.TProtocol) error { p.VerificationConfig = _field return nil } +func (p *Experiment) ReadField120(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.LatestRunID = _field + return nil +} func (p *Experiment) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 @@ -6818,6 +6858,10 @@ func (p *Experiment) Write(oprot thrift.TProtocol) (err error) { fieldId = 119 goto WriteFieldError } + if err = p.writeField120(oprot); err != nil { + fieldId = 120 + goto WriteFieldError + } } if err = oprot.WriteFieldStop(); err != nil { goto WriteFieldStopError @@ -7813,6 +7857,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 119 end error: ", p), err) } +func (p *Experiment) writeField120(oprot thrift.TProtocol) (err error) { + if p.IsSetLatestRunID() { + if err = oprot.WriteFieldBegin("latest_run_id", thrift.I64, 120); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.LatestRunID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 120 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 120 end error: ", p), err) +} func (p *Experiment) String() string { if p == nil { @@ -7981,6 +8043,9 @@ func (p *Experiment) DeepEqual(ano *Experiment) bool { if !p.Field119DeepEqual(ano.VerificationConfig) { return false } + if !p.Field120DeepEqual(ano.LatestRunID) { + return false + } return true } @@ -8538,6 +8603,18 @@ func (p *Experiment) Field119DeepEqual(src *VerificationConfig) bool { } return true } +func (p *Experiment) Field120DeepEqual(src *int64) bool { + + if p.LatestRunID == src { + return true + } else if p.LatestRunID == nil || src == nil { + return false + } + if *p.LatestRunID != *src { + return false + } + return true +} // 实验模板基础信息 type ExptTemplateMeta struct { diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go b/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go index 6dcf1f7cf6..cc7f049548 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go @@ -2961,6 +2961,20 @@ func (p *Experiment) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 120: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField120(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } default: l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) offset += l @@ -3760,6 +3774,20 @@ func (p *Experiment) FastReadField119(buf []byte) (int, error) { return offset, nil } +func (p *Experiment) FastReadField120(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.LatestRunID = _field + return offset, nil +} + func (p *Experiment) FastWrite(buf []byte) int { return p.FastWriteNocopy(buf, nil) } @@ -3783,6 +3811,7 @@ func (p *Experiment) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { offset += p.fastWriteField113(buf[offset:], w) offset += p.fastWriteField114(buf[offset:], w) offset += p.fastWriteField116(buf[offset:], w) + offset += p.fastWriteField120(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) @@ -3877,6 +3906,7 @@ func (p *Experiment) BLength() int { l += p.field117Length() l += p.field118Length() l += p.field119Length() + l += p.field120Length() } l += thrift.Binary.FieldStopLength() return l @@ -4391,6 +4421,15 @@ func (p *Experiment) fastWriteField119(buf []byte, w thrift.NocopyWriter) int { return offset } +func (p *Experiment) fastWriteField120(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetLatestRunID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 120) + offset += thrift.Binary.WriteI64(buf[offset:], *p.LatestRunID) + } + return offset +} + func (p *Experiment) field1Length() int { l := 0 if p.IsSetID() { @@ -4878,6 +4917,15 @@ func (p *Experiment) field119Length() int { return l } +func (p *Experiment) field120Length() int { + l := 0 + if p.IsSetLatestRunID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + func (p *Experiment) DeepCopy(s interface{}) error { src, ok := s.(*Experiment) if !ok { @@ -5281,6 +5329,11 @@ func (p *Experiment) DeepCopy(s interface{}) error { } p.VerificationConfig = _verificationConfig + if src.LatestRunID != nil { + tmp := *src.LatestRunID + p.LatestRunID = &tmp + } + return nil } diff --git a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment.go b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment.go index ece70779ff..cd6bf1f3d2 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment.go @@ -9255,6 +9255,7 @@ type Experiment struct { // 单 item 预期资源消耗向量; "有则回显、无则省略"(legacy 实验确实没申报)。 ExpectedQuotaConsumption *ExpectedQuotaConsumption `thrift:"expected_quota_consumption,118,optional" frugal:"118,optional,ExpectedQuotaConsumption" form:"expected_quota_consumption" json:"expected_quota_consumption,omitempty" query:"expected_quota_consumption"` VerificationConfig *VerificationConfig `thrift:"verification_config,119,optional" frugal:"119,optional,VerificationConfig" form:"verification_config" json:"verification_config,omitempty" query:"verification_config"` + LatestRunID *int64 `thrift:"latest_run_id,120,optional" frugal:"120,optional,i64" json:"latest_run_id,omitempty" form:"latest_run_id" query:"latest_run_id"` // 注: scheduler_scope **不进读模型**。它是不透明调度域 ID, 对调用方无可用语义却泄露部署拓扑; // 内部运维需要时直接查 experiment.scheduler_scope 列。 BaseInfo *common.BaseInfo `thrift:"base_info,100,optional" frugal:"100,optional,common.BaseInfo" form:"base_info" json:"base_info,omitempty" query:"base_info"` @@ -9651,6 +9652,18 @@ func (p *Experiment) GetVerificationConfig() (v *VerificationConfig) { return p.VerificationConfig } +var Experiment_LatestRunID_DEFAULT int64 + +func (p *Experiment) GetLatestRunID() (v int64) { + if p == nil { + return + } + if !p.IsSetLatestRunID() { + return Experiment_LatestRunID_DEFAULT + } + return *p.LatestRunID +} + var Experiment_BaseInfo_DEFAULT *common.BaseInfo func (p *Experiment) GetBaseInfo() (v *common.BaseInfo) { @@ -9758,6 +9771,9 @@ func (p *Experiment) SetExpectedQuotaConsumption(val *ExpectedQuotaConsumption) func (p *Experiment) SetVerificationConfig(val *VerificationConfig) { p.VerificationConfig = val } +func (p *Experiment) SetLatestRunID(val *int64) { + p.LatestRunID = val +} func (p *Experiment) SetBaseInfo(val *common.BaseInfo) { p.BaseInfo = val } @@ -9795,6 +9811,7 @@ var fieldIDToName_Experiment = map[int16]string{ 117: "scheduler_mode", 118: "expected_quota_consumption", 119: "verification_config", + 120: "latest_run_id", 100: "base_info", } @@ -9926,6 +9943,10 @@ func (p *Experiment) IsSetVerificationConfig() bool { return p.VerificationConfig != nil } +func (p *Experiment) IsSetLatestRunID() bool { + return p.LatestRunID != nil +} + func (p *Experiment) IsSetBaseInfo() bool { return p.BaseInfo != nil } @@ -10204,6 +10225,14 @@ func (p *Experiment) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 120: + if fieldTypeId == thrift.I64 { + if err = p.ReadField120(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 100: if fieldTypeId == thrift.STRUCT { if err = p.ReadField100(iprot); err != nil { @@ -10611,6 +10640,17 @@ func (p *Experiment) ReadField119(iprot thrift.TProtocol) error { p.VerificationConfig = _field return nil } +func (p *Experiment) ReadField120(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.LatestRunID = _field + return nil +} func (p *Experiment) ReadField100(iprot thrift.TProtocol) error { _field := common.NewBaseInfo() if err := _field.Read(iprot); err != nil { @@ -10754,6 +10794,10 @@ func (p *Experiment) Write(oprot thrift.TProtocol) (err error) { fieldId = 119 goto WriteFieldError } + if err = p.writeField120(oprot); err != nil { + fieldId = 120 + goto WriteFieldError + } if err = p.writeField100(oprot); err != nil { fieldId = 100 goto WriteFieldError @@ -11384,6 +11428,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 119 end error: ", p), err) } +func (p *Experiment) writeField120(oprot thrift.TProtocol) (err error) { + if p.IsSetLatestRunID() { + if err = oprot.WriteFieldBegin("latest_run_id", thrift.I64, 120); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.LatestRunID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 120 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 120 end error: ", p), err) +} func (p *Experiment) writeField100(oprot thrift.TProtocol) (err error) { if p.IsSetBaseInfo() { if err = oprot.WriteFieldBegin("base_info", thrift.STRUCT, 100); err != nil { @@ -11513,6 +11575,9 @@ func (p *Experiment) DeepEqual(ano *Experiment) bool { if !p.Field119DeepEqual(ano.VerificationConfig) { return false } + if !p.Field120DeepEqual(ano.LatestRunID) { + return false + } if !p.Field100DeepEqual(ano.BaseInfo) { return false } @@ -11857,6 +11922,18 @@ func (p *Experiment) Field119DeepEqual(src *VerificationConfig) bool { } return true } +func (p *Experiment) Field120DeepEqual(src *int64) bool { + + if p.LatestRunID == src { + return true + } else if p.LatestRunID == nil || src == nil { + return false + } + if *p.LatestRunID != *src { + return false + } + return true +} func (p *Experiment) Field100DeepEqual(src *common.BaseInfo) bool { if !p.BaseInfo.DeepEqual(src) { @@ -11865,6 +11942,264 @@ func (p *Experiment) Field100DeepEqual(src *common.BaseInfo) bool { return true } +type RetryRejectedItem struct { + ItemID *int64 `thrift:"item_id,1,optional" frugal:"1,optional,i64" json:"item_id,omitempty" form:"item_id" query:"item_id"` + Reason *string `thrift:"reason,2,optional" frugal:"2,optional,string" form:"reason" json:"reason,omitempty" query:"reason"` +} + +func NewRetryRejectedItem() *RetryRejectedItem { + return &RetryRejectedItem{} +} + +func (p *RetryRejectedItem) InitDefault() { +} + +var RetryRejectedItem_ItemID_DEFAULT int64 + +func (p *RetryRejectedItem) GetItemID() (v int64) { + if p == nil { + return + } + if !p.IsSetItemID() { + return RetryRejectedItem_ItemID_DEFAULT + } + return *p.ItemID +} + +var RetryRejectedItem_Reason_DEFAULT string + +func (p *RetryRejectedItem) GetReason() (v string) { + if p == nil { + return + } + if !p.IsSetReason() { + return RetryRejectedItem_Reason_DEFAULT + } + return *p.Reason +} +func (p *RetryRejectedItem) SetItemID(val *int64) { + p.ItemID = val +} +func (p *RetryRejectedItem) SetReason(val *string) { + p.Reason = val +} + +var fieldIDToName_RetryRejectedItem = map[int16]string{ + 1: "item_id", + 2: "reason", +} + +func (p *RetryRejectedItem) IsSetItemID() bool { + return p.ItemID != nil +} + +func (p *RetryRejectedItem) IsSetReason() bool { + return p.Reason != nil +} + +func (p *RetryRejectedItem) Read(iprot thrift.TProtocol) (err error) { + var fieldTypeId thrift.TType + var fieldId int16 + + if _, err = iprot.ReadStructBegin(); err != nil { + goto ReadStructBeginError + } + + for { + _, fieldTypeId, fieldId, err = iprot.ReadFieldBegin() + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + + switch fieldId { + case 1: + if fieldTypeId == thrift.I64 { + if err = p.ReadField1(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 2: + if fieldTypeId == thrift.STRING { + if err = p.ReadField2(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + default: + if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + } + if err = iprot.ReadFieldEnd(); err != nil { + goto ReadFieldEndError + } + } + if err = iprot.ReadStructEnd(); err != nil { + goto ReadStructEndError + } + + return nil +ReadStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T read struct begin error: ", p), err) +ReadFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_RetryRejectedItem[fieldId]), err) +SkipFieldError: + return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) + +ReadFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T read field end error", p), err) +ReadStructEndError: + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) +} + +func (p *RetryRejectedItem) ReadField1(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.ItemID = _field + return nil +} +func (p *RetryRejectedItem) ReadField2(iprot thrift.TProtocol) error { + + var _field *string + if v, err := iprot.ReadString(); err != nil { + return err + } else { + _field = &v + } + p.Reason = _field + return nil +} + +func (p *RetryRejectedItem) Write(oprot thrift.TProtocol) (err error) { + var fieldId int16 + if err = oprot.WriteStructBegin("RetryRejectedItem"); err != nil { + goto WriteStructBeginError + } + if p != nil { + if err = p.writeField1(oprot); err != nil { + fieldId = 1 + goto WriteFieldError + } + if err = p.writeField2(oprot); err != nil { + fieldId = 2 + goto WriteFieldError + } + } + if err = oprot.WriteFieldStop(); err != nil { + goto WriteFieldStopError + } + if err = oprot.WriteStructEnd(); err != nil { + goto WriteStructEndError + } + return nil +WriteStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) +WriteFieldError: + return thrift.PrependError(fmt.Sprintf("%T write field %d error: ", p, fieldId), err) +WriteFieldStopError: + return thrift.PrependError(fmt.Sprintf("%T write field stop error: ", p), err) +WriteStructEndError: + return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) +} + +func (p *RetryRejectedItem) writeField1(oprot thrift.TProtocol) (err error) { + if p.IsSetItemID() { + if err = oprot.WriteFieldBegin("item_id", thrift.I64, 1); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.ItemID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) +} +func (p *RetryRejectedItem) writeField2(oprot thrift.TProtocol) (err error) { + if p.IsSetReason() { + if err = oprot.WriteFieldBegin("reason", thrift.STRING, 2); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteString(*p.Reason); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 2 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 2 end error: ", p), err) +} + +func (p *RetryRejectedItem) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("RetryRejectedItem(%+v)", *p) + +} + +func (p *RetryRejectedItem) DeepEqual(ano *RetryRejectedItem) bool { + if p == ano { + return true + } else if p == nil || ano == nil { + return false + } + if !p.Field1DeepEqual(ano.ItemID) { + return false + } + if !p.Field2DeepEqual(ano.Reason) { + return false + } + return true +} + +func (p *RetryRejectedItem) Field1DeepEqual(src *int64) bool { + + if p.ItemID == src { + return true + } else if p.ItemID == nil || src == nil { + return false + } + if *p.ItemID != *src { + return false + } + return true +} +func (p *RetryRejectedItem) Field2DeepEqual(src *string) bool { + + if p.Reason == src { + return true + } else if p.Reason == nil || src == nil { + return false + } + if strings.Compare(*p.Reason, *src) != 0 { + return false + } + return true +} + // 单资源预期消耗。与 domain/expt.thrift 的 ExpectedResourceConsumption 同构 —— // 本文件另立一份而非 include, 与 RunModeConfig / ExptEvalSetSourceType 同套模式, 避免符号冲突。 type ExpectedResourceConsumption struct { diff --git a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment_validator.go b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment_validator.go index 2d21124df0..9eb60bc730 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment_validator.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/experiment_validator.go @@ -191,6 +191,9 @@ func (p *Experiment) IsValid() error { } return nil } +func (p *RetryRejectedItem) IsValid() error { + return nil +} func (p *ExpectedResourceConsumption) IsValid() error { return nil } diff --git a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/k-experiment.go b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/k-experiment.go index 38f86b7cba..d007b1619b 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/k-experiment.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/experiment/k-experiment.go @@ -6864,6 +6864,20 @@ func (p *Experiment) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 120: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField120(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 100: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField100(buf[offset:]) @@ -7368,6 +7382,20 @@ func (p *Experiment) FastReadField119(buf []byte) (int, error) { return offset, nil } +func (p *Experiment) FastReadField120(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.LatestRunID = _field + return offset, nil +} + func (p *Experiment) FastReadField100(buf []byte) (int, error) { offset := 0 _field := common.NewBaseInfo() @@ -7398,6 +7426,7 @@ func (p *Experiment) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { offset += p.fastWriteField113(buf[offset:], w) offset += p.fastWriteField114(buf[offset:], w) offset += p.fastWriteField116(buf[offset:], w) + offset += p.fastWriteField120(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) @@ -7460,6 +7489,7 @@ func (p *Experiment) BLength() int { l += p.field117Length() l += p.field118Length() l += p.field119Length() + l += p.field120Length() l += p.field100Length() } l += thrift.Binary.FieldStopLength() @@ -7782,6 +7812,15 @@ func (p *Experiment) fastWriteField119(buf []byte, w thrift.NocopyWriter) int { return offset } +func (p *Experiment) fastWriteField120(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetLatestRunID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 120) + offset += thrift.Binary.WriteI64(buf[offset:], *p.LatestRunID) + } + return offset +} + func (p *Experiment) fastWriteField100(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetBaseInfo() { @@ -8095,6 +8134,15 @@ func (p *Experiment) field119Length() int { return l } +func (p *Experiment) field120Length() int { + l := 0 + if p.IsSetLatestRunID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + func (p *Experiment) field100Length() int { l := 0 if p.IsSetBaseInfo() { @@ -8362,6 +8410,11 @@ func (p *Experiment) DeepCopy(s interface{}) error { } p.VerificationConfig = _verificationConfig + if src.LatestRunID != nil { + tmp := *src.LatestRunID + p.LatestRunID = &tmp + } + var _baseInfo *common.BaseInfo if src.BaseInfo != nil { _baseInfo = &common.BaseInfo{} @@ -8374,6 +8427,179 @@ func (p *Experiment) DeepCopy(s interface{}) error { return nil } +func (p *RetryRejectedItem) FastRead(buf []byte) (int, error) { + + var err error + var offset int + var l int + var fieldTypeId thrift.TType + var fieldId int16 + for { + fieldTypeId, fieldId, l, err = thrift.Binary.ReadFieldBegin(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField1(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 2: + if fieldTypeId == thrift.STRING { + l, err = p.FastReadField2(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + default: + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + } + + return offset, nil +ReadFieldBeginError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_RetryRejectedItem[fieldId]), err) +SkipFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) +} + +func (p *RetryRejectedItem) FastReadField1(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemID = _field + return offset, nil +} + +func (p *RetryRejectedItem) FastReadField2(buf []byte) (int, error) { + offset := 0 + + var _field *string + if v, l, err := thrift.Binary.ReadString(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.Reason = _field + return offset, nil +} + +func (p *RetryRejectedItem) FastWrite(buf []byte) int { + return p.FastWriteNocopy(buf, nil) +} + +func (p *RetryRejectedItem) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p != nil { + offset += p.fastWriteField1(buf[offset:], w) + offset += p.fastWriteField2(buf[offset:], w) + } + offset += thrift.Binary.WriteFieldStop(buf[offset:]) + return offset +} + +func (p *RetryRejectedItem) BLength() int { + l := 0 + if p != nil { + l += p.field1Length() + l += p.field2Length() + } + l += thrift.Binary.FieldStopLength() + return l +} + +func (p *RetryRejectedItem) fastWriteField1(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 1) + offset += thrift.Binary.WriteI64(buf[offset:], *p.ItemID) + } + return offset +} + +func (p *RetryRejectedItem) fastWriteField2(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetReason() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRING, 2) + offset += thrift.Binary.WriteStringNocopy(buf[offset:], w, *p.Reason) + } + return offset +} + +func (p *RetryRejectedItem) field1Length() int { + l := 0 + if p.IsSetItemID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + +func (p *RetryRejectedItem) field2Length() int { + l := 0 + if p.IsSetReason() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.StringLengthNocopy(*p.Reason) + } + return l +} + +func (p *RetryRejectedItem) DeepCopy(s interface{}) error { + src, ok := s.(*RetryRejectedItem) + if !ok { + return fmt.Errorf("%T's type not matched %T", s, p) + } + + if src.ItemID != nil { + tmp := *src.ItemID + p.ItemID = &tmp + } + + if src.Reason != nil { + var tmp string + if *src.Reason != "" { + tmp = kutils.StringDeepCopy(*src.Reason) + } + p.Reason = &tmp + } + + return nil +} + func (p *ExpectedResourceConsumption) FastRead(buf []byte) (int, error) { var err error diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go index ff7725ea00..c298b91b04 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go @@ -13601,12 +13601,14 @@ func (p *RunExperimentResponse) Field255DeepEqual(src *base.BaseResp) bool { } type RetryExperimentRequest struct { - RetryMode *expt.ExptRetryMode `thrift:"retry_mode,1,optional" frugal:"1,optional,ExptRetryMode" form:"retry_mode" json:"retry_mode,omitempty"` - WorkspaceID *int64 `thrift:"workspace_id,2,optional" frugal:"2,optional,i64" json:"workspace_id" form:"workspace_id" ` - ExptID *int64 `thrift:"expt_id,3,optional" frugal:"3,optional,i64" json:"expt_id" path:"expt_id" ` - ItemIds []int64 `thrift:"item_ids,4,optional" frugal:"4,optional,list" json:"item_ids" form:"item_ids" ` - Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` - Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` + RetryMode *expt.ExptRetryMode `thrift:"retry_mode,1,optional" frugal:"1,optional,ExptRetryMode" form:"retry_mode" json:"retry_mode,omitempty"` + WorkspaceID *int64 `thrift:"workspace_id,2,optional" frugal:"2,optional,i64" json:"workspace_id" form:"workspace_id" ` + ExptID *int64 `thrift:"expt_id,3,optional" frugal:"3,optional,i64" json:"expt_id" path:"expt_id" ` + ItemIds []int64 `thrift:"item_ids,4,optional" frugal:"4,optional,list" json:"item_ids" form:"item_ids" ` + ExpectedRunID *int64 `thrift:"expected_run_id,5,optional" frugal:"5,optional,i64" json:"expected_run_id,omitempty" form:"expected_run_id" ` + RequestID *string `thrift:"request_id,6,optional" frugal:"6,optional,string" form:"request_id" json:"request_id,omitempty"` + Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` + Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } func NewRetryExperimentRequest() *RetryExperimentRequest { @@ -13664,6 +13666,30 @@ func (p *RetryExperimentRequest) GetItemIds() (v []int64) { return p.ItemIds } +var RetryExperimentRequest_ExpectedRunID_DEFAULT int64 + +func (p *RetryExperimentRequest) GetExpectedRunID() (v int64) { + if p == nil { + return + } + if !p.IsSetExpectedRunID() { + return RetryExperimentRequest_ExpectedRunID_DEFAULT + } + return *p.ExpectedRunID +} + +var RetryExperimentRequest_RequestID_DEFAULT string + +func (p *RetryExperimentRequest) GetRequestID() (v string) { + if p == nil { + return + } + if !p.IsSetRequestID() { + return RetryExperimentRequest_RequestID_DEFAULT + } + return *p.RequestID +} + var RetryExperimentRequest_Ext_DEFAULT map[string]string func (p *RetryExperimentRequest) GetExt() (v map[string]string) { @@ -13699,6 +13725,12 @@ func (p *RetryExperimentRequest) SetExptID(val *int64) { func (p *RetryExperimentRequest) SetItemIds(val []int64) { p.ItemIds = val } +func (p *RetryExperimentRequest) SetExpectedRunID(val *int64) { + p.ExpectedRunID = val +} +func (p *RetryExperimentRequest) SetRequestID(val *string) { + p.RequestID = val +} func (p *RetryExperimentRequest) SetExt(val map[string]string) { p.Ext = val } @@ -13711,6 +13743,8 @@ var fieldIDToName_RetryExperimentRequest = map[int16]string{ 2: "workspace_id", 3: "expt_id", 4: "item_ids", + 5: "expected_run_id", + 6: "request_id", 100: "ext", 255: "Base", } @@ -13731,6 +13765,14 @@ func (p *RetryExperimentRequest) IsSetItemIds() bool { return p.ItemIds != nil } +func (p *RetryExperimentRequest) IsSetExpectedRunID() bool { + return p.ExpectedRunID != nil +} + +func (p *RetryExperimentRequest) IsSetRequestID() bool { + return p.RequestID != nil +} + func (p *RetryExperimentRequest) IsSetExt() bool { return p.Ext != nil } @@ -13789,6 +13831,22 @@ func (p *RetryExperimentRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 5: + if fieldTypeId == thrift.I64 { + if err = p.ReadField5(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 6: + if fieldTypeId == thrift.STRING { + if err = p.ReadField6(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 100: if fieldTypeId == thrift.MAP { if err = p.ReadField100(iprot); err != nil { @@ -13891,6 +13949,28 @@ func (p *RetryExperimentRequest) ReadField4(iprot thrift.TProtocol) error { p.ItemIds = _field return nil } +func (p *RetryExperimentRequest) ReadField5(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.ExpectedRunID = _field + return nil +} +func (p *RetryExperimentRequest) ReadField6(iprot thrift.TProtocol) error { + + var _field *string + if v, err := iprot.ReadString(); err != nil { + return err + } else { + _field = &v + } + p.RequestID = _field + return nil +} func (p *RetryExperimentRequest) ReadField100(iprot thrift.TProtocol) error { _, _, size, err := iprot.ReadMapBegin() if err != nil { @@ -13951,6 +14031,14 @@ func (p *RetryExperimentRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 4 goto WriteFieldError } + if err = p.writeField5(oprot); err != nil { + fieldId = 5 + goto WriteFieldError + } + if err = p.writeField6(oprot); err != nil { + fieldId = 6 + goto WriteFieldError + } if err = p.writeField100(oprot); err != nil { fieldId = 100 goto WriteFieldError @@ -14057,6 +14145,42 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 4 end error: ", p), err) } +func (p *RetryExperimentRequest) writeField5(oprot thrift.TProtocol) (err error) { + if p.IsSetExpectedRunID() { + if err = oprot.WriteFieldBegin("expected_run_id", thrift.I64, 5); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.ExpectedRunID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) +} +func (p *RetryExperimentRequest) writeField6(oprot thrift.TProtocol) (err error) { + if p.IsSetRequestID() { + if err = oprot.WriteFieldBegin("request_id", thrift.STRING, 6); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteString(*p.RequestID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 end error: ", p), err) +} func (p *RetryExperimentRequest) writeField100(oprot thrift.TProtocol) (err error) { if p.IsSetExt() { if err = oprot.WriteFieldBegin("ext", thrift.MAP, 100); err != nil { @@ -14131,6 +14255,12 @@ func (p *RetryExperimentRequest) DeepEqual(ano *RetryExperimentRequest) bool { if !p.Field4DeepEqual(ano.ItemIds) { return false } + if !p.Field5DeepEqual(ano.ExpectedRunID) { + return false + } + if !p.Field6DeepEqual(ano.RequestID) { + return false + } if !p.Field100DeepEqual(ano.Ext) { return false } @@ -14189,6 +14319,30 @@ func (p *RetryExperimentRequest) Field4DeepEqual(src []int64) bool { } return true } +func (p *RetryExperimentRequest) Field5DeepEqual(src *int64) bool { + + if p.ExpectedRunID == src { + return true + } else if p.ExpectedRunID == nil || src == nil { + return false + } + if *p.ExpectedRunID != *src { + return false + } + return true +} +func (p *RetryExperimentRequest) Field6DeepEqual(src *string) bool { + + if p.RequestID == src { + return true + } else if p.RequestID == nil || src == nil { + return false + } + if strings.Compare(*p.RequestID, *src) != 0 { + return false + } + return true +} func (p *RetryExperimentRequest) Field100DeepEqual(src map[string]string) bool { if len(p.Ext) != len(src) { @@ -14210,62 +14364,62 @@ func (p *RetryExperimentRequest) Field255DeepEqual(src *base.Base) bool { return true } -type RetryExperimentResponse struct { - RunID *int64 `thrift:"run_id,1,optional" frugal:"1,optional,i64" json:"run_id" form:"run_id" ` - BaseResp *base.BaseResp `thrift:"BaseResp,255" frugal:"255,default,base.BaseResp" form:"BaseResp" json:"BaseResp" query:"BaseResp"` +type RetryRejectedItem struct { + ItemID *int64 `thrift:"item_id,1,optional" frugal:"1,optional,i64" json:"item_id,omitempty" form:"item_id" query:"item_id"` + Reason *string `thrift:"reason,2,optional" frugal:"2,optional,string" form:"reason" json:"reason,omitempty" query:"reason"` } -func NewRetryExperimentResponse() *RetryExperimentResponse { - return &RetryExperimentResponse{} +func NewRetryRejectedItem() *RetryRejectedItem { + return &RetryRejectedItem{} } -func (p *RetryExperimentResponse) InitDefault() { +func (p *RetryRejectedItem) InitDefault() { } -var RetryExperimentResponse_RunID_DEFAULT int64 +var RetryRejectedItem_ItemID_DEFAULT int64 -func (p *RetryExperimentResponse) GetRunID() (v int64) { +func (p *RetryRejectedItem) GetItemID() (v int64) { if p == nil { return } - if !p.IsSetRunID() { - return RetryExperimentResponse_RunID_DEFAULT + if !p.IsSetItemID() { + return RetryRejectedItem_ItemID_DEFAULT } - return *p.RunID + return *p.ItemID } -var RetryExperimentResponse_BaseResp_DEFAULT *base.BaseResp +var RetryRejectedItem_Reason_DEFAULT string -func (p *RetryExperimentResponse) GetBaseResp() (v *base.BaseResp) { +func (p *RetryRejectedItem) GetReason() (v string) { if p == nil { return } - if !p.IsSetBaseResp() { - return RetryExperimentResponse_BaseResp_DEFAULT + if !p.IsSetReason() { + return RetryRejectedItem_Reason_DEFAULT } - return p.BaseResp + return *p.Reason } -func (p *RetryExperimentResponse) SetRunID(val *int64) { - p.RunID = val +func (p *RetryRejectedItem) SetItemID(val *int64) { + p.ItemID = val } -func (p *RetryExperimentResponse) SetBaseResp(val *base.BaseResp) { - p.BaseResp = val +func (p *RetryRejectedItem) SetReason(val *string) { + p.Reason = val } -var fieldIDToName_RetryExperimentResponse = map[int16]string{ - 1: "run_id", - 255: "BaseResp", +var fieldIDToName_RetryRejectedItem = map[int16]string{ + 1: "item_id", + 2: "reason", } -func (p *RetryExperimentResponse) IsSetRunID() bool { - return p.RunID != nil +func (p *RetryRejectedItem) IsSetItemID() bool { + return p.ItemID != nil } -func (p *RetryExperimentResponse) IsSetBaseResp() bool { - return p.BaseResp != nil +func (p *RetryRejectedItem) IsSetReason() bool { + return p.Reason != nil } -func (p *RetryExperimentResponse) Read(iprot thrift.TProtocol) (err error) { +func (p *RetryRejectedItem) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -14291,9 +14445,9 @@ func (p *RetryExperimentResponse) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } - case 255: - if fieldTypeId == thrift.STRUCT { - if err = p.ReadField255(iprot); err != nil { + case 2: + if fieldTypeId == thrift.STRING { + if err = p.ReadField2(iprot); err != nil { goto ReadFieldError } } else if err = iprot.Skip(fieldTypeId); err != nil { @@ -14318,7 +14472,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_RetryExperimentResponse[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_RetryRejectedItem[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -14328,7 +14482,7 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *RetryExperimentResponse) ReadField1(iprot thrift.TProtocol) error { +func (p *RetryRejectedItem) ReadField1(iprot thrift.TProtocol) error { var _field *int64 if v, err := iprot.ReadI64(); err != nil { @@ -14336,21 +14490,24 @@ func (p *RetryExperimentResponse) ReadField1(iprot thrift.TProtocol) error { } else { _field = &v } - p.RunID = _field + p.ItemID = _field return nil } -func (p *RetryExperimentResponse) ReadField255(iprot thrift.TProtocol) error { - _field := base.NewBaseResp() - if err := _field.Read(iprot); err != nil { +func (p *RetryRejectedItem) ReadField2(iprot thrift.TProtocol) error { + + var _field *string + if v, err := iprot.ReadString(); err != nil { return err + } else { + _field = &v } - p.BaseResp = _field + p.Reason = _field return nil } -func (p *RetryExperimentResponse) Write(oprot thrift.TProtocol) (err error) { +func (p *RetryRejectedItem) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("RetryExperimentResponse"); err != nil { + if err = oprot.WriteStructBegin("RetryRejectedItem"); err != nil { goto WriteStructBeginError } if p != nil { @@ -14358,8 +14515,8 @@ func (p *RetryExperimentResponse) Write(oprot thrift.TProtocol) (err error) { fieldId = 1 goto WriteFieldError } - if err = p.writeField255(oprot); err != nil { - fieldId = 255 + if err = p.writeField2(oprot); err != nil { + fieldId = 2 goto WriteFieldError } } @@ -14380,12 +14537,12 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *RetryExperimentResponse) writeField1(oprot thrift.TProtocol) (err error) { - if p.IsSetRunID() { - if err = oprot.WriteFieldBegin("run_id", thrift.I64, 1); err != nil { +func (p *RetryRejectedItem) writeField1(oprot thrift.TProtocol) (err error) { + if p.IsSetItemID() { + if err = oprot.WriteFieldBegin("item_id", thrift.I64, 1); err != nil { goto WriteFieldBeginError } - if err := oprot.WriteI64(*p.RunID); err != nil { + if err := oprot.WriteI64(*p.ItemID); err != nil { return err } if err = oprot.WriteFieldEnd(); err != nil { @@ -14398,54 +14555,638 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) } -func (p *RetryExperimentResponse) writeField255(oprot thrift.TProtocol) (err error) { - if err = oprot.WriteFieldBegin("BaseResp", thrift.STRUCT, 255); err != nil { - goto WriteFieldBeginError - } - if err := p.BaseResp.Write(oprot); err != nil { - return err - } - if err = oprot.WriteFieldEnd(); err != nil { - goto WriteFieldEndError +func (p *RetryRejectedItem) writeField2(oprot thrift.TProtocol) (err error) { + if p.IsSetReason() { + if err = oprot.WriteFieldBegin("reason", thrift.STRING, 2); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteString(*p.Reason); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } } return nil WriteFieldBeginError: - return thrift.PrependError(fmt.Sprintf("%T write field 255 begin error: ", p), err) + return thrift.PrependError(fmt.Sprintf("%T write field 2 begin error: ", p), err) WriteFieldEndError: - return thrift.PrependError(fmt.Sprintf("%T write field 255 end error: ", p), err) + return thrift.PrependError(fmt.Sprintf("%T write field 2 end error: ", p), err) } -func (p *RetryExperimentResponse) String() string { +func (p *RetryRejectedItem) String() string { if p == nil { return "" } - return fmt.Sprintf("RetryExperimentResponse(%+v)", *p) + return fmt.Sprintf("RetryRejectedItem(%+v)", *p) } -func (p *RetryExperimentResponse) DeepEqual(ano *RetryExperimentResponse) bool { +func (p *RetryRejectedItem) DeepEqual(ano *RetryRejectedItem) bool { if p == ano { return true } else if p == nil || ano == nil { return false } - if !p.Field1DeepEqual(ano.RunID) { + if !p.Field1DeepEqual(ano.ItemID) { return false } - if !p.Field255DeepEqual(ano.BaseResp) { + if !p.Field2DeepEqual(ano.Reason) { return false } return true } -func (p *RetryExperimentResponse) Field1DeepEqual(src *int64) bool { +func (p *RetryRejectedItem) Field1DeepEqual(src *int64) bool { - if p.RunID == src { + if p.ItemID == src { return true - } else if p.RunID == nil || src == nil { + } else if p.ItemID == nil || src == nil { return false } - if *p.RunID != *src { + if *p.ItemID != *src { + return false + } + return true +} +func (p *RetryRejectedItem) Field2DeepEqual(src *string) bool { + + if p.Reason == src { + return true + } else if p.Reason == nil || src == nil { + return false + } + if strings.Compare(*p.Reason, *src) != 0 { + return false + } + return true +} + +type RetryExperimentResponse struct { + RunID *int64 `thrift:"run_id,1,optional" frugal:"1,optional,i64" json:"run_id" form:"run_id" ` + RequestID *string `thrift:"request_id,2,optional" frugal:"2,optional,string" form:"request_id" json:"request_id,omitempty" query:"request_id"` + AcceptedCount *int32 `thrift:"accepted_count,3,optional" frugal:"3,optional,i32" form:"accepted_count" json:"accepted_count,omitempty" query:"accepted_count"` + RejectedItems []*RetryRejectedItem `thrift:"rejected_items,4,optional" frugal:"4,optional,list" form:"rejected_items" json:"rejected_items,omitempty" query:"rejected_items"` + SnapshotAtMs *int64 `thrift:"snapshot_at_ms,5,optional" frugal:"5,optional,i64" json:"snapshot_at_ms,omitempty" form:"snapshot_at_ms" query:"snapshot_at_ms"` + BaseResp *base.BaseResp `thrift:"BaseResp,255" frugal:"255,default,base.BaseResp" form:"BaseResp" json:"BaseResp" query:"BaseResp"` +} + +func NewRetryExperimentResponse() *RetryExperimentResponse { + return &RetryExperimentResponse{} +} + +func (p *RetryExperimentResponse) InitDefault() { +} + +var RetryExperimentResponse_RunID_DEFAULT int64 + +func (p *RetryExperimentResponse) GetRunID() (v int64) { + if p == nil { + return + } + if !p.IsSetRunID() { + return RetryExperimentResponse_RunID_DEFAULT + } + return *p.RunID +} + +var RetryExperimentResponse_RequestID_DEFAULT string + +func (p *RetryExperimentResponse) GetRequestID() (v string) { + if p == nil { + return + } + if !p.IsSetRequestID() { + return RetryExperimentResponse_RequestID_DEFAULT + } + return *p.RequestID +} + +var RetryExperimentResponse_AcceptedCount_DEFAULT int32 + +func (p *RetryExperimentResponse) GetAcceptedCount() (v int32) { + if p == nil { + return + } + if !p.IsSetAcceptedCount() { + return RetryExperimentResponse_AcceptedCount_DEFAULT + } + return *p.AcceptedCount +} + +var RetryExperimentResponse_RejectedItems_DEFAULT []*RetryRejectedItem + +func (p *RetryExperimentResponse) GetRejectedItems() (v []*RetryRejectedItem) { + if p == nil { + return + } + if !p.IsSetRejectedItems() { + return RetryExperimentResponse_RejectedItems_DEFAULT + } + return p.RejectedItems +} + +var RetryExperimentResponse_SnapshotAtMs_DEFAULT int64 + +func (p *RetryExperimentResponse) GetSnapshotAtMs() (v int64) { + if p == nil { + return + } + if !p.IsSetSnapshotAtMs() { + return RetryExperimentResponse_SnapshotAtMs_DEFAULT + } + return *p.SnapshotAtMs +} + +var RetryExperimentResponse_BaseResp_DEFAULT *base.BaseResp + +func (p *RetryExperimentResponse) GetBaseResp() (v *base.BaseResp) { + if p == nil { + return + } + if !p.IsSetBaseResp() { + return RetryExperimentResponse_BaseResp_DEFAULT + } + return p.BaseResp +} +func (p *RetryExperimentResponse) SetRunID(val *int64) { + p.RunID = val +} +func (p *RetryExperimentResponse) SetRequestID(val *string) { + p.RequestID = val +} +func (p *RetryExperimentResponse) SetAcceptedCount(val *int32) { + p.AcceptedCount = val +} +func (p *RetryExperimentResponse) SetRejectedItems(val []*RetryRejectedItem) { + p.RejectedItems = val +} +func (p *RetryExperimentResponse) SetSnapshotAtMs(val *int64) { + p.SnapshotAtMs = val +} +func (p *RetryExperimentResponse) SetBaseResp(val *base.BaseResp) { + p.BaseResp = val +} + +var fieldIDToName_RetryExperimentResponse = map[int16]string{ + 1: "run_id", + 2: "request_id", + 3: "accepted_count", + 4: "rejected_items", + 5: "snapshot_at_ms", + 255: "BaseResp", +} + +func (p *RetryExperimentResponse) IsSetRunID() bool { + return p.RunID != nil +} + +func (p *RetryExperimentResponse) IsSetRequestID() bool { + return p.RequestID != nil +} + +func (p *RetryExperimentResponse) IsSetAcceptedCount() bool { + return p.AcceptedCount != nil +} + +func (p *RetryExperimentResponse) IsSetRejectedItems() bool { + return p.RejectedItems != nil +} + +func (p *RetryExperimentResponse) IsSetSnapshotAtMs() bool { + return p.SnapshotAtMs != nil +} + +func (p *RetryExperimentResponse) IsSetBaseResp() bool { + return p.BaseResp != nil +} + +func (p *RetryExperimentResponse) Read(iprot thrift.TProtocol) (err error) { + var fieldTypeId thrift.TType + var fieldId int16 + + if _, err = iprot.ReadStructBegin(); err != nil { + goto ReadStructBeginError + } + + for { + _, fieldTypeId, fieldId, err = iprot.ReadFieldBegin() + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + + switch fieldId { + case 1: + if fieldTypeId == thrift.I64 { + if err = p.ReadField1(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 2: + if fieldTypeId == thrift.STRING { + if err = p.ReadField2(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 3: + if fieldTypeId == thrift.I32 { + if err = p.ReadField3(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 4: + if fieldTypeId == thrift.LIST { + if err = p.ReadField4(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 5: + if fieldTypeId == thrift.I64 { + if err = p.ReadField5(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 255: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField255(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + default: + if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + } + if err = iprot.ReadFieldEnd(); err != nil { + goto ReadFieldEndError + } + } + if err = iprot.ReadStructEnd(); err != nil { + goto ReadStructEndError + } + + return nil +ReadStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T read struct begin error: ", p), err) +ReadFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_RetryExperimentResponse[fieldId]), err) +SkipFieldError: + return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) + +ReadFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T read field end error", p), err) +ReadStructEndError: + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) +} + +func (p *RetryExperimentResponse) ReadField1(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.RunID = _field + return nil +} +func (p *RetryExperimentResponse) ReadField2(iprot thrift.TProtocol) error { + + var _field *string + if v, err := iprot.ReadString(); err != nil { + return err + } else { + _field = &v + } + p.RequestID = _field + return nil +} +func (p *RetryExperimentResponse) ReadField3(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.AcceptedCount = _field + return nil +} +func (p *RetryExperimentResponse) ReadField4(iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin() + if err != nil { + return err + } + _field := make([]*RetryRejectedItem, 0, size) + values := make([]RetryRejectedItem, size) + for i := 0; i < size; i++ { + _elem := &values[i] + _elem.InitDefault() + + if err := _elem.Read(iprot); err != nil { + return err + } + + _field = append(_field, _elem) + } + if err := iprot.ReadListEnd(); err != nil { + return err + } + p.RejectedItems = _field + return nil +} +func (p *RetryExperimentResponse) ReadField5(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.SnapshotAtMs = _field + return nil +} +func (p *RetryExperimentResponse) ReadField255(iprot thrift.TProtocol) error { + _field := base.NewBaseResp() + if err := _field.Read(iprot); err != nil { + return err + } + p.BaseResp = _field + return nil +} + +func (p *RetryExperimentResponse) Write(oprot thrift.TProtocol) (err error) { + var fieldId int16 + if err = oprot.WriteStructBegin("RetryExperimentResponse"); err != nil { + goto WriteStructBeginError + } + if p != nil { + if err = p.writeField1(oprot); err != nil { + fieldId = 1 + goto WriteFieldError + } + if err = p.writeField2(oprot); err != nil { + fieldId = 2 + goto WriteFieldError + } + if err = p.writeField3(oprot); err != nil { + fieldId = 3 + goto WriteFieldError + } + if err = p.writeField4(oprot); err != nil { + fieldId = 4 + goto WriteFieldError + } + if err = p.writeField5(oprot); err != nil { + fieldId = 5 + goto WriteFieldError + } + if err = p.writeField255(oprot); err != nil { + fieldId = 255 + goto WriteFieldError + } + } + if err = oprot.WriteFieldStop(); err != nil { + goto WriteFieldStopError + } + if err = oprot.WriteStructEnd(); err != nil { + goto WriteStructEndError + } + return nil +WriteStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) +WriteFieldError: + return thrift.PrependError(fmt.Sprintf("%T write field %d error: ", p, fieldId), err) +WriteFieldStopError: + return thrift.PrependError(fmt.Sprintf("%T write field stop error: ", p), err) +WriteStructEndError: + return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) +} + +func (p *RetryExperimentResponse) writeField1(oprot thrift.TProtocol) (err error) { + if p.IsSetRunID() { + if err = oprot.WriteFieldBegin("run_id", thrift.I64, 1); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.RunID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) +} +func (p *RetryExperimentResponse) writeField2(oprot thrift.TProtocol) (err error) { + if p.IsSetRequestID() { + if err = oprot.WriteFieldBegin("request_id", thrift.STRING, 2); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteString(*p.RequestID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 2 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 2 end error: ", p), err) +} +func (p *RetryExperimentResponse) writeField3(oprot thrift.TProtocol) (err error) { + if p.IsSetAcceptedCount() { + if err = oprot.WriteFieldBegin("accepted_count", thrift.I32, 3); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.AcceptedCount); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 3 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 3 end error: ", p), err) +} +func (p *RetryExperimentResponse) writeField4(oprot thrift.TProtocol) (err error) { + if p.IsSetRejectedItems() { + if err = oprot.WriteFieldBegin("rejected_items", thrift.LIST, 4); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteListBegin(thrift.STRUCT, len(p.RejectedItems)); err != nil { + return err + } + for _, v := range p.RejectedItems { + if err := v.Write(oprot); err != nil { + return err + } + } + if err := oprot.WriteListEnd(); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 4 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 4 end error: ", p), err) +} +func (p *RetryExperimentResponse) writeField5(oprot thrift.TProtocol) (err error) { + if p.IsSetSnapshotAtMs() { + if err = oprot.WriteFieldBegin("snapshot_at_ms", thrift.I64, 5); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.SnapshotAtMs); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) +} +func (p *RetryExperimentResponse) writeField255(oprot thrift.TProtocol) (err error) { + if err = oprot.WriteFieldBegin("BaseResp", thrift.STRUCT, 255); err != nil { + goto WriteFieldBeginError + } + if err := p.BaseResp.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 255 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 255 end error: ", p), err) +} + +func (p *RetryExperimentResponse) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("RetryExperimentResponse(%+v)", *p) + +} + +func (p *RetryExperimentResponse) DeepEqual(ano *RetryExperimentResponse) bool { + if p == ano { + return true + } else if p == nil || ano == nil { + return false + } + if !p.Field1DeepEqual(ano.RunID) { + return false + } + if !p.Field2DeepEqual(ano.RequestID) { + return false + } + if !p.Field3DeepEqual(ano.AcceptedCount) { + return false + } + if !p.Field4DeepEqual(ano.RejectedItems) { + return false + } + if !p.Field5DeepEqual(ano.SnapshotAtMs) { + return false + } + if !p.Field255DeepEqual(ano.BaseResp) { + return false + } + return true +} + +func (p *RetryExperimentResponse) Field1DeepEqual(src *int64) bool { + + if p.RunID == src { + return true + } else if p.RunID == nil || src == nil { + return false + } + if *p.RunID != *src { + return false + } + return true +} +func (p *RetryExperimentResponse) Field2DeepEqual(src *string) bool { + + if p.RequestID == src { + return true + } else if p.RequestID == nil || src == nil { + return false + } + if strings.Compare(*p.RequestID, *src) != 0 { + return false + } + return true +} +func (p *RetryExperimentResponse) Field3DeepEqual(src *int32) bool { + + if p.AcceptedCount == src { + return true + } else if p.AcceptedCount == nil || src == nil { + return false + } + if *p.AcceptedCount != *src { + return false + } + return true +} +func (p *RetryExperimentResponse) Field4DeepEqual(src []*RetryRejectedItem) bool { + + if len(p.RejectedItems) != len(src) { + return false + } + for i, v := range p.RejectedItems { + _src := src[i] + if !v.DeepEqual(_src) { + return false + } + } + return true +} +func (p *RetryExperimentResponse) Field5DeepEqual(src *int64) bool { + + if p.SnapshotAtMs == src { + return true + } else if p.SnapshotAtMs == nil || src == nil { + return false + } + if *p.SnapshotAtMs != *src { return false } return true diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt_validator.go b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt_validator.go index 695eab362a..3ab65eb5cc 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt_validator.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt_validator.go @@ -329,6 +329,9 @@ func (p *RetryExperimentRequest) IsValid() error { } return nil } +func (p *RetryRejectedItem) IsValid() error { + return nil +} func (p *RetryExperimentResponse) IsValid() error { if p.BaseResp != nil { if err := p.BaseResp.IsValid(); err != nil { diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go index 2bf8083c56..765a96a80c 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go @@ -10064,6 +10064,34 @@ func (p *RetryExperimentRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 5: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField5(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 6: + if fieldTypeId == thrift.STRING { + l, err = p.FastReadField6(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 100: if fieldTypeId == thrift.MAP { l, err = p.FastReadField100(buf[offset:]) @@ -10178,6 +10206,34 @@ func (p *RetryExperimentRequest) FastReadField4(buf []byte) (int, error) { return offset, nil } +func (p *RetryExperimentRequest) FastReadField5(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ExpectedRunID = _field + return offset, nil +} + +func (p *RetryExperimentRequest) FastReadField6(buf []byte) (int, error) { + offset := 0 + + var _field *string + if v, l, err := thrift.Binary.ReadString(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.RequestID = _field + return offset, nil +} + func (p *RetryExperimentRequest) FastReadField100(buf []byte) (int, error) { offset := 0 @@ -10231,8 +10287,10 @@ func (p *RetryExperimentRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWrit if p != nil { offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) + offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) + offset += p.fastWriteField6(buf[offset:], w) offset += p.fastWriteField100(buf[offset:], w) offset += p.fastWriteField255(buf[offset:], w) } @@ -10247,6 +10305,8 @@ func (p *RetryExperimentRequest) BLength() int { l += p.field2Length() l += p.field3Length() l += p.field4Length() + l += p.field5Length() + l += p.field6Length() l += p.field100Length() l += p.field255Length() } @@ -10297,6 +10357,24 @@ func (p *RetryExperimentRequest) fastWriteField4(buf []byte, w thrift.NocopyWrit return offset } +func (p *RetryExperimentRequest) fastWriteField5(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetExpectedRunID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 5) + offset += thrift.Binary.WriteI64(buf[offset:], *p.ExpectedRunID) + } + return offset +} + +func (p *RetryExperimentRequest) fastWriteField6(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetRequestID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRING, 6) + offset += thrift.Binary.WriteStringNocopy(buf[offset:], w, *p.RequestID) + } + return offset +} + func (p *RetryExperimentRequest) fastWriteField100(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetExt() { @@ -10361,6 +10439,24 @@ func (p *RetryExperimentRequest) field4Length() int { return l } +func (p *RetryExperimentRequest) field5Length() int { + l := 0 + if p.IsSetExpectedRunID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + +func (p *RetryExperimentRequest) field6Length() int { + l := 0 + if p.IsSetRequestID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.StringLengthNocopy(*p.RequestID) + } + return l +} + func (p *RetryExperimentRequest) field100Length() int { l := 0 if p.IsSetExt() { @@ -10415,6 +10511,19 @@ func (p *RetryExperimentRequest) DeepCopy(s interface{}) error { } } + if src.ExpectedRunID != nil { + tmp := *src.ExpectedRunID + p.ExpectedRunID = &tmp + } + + if src.RequestID != nil { + var tmp string + if *src.RequestID != "" { + tmp = kutils.StringDeepCopy(*src.RequestID) + } + p.RequestID = &tmp + } + if src.Ext != nil { p.Ext = make(map[string]string, len(src.Ext)) for key, val := range src.Ext { @@ -10444,6 +10553,179 @@ func (p *RetryExperimentRequest) DeepCopy(s interface{}) error { return nil } +func (p *RetryRejectedItem) FastRead(buf []byte) (int, error) { + + var err error + var offset int + var l int + var fieldTypeId thrift.TType + var fieldId int16 + for { + fieldTypeId, fieldId, l, err = thrift.Binary.ReadFieldBegin(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField1(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 2: + if fieldTypeId == thrift.STRING { + l, err = p.FastReadField2(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + default: + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + } + + return offset, nil +ReadFieldBeginError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_RetryRejectedItem[fieldId]), err) +SkipFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) +} + +func (p *RetryRejectedItem) FastReadField1(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemID = _field + return offset, nil +} + +func (p *RetryRejectedItem) FastReadField2(buf []byte) (int, error) { + offset := 0 + + var _field *string + if v, l, err := thrift.Binary.ReadString(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.Reason = _field + return offset, nil +} + +func (p *RetryRejectedItem) FastWrite(buf []byte) int { + return p.FastWriteNocopy(buf, nil) +} + +func (p *RetryRejectedItem) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p != nil { + offset += p.fastWriteField1(buf[offset:], w) + offset += p.fastWriteField2(buf[offset:], w) + } + offset += thrift.Binary.WriteFieldStop(buf[offset:]) + return offset +} + +func (p *RetryRejectedItem) BLength() int { + l := 0 + if p != nil { + l += p.field1Length() + l += p.field2Length() + } + l += thrift.Binary.FieldStopLength() + return l +} + +func (p *RetryRejectedItem) fastWriteField1(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 1) + offset += thrift.Binary.WriteI64(buf[offset:], *p.ItemID) + } + return offset +} + +func (p *RetryRejectedItem) fastWriteField2(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetReason() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRING, 2) + offset += thrift.Binary.WriteStringNocopy(buf[offset:], w, *p.Reason) + } + return offset +} + +func (p *RetryRejectedItem) field1Length() int { + l := 0 + if p.IsSetItemID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + +func (p *RetryRejectedItem) field2Length() int { + l := 0 + if p.IsSetReason() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.StringLengthNocopy(*p.Reason) + } + return l +} + +func (p *RetryRejectedItem) DeepCopy(s interface{}) error { + src, ok := s.(*RetryRejectedItem) + if !ok { + return fmt.Errorf("%T's type not matched %T", s, p) + } + + if src.ItemID != nil { + tmp := *src.ItemID + p.ItemID = &tmp + } + + if src.Reason != nil { + var tmp string + if *src.Reason != "" { + tmp = kutils.StringDeepCopy(*src.Reason) + } + p.Reason = &tmp + } + + return nil +} + func (p *RetryExperimentResponse) FastRead(buf []byte) (int, error) { var err error @@ -10475,6 +10757,62 @@ func (p *RetryExperimentResponse) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 2: + if fieldTypeId == thrift.STRING { + l, err = p.FastReadField2(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 3: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField3(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 4: + if fieldTypeId == thrift.LIST { + l, err = p.FastReadField4(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 5: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField5(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 255: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField255(buf[offset:]) @@ -10521,6 +10859,73 @@ func (p *RetryExperimentResponse) FastReadField1(buf []byte) (int, error) { return offset, nil } +func (p *RetryExperimentResponse) FastReadField2(buf []byte) (int, error) { + offset := 0 + + var _field *string + if v, l, err := thrift.Binary.ReadString(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.RequestID = _field + return offset, nil +} + +func (p *RetryExperimentResponse) FastReadField3(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.AcceptedCount = _field + return offset, nil +} + +func (p *RetryExperimentResponse) FastReadField4(buf []byte) (int, error) { + offset := 0 + + _, size, l, err := thrift.Binary.ReadListBegin(buf[offset:]) + offset += l + if err != nil { + return offset, err + } + _field := make([]*RetryRejectedItem, 0, size) + values := make([]RetryRejectedItem, size) + for i := 0; i < size; i++ { + _elem := &values[i] + _elem.InitDefault() + if l, err := _elem.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + + _field = append(_field, _elem) + } + p.RejectedItems = _field + return offset, nil +} + +func (p *RetryExperimentResponse) FastReadField5(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.SnapshotAtMs = _field + return offset, nil +} + func (p *RetryExperimentResponse) FastReadField255(buf []byte) (int, error) { offset := 0 _field := base.NewBaseResp() @@ -10541,6 +10946,10 @@ func (p *RetryExperimentResponse) FastWriteNocopy(buf []byte, w thrift.NocopyWri offset := 0 if p != nil { offset += p.fastWriteField1(buf[offset:], w) + offset += p.fastWriteField3(buf[offset:], w) + offset += p.fastWriteField5(buf[offset:], w) + offset += p.fastWriteField2(buf[offset:], w) + offset += p.fastWriteField4(buf[offset:], w) offset += p.fastWriteField255(buf[offset:], w) } offset += thrift.Binary.WriteFieldStop(buf[offset:]) @@ -10551,6 +10960,10 @@ func (p *RetryExperimentResponse) BLength() int { l := 0 if p != nil { l += p.field1Length() + l += p.field2Length() + l += p.field3Length() + l += p.field4Length() + l += p.field5Length() l += p.field255Length() } l += thrift.Binary.FieldStopLength() @@ -10566,6 +10979,49 @@ func (p *RetryExperimentResponse) fastWriteField1(buf []byte, w thrift.NocopyWri return offset } +func (p *RetryExperimentResponse) fastWriteField2(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetRequestID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRING, 2) + offset += thrift.Binary.WriteStringNocopy(buf[offset:], w, *p.RequestID) + } + return offset +} + +func (p *RetryExperimentResponse) fastWriteField3(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetAcceptedCount() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 3) + offset += thrift.Binary.WriteI32(buf[offset:], *p.AcceptedCount) + } + return offset +} + +func (p *RetryExperimentResponse) fastWriteField4(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetRejectedItems() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.LIST, 4) + listBeginOffset := offset + offset += thrift.Binary.ListBeginLength() + var length int + for _, v := range p.RejectedItems { + length++ + offset += v.FastWriteNocopy(buf[offset:], w) + } + thrift.Binary.WriteListBegin(buf[listBeginOffset:], thrift.STRUCT, length) + } + return offset +} + +func (p *RetryExperimentResponse) fastWriteField5(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetSnapshotAtMs() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 5) + offset += thrift.Binary.WriteI64(buf[offset:], *p.SnapshotAtMs) + } + return offset +} + func (p *RetryExperimentResponse) fastWriteField255(buf []byte, w thrift.NocopyWriter) int { offset := 0 offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 255) @@ -10582,6 +11038,46 @@ func (p *RetryExperimentResponse) field1Length() int { return l } +func (p *RetryExperimentResponse) field2Length() int { + l := 0 + if p.IsSetRequestID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.StringLengthNocopy(*p.RequestID) + } + return l +} + +func (p *RetryExperimentResponse) field3Length() int { + l := 0 + if p.IsSetAcceptedCount() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + +func (p *RetryExperimentResponse) field4Length() int { + l := 0 + if p.IsSetRejectedItems() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.ListBeginLength() + for _, v := range p.RejectedItems { + _ = v + l += v.BLength() + } + } + return l +} + +func (p *RetryExperimentResponse) field5Length() int { + l := 0 + if p.IsSetSnapshotAtMs() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + func (p *RetryExperimentResponse) field255Length() int { l := 0 l += thrift.Binary.FieldBeginLength() @@ -10600,6 +11096,39 @@ func (p *RetryExperimentResponse) DeepCopy(s interface{}) error { p.RunID = &tmp } + if src.RequestID != nil { + var tmp string + if *src.RequestID != "" { + tmp = kutils.StringDeepCopy(*src.RequestID) + } + p.RequestID = &tmp + } + + if src.AcceptedCount != nil { + tmp := *src.AcceptedCount + p.AcceptedCount = &tmp + } + + if src.RejectedItems != nil { + p.RejectedItems = make([]*RetryRejectedItem, 0, len(src.RejectedItems)) + for _, elem := range src.RejectedItems { + var _elem *RetryRejectedItem + if elem != nil { + _elem = &RetryRejectedItem{} + if err := _elem.DeepCopy(elem); err != nil { + return err + } + } + + p.RejectedItems = append(p.RejectedItems, _elem) + } + } + + if src.SnapshotAtMs != nil { + tmp := *src.SnapshotAtMs + p.SnapshotAtMs = &tmp + } + var _baseResp *base.BaseResp if src.BaseResp != nil { _baseResp = &base.BaseResp{} diff --git a/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go b/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go index 8fa3e32d1c..b6c0171b2e 100644 --- a/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go +++ b/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go @@ -40661,12 +40661,14 @@ func (p *GetExperimentAggrResultOpenAPIData) Field2DeepEqual(src *experiment.Eva } type RetryExperimentOApiRequest struct { - WorkspaceID *int64 `thrift:"workspace_id,1,optional" frugal:"1,optional,i64" json:"workspace_id" form:"workspace_id" ` - ExperimentID *int64 `thrift:"experiment_id,2,optional" frugal:"2,optional,i64" json:"experiment_id" path:"experiment_id" ` - RetryMode *experiment.ExptRetryMode `thrift:"retry_mode,3,optional" frugal:"3,optional,string" form:"retry_mode" json:"retry_mode,omitempty"` - ItemIds []int64 `thrift:"item_ids,4,optional" frugal:"4,optional,list" json:"item_ids" form:"item_ids" ` - Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` - Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` + WorkspaceID *int64 `thrift:"workspace_id,1,optional" frugal:"1,optional,i64" json:"workspace_id" form:"workspace_id" ` + ExperimentID *int64 `thrift:"experiment_id,2,optional" frugal:"2,optional,i64" json:"experiment_id" path:"experiment_id" ` + RetryMode *experiment.ExptRetryMode `thrift:"retry_mode,3,optional" frugal:"3,optional,string" form:"retry_mode" json:"retry_mode,omitempty"` + ItemIds []int64 `thrift:"item_ids,4,optional" frugal:"4,optional,list" json:"item_ids" form:"item_ids" ` + ExpectedRunID *int64 `thrift:"expected_run_id,5,optional" frugal:"5,optional,i64" json:"expected_run_id,omitempty" form:"expected_run_id" ` + RequestID *string `thrift:"request_id,6,optional" frugal:"6,optional,string" form:"request_id" json:"request_id,omitempty"` + Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` + Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } func NewRetryExperimentOApiRequest() *RetryExperimentOApiRequest { @@ -40724,6 +40726,30 @@ func (p *RetryExperimentOApiRequest) GetItemIds() (v []int64) { return p.ItemIds } +var RetryExperimentOApiRequest_ExpectedRunID_DEFAULT int64 + +func (p *RetryExperimentOApiRequest) GetExpectedRunID() (v int64) { + if p == nil { + return + } + if !p.IsSetExpectedRunID() { + return RetryExperimentOApiRequest_ExpectedRunID_DEFAULT + } + return *p.ExpectedRunID +} + +var RetryExperimentOApiRequest_RequestID_DEFAULT string + +func (p *RetryExperimentOApiRequest) GetRequestID() (v string) { + if p == nil { + return + } + if !p.IsSetRequestID() { + return RetryExperimentOApiRequest_RequestID_DEFAULT + } + return *p.RequestID +} + var RetryExperimentOApiRequest_Ext_DEFAULT map[string]string func (p *RetryExperimentOApiRequest) GetExt() (v map[string]string) { @@ -40759,6 +40785,12 @@ func (p *RetryExperimentOApiRequest) SetRetryMode(val *experiment.ExptRetryMode) func (p *RetryExperimentOApiRequest) SetItemIds(val []int64) { p.ItemIds = val } +func (p *RetryExperimentOApiRequest) SetExpectedRunID(val *int64) { + p.ExpectedRunID = val +} +func (p *RetryExperimentOApiRequest) SetRequestID(val *string) { + p.RequestID = val +} func (p *RetryExperimentOApiRequest) SetExt(val map[string]string) { p.Ext = val } @@ -40771,6 +40803,8 @@ var fieldIDToName_RetryExperimentOApiRequest = map[int16]string{ 2: "experiment_id", 3: "retry_mode", 4: "item_ids", + 5: "expected_run_id", + 6: "request_id", 100: "ext", 255: "Base", } @@ -40791,6 +40825,14 @@ func (p *RetryExperimentOApiRequest) IsSetItemIds() bool { return p.ItemIds != nil } +func (p *RetryExperimentOApiRequest) IsSetExpectedRunID() bool { + return p.ExpectedRunID != nil +} + +func (p *RetryExperimentOApiRequest) IsSetRequestID() bool { + return p.RequestID != nil +} + func (p *RetryExperimentOApiRequest) IsSetExt() bool { return p.Ext != nil } @@ -40849,6 +40891,22 @@ func (p *RetryExperimentOApiRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 5: + if fieldTypeId == thrift.I64 { + if err = p.ReadField5(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 6: + if fieldTypeId == thrift.STRING { + if err = p.ReadField6(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 100: if fieldTypeId == thrift.MAP { if err = p.ReadField100(iprot); err != nil { @@ -40950,6 +41008,28 @@ func (p *RetryExperimentOApiRequest) ReadField4(iprot thrift.TProtocol) error { p.ItemIds = _field return nil } +func (p *RetryExperimentOApiRequest) ReadField5(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.ExpectedRunID = _field + return nil +} +func (p *RetryExperimentOApiRequest) ReadField6(iprot thrift.TProtocol) error { + + var _field *string + if v, err := iprot.ReadString(); err != nil { + return err + } else { + _field = &v + } + p.RequestID = _field + return nil +} func (p *RetryExperimentOApiRequest) ReadField100(iprot thrift.TProtocol) error { _, _, size, err := iprot.ReadMapBegin() if err != nil { @@ -41010,6 +41090,14 @@ func (p *RetryExperimentOApiRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 4 goto WriteFieldError } + if err = p.writeField5(oprot); err != nil { + fieldId = 5 + goto WriteFieldError + } + if err = p.writeField6(oprot); err != nil { + fieldId = 6 + goto WriteFieldError + } if err = p.writeField100(oprot); err != nil { fieldId = 100 goto WriteFieldError @@ -41116,6 +41204,42 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 4 end error: ", p), err) } +func (p *RetryExperimentOApiRequest) writeField5(oprot thrift.TProtocol) (err error) { + if p.IsSetExpectedRunID() { + if err = oprot.WriteFieldBegin("expected_run_id", thrift.I64, 5); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.ExpectedRunID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) +} +func (p *RetryExperimentOApiRequest) writeField6(oprot thrift.TProtocol) (err error) { + if p.IsSetRequestID() { + if err = oprot.WriteFieldBegin("request_id", thrift.STRING, 6); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteString(*p.RequestID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 end error: ", p), err) +} func (p *RetryExperimentOApiRequest) writeField100(oprot thrift.TProtocol) (err error) { if p.IsSetExt() { if err = oprot.WriteFieldBegin("ext", thrift.MAP, 100); err != nil { @@ -41190,6 +41314,12 @@ func (p *RetryExperimentOApiRequest) DeepEqual(ano *RetryExperimentOApiRequest) if !p.Field4DeepEqual(ano.ItemIds) { return false } + if !p.Field5DeepEqual(ano.ExpectedRunID) { + return false + } + if !p.Field6DeepEqual(ano.RequestID) { + return false + } if !p.Field100DeepEqual(ano.Ext) { return false } @@ -41248,6 +41378,30 @@ func (p *RetryExperimentOApiRequest) Field4DeepEqual(src []int64) bool { } return true } +func (p *RetryExperimentOApiRequest) Field5DeepEqual(src *int64) bool { + + if p.ExpectedRunID == src { + return true + } else if p.ExpectedRunID == nil || src == nil { + return false + } + if *p.ExpectedRunID != *src { + return false + } + return true +} +func (p *RetryExperimentOApiRequest) Field6DeepEqual(src *string) bool { + + if p.RequestID == src { + return true + } else if p.RequestID == nil || src == nil { + return false + } + if strings.Compare(*p.RequestID, *src) != 0 { + return false + } + return true +} func (p *RetryExperimentOApiRequest) Field100DeepEqual(src map[string]string) bool { if len(p.Ext) != len(src) { @@ -41664,7 +41818,12 @@ func (p *RetryExperimentOApiResponse) Field255DeepEqual(src *base.BaseResp) bool } type RetryExperimentOpenAPIData struct { - RunID *int64 `thrift:"run_id,3,optional" frugal:"3,optional,i64" form:"run_id" json:"run_id,omitempty" query:"run_id"` + RunID *int64 `thrift:"run_id,3,optional" frugal:"3,optional,i64" json:"run_id,omitempty" form:"run_id" query:"run_id"` + RequestID *string `thrift:"request_id,4,optional" frugal:"4,optional,string" form:"request_id" json:"request_id,omitempty" query:"request_id"` + AcceptedCount *int32 `thrift:"accepted_count,5,optional" frugal:"5,optional,i32" form:"accepted_count" json:"accepted_count,omitempty" query:"accepted_count"` + // Keep an explicit empty list in a current-run receipt. + RejectedItems []*experiment.RetryRejectedItem `thrift:"rejected_items,6,optional" frugal:"6,optional,list" json:"rejected_items" form:"rejected_items" query:"rejected_items"` + SnapshotAtMs *int64 `thrift:"snapshot_at_ms,7,optional" frugal:"7,optional,i64" json:"snapshot_at_ms,omitempty" form:"snapshot_at_ms" query:"snapshot_at_ms"` } func NewRetryExperimentOpenAPIData() *RetryExperimentOpenAPIData { @@ -41685,18 +41844,98 @@ func (p *RetryExperimentOpenAPIData) GetRunID() (v int64) { } return *p.RunID } + +var RetryExperimentOpenAPIData_RequestID_DEFAULT string + +func (p *RetryExperimentOpenAPIData) GetRequestID() (v string) { + if p == nil { + return + } + if !p.IsSetRequestID() { + return RetryExperimentOpenAPIData_RequestID_DEFAULT + } + return *p.RequestID +} + +var RetryExperimentOpenAPIData_AcceptedCount_DEFAULT int32 + +func (p *RetryExperimentOpenAPIData) GetAcceptedCount() (v int32) { + if p == nil { + return + } + if !p.IsSetAcceptedCount() { + return RetryExperimentOpenAPIData_AcceptedCount_DEFAULT + } + return *p.AcceptedCount +} + +var RetryExperimentOpenAPIData_RejectedItems_DEFAULT []*experiment.RetryRejectedItem + +func (p *RetryExperimentOpenAPIData) GetRejectedItems() (v []*experiment.RetryRejectedItem) { + if p == nil { + return + } + if !p.IsSetRejectedItems() { + return RetryExperimentOpenAPIData_RejectedItems_DEFAULT + } + return p.RejectedItems +} + +var RetryExperimentOpenAPIData_SnapshotAtMs_DEFAULT int64 + +func (p *RetryExperimentOpenAPIData) GetSnapshotAtMs() (v int64) { + if p == nil { + return + } + if !p.IsSetSnapshotAtMs() { + return RetryExperimentOpenAPIData_SnapshotAtMs_DEFAULT + } + return *p.SnapshotAtMs +} func (p *RetryExperimentOpenAPIData) SetRunID(val *int64) { p.RunID = val } +func (p *RetryExperimentOpenAPIData) SetRequestID(val *string) { + p.RequestID = val +} +func (p *RetryExperimentOpenAPIData) SetAcceptedCount(val *int32) { + p.AcceptedCount = val +} +func (p *RetryExperimentOpenAPIData) SetRejectedItems(val []*experiment.RetryRejectedItem) { + p.RejectedItems = val +} +func (p *RetryExperimentOpenAPIData) SetSnapshotAtMs(val *int64) { + p.SnapshotAtMs = val +} var fieldIDToName_RetryExperimentOpenAPIData = map[int16]string{ 3: "run_id", + 4: "request_id", + 5: "accepted_count", + 6: "rejected_items", + 7: "snapshot_at_ms", } func (p *RetryExperimentOpenAPIData) IsSetRunID() bool { return p.RunID != nil } +func (p *RetryExperimentOpenAPIData) IsSetRequestID() bool { + return p.RequestID != nil +} + +func (p *RetryExperimentOpenAPIData) IsSetAcceptedCount() bool { + return p.AcceptedCount != nil +} + +func (p *RetryExperimentOpenAPIData) IsSetRejectedItems() bool { + return p.RejectedItems != nil +} + +func (p *RetryExperimentOpenAPIData) IsSetSnapshotAtMs() bool { + return p.SnapshotAtMs != nil +} + func (p *RetryExperimentOpenAPIData) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -41723,6 +41962,38 @@ func (p *RetryExperimentOpenAPIData) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 4: + if fieldTypeId == thrift.STRING { + if err = p.ReadField4(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 5: + if fieldTypeId == thrift.I32 { + if err = p.ReadField5(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 6: + if fieldTypeId == thrift.LIST { + if err = p.ReadField6(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 7: + if fieldTypeId == thrift.I64 { + if err = p.ReadField7(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } default: if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError @@ -41763,6 +42034,62 @@ func (p *RetryExperimentOpenAPIData) ReadField3(iprot thrift.TProtocol) error { p.RunID = _field return nil } +func (p *RetryExperimentOpenAPIData) ReadField4(iprot thrift.TProtocol) error { + + var _field *string + if v, err := iprot.ReadString(); err != nil { + return err + } else { + _field = &v + } + p.RequestID = _field + return nil +} +func (p *RetryExperimentOpenAPIData) ReadField5(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.AcceptedCount = _field + return nil +} +func (p *RetryExperimentOpenAPIData) ReadField6(iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin() + if err != nil { + return err + } + _field := make([]*experiment.RetryRejectedItem, 0, size) + values := make([]experiment.RetryRejectedItem, size) + for i := 0; i < size; i++ { + _elem := &values[i] + _elem.InitDefault() + + if err := _elem.Read(iprot); err != nil { + return err + } + + _field = append(_field, _elem) + } + if err := iprot.ReadListEnd(); err != nil { + return err + } + p.RejectedItems = _field + return nil +} +func (p *RetryExperimentOpenAPIData) ReadField7(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.SnapshotAtMs = _field + return nil +} func (p *RetryExperimentOpenAPIData) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 @@ -41774,6 +42101,22 @@ func (p *RetryExperimentOpenAPIData) Write(oprot thrift.TProtocol) (err error) { fieldId = 3 goto WriteFieldError } + if err = p.writeField4(oprot); err != nil { + fieldId = 4 + goto WriteFieldError + } + if err = p.writeField5(oprot); err != nil { + fieldId = 5 + goto WriteFieldError + } + if err = p.writeField6(oprot); err != nil { + fieldId = 6 + goto WriteFieldError + } + if err = p.writeField7(oprot); err != nil { + fieldId = 7 + goto WriteFieldError + } } if err = oprot.WriteFieldStop(); err != nil { goto WriteFieldStopError @@ -41810,6 +42153,86 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 3 end error: ", p), err) } +func (p *RetryExperimentOpenAPIData) writeField4(oprot thrift.TProtocol) (err error) { + if p.IsSetRequestID() { + if err = oprot.WriteFieldBegin("request_id", thrift.STRING, 4); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteString(*p.RequestID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 4 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 4 end error: ", p), err) +} +func (p *RetryExperimentOpenAPIData) writeField5(oprot thrift.TProtocol) (err error) { + if p.IsSetAcceptedCount() { + if err = oprot.WriteFieldBegin("accepted_count", thrift.I32, 5); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.AcceptedCount); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) +} +func (p *RetryExperimentOpenAPIData) writeField6(oprot thrift.TProtocol) (err error) { + if p.IsSetRejectedItems() { + if err = oprot.WriteFieldBegin("rejected_items", thrift.LIST, 6); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteListBegin(thrift.STRUCT, len(p.RejectedItems)); err != nil { + return err + } + for _, v := range p.RejectedItems { + if err := v.Write(oprot); err != nil { + return err + } + } + if err := oprot.WriteListEnd(); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 end error: ", p), err) +} +func (p *RetryExperimentOpenAPIData) writeField7(oprot thrift.TProtocol) (err error) { + if p.IsSetSnapshotAtMs() { + if err = oprot.WriteFieldBegin("snapshot_at_ms", thrift.I64, 7); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.SnapshotAtMs); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 7 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 7 end error: ", p), err) +} func (p *RetryExperimentOpenAPIData) String() string { if p == nil { @@ -41828,6 +42251,18 @@ func (p *RetryExperimentOpenAPIData) DeepEqual(ano *RetryExperimentOpenAPIData) if !p.Field3DeepEqual(ano.RunID) { return false } + if !p.Field4DeepEqual(ano.RequestID) { + return false + } + if !p.Field5DeepEqual(ano.AcceptedCount) { + return false + } + if !p.Field6DeepEqual(ano.RejectedItems) { + return false + } + if !p.Field7DeepEqual(ano.SnapshotAtMs) { + return false + } return true } @@ -41843,6 +42278,55 @@ func (p *RetryExperimentOpenAPIData) Field3DeepEqual(src *int64) bool { } return true } +func (p *RetryExperimentOpenAPIData) Field4DeepEqual(src *string) bool { + + if p.RequestID == src { + return true + } else if p.RequestID == nil || src == nil { + return false + } + if strings.Compare(*p.RequestID, *src) != 0 { + return false + } + return true +} +func (p *RetryExperimentOpenAPIData) Field5DeepEqual(src *int32) bool { + + if p.AcceptedCount == src { + return true + } else if p.AcceptedCount == nil || src == nil { + return false + } + if *p.AcceptedCount != *src { + return false + } + return true +} +func (p *RetryExperimentOpenAPIData) Field6DeepEqual(src []*experiment.RetryRejectedItem) bool { + + if len(p.RejectedItems) != len(src) { + return false + } + for i, v := range p.RejectedItems { + _src := src[i] + if !v.DeepEqual(_src) { + return false + } + } + return true +} +func (p *RetryExperimentOpenAPIData) Field7DeepEqual(src *int64) bool { + + if p.SnapshotAtMs == src { + return true + } else if p.SnapshotAtMs == nil || src == nil { + return false + } + if *p.SnapshotAtMs != *src { + return false + } + return true +} // KillExperimentOApiRequest 通过 OpenAPI 终止一个进行中的实验。 // 仅对处于 Processing 状态的实验生效，成功后实验会被置为 Terminated。 diff --git a/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go b/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go index 0e5f64cd35..79f0195922 100644 --- a/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go +++ b/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go @@ -29270,6 +29270,34 @@ func (p *RetryExperimentOApiRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 5: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField5(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 6: + if fieldTypeId == thrift.STRING { + l, err = p.FastReadField6(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 100: if fieldTypeId == thrift.MAP { l, err = p.FastReadField100(buf[offset:]) @@ -29382,6 +29410,34 @@ func (p *RetryExperimentOApiRequest) FastReadField4(buf []byte) (int, error) { return offset, nil } +func (p *RetryExperimentOApiRequest) FastReadField5(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ExpectedRunID = _field + return offset, nil +} + +func (p *RetryExperimentOApiRequest) FastReadField6(buf []byte) (int, error) { + offset := 0 + + var _field *string + if v, l, err := thrift.Binary.ReadString(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.RequestID = _field + return offset, nil +} + func (p *RetryExperimentOApiRequest) FastReadField100(buf []byte) (int, error) { offset := 0 @@ -29435,8 +29491,10 @@ func (p *RetryExperimentOApiRequest) FastWriteNocopy(buf []byte, w thrift.Nocopy if p != nil { offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) + offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) + offset += p.fastWriteField6(buf[offset:], w) offset += p.fastWriteField100(buf[offset:], w) offset += p.fastWriteField255(buf[offset:], w) } @@ -29451,6 +29509,8 @@ func (p *RetryExperimentOApiRequest) BLength() int { l += p.field2Length() l += p.field3Length() l += p.field4Length() + l += p.field5Length() + l += p.field6Length() l += p.field100Length() l += p.field255Length() } @@ -29501,6 +29561,24 @@ func (p *RetryExperimentOApiRequest) fastWriteField4(buf []byte, w thrift.Nocopy return offset } +func (p *RetryExperimentOApiRequest) fastWriteField5(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetExpectedRunID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 5) + offset += thrift.Binary.WriteI64(buf[offset:], *p.ExpectedRunID) + } + return offset +} + +func (p *RetryExperimentOApiRequest) fastWriteField6(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetRequestID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRING, 6) + offset += thrift.Binary.WriteStringNocopy(buf[offset:], w, *p.RequestID) + } + return offset +} + func (p *RetryExperimentOApiRequest) fastWriteField100(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetExt() { @@ -29565,6 +29643,24 @@ func (p *RetryExperimentOApiRequest) field4Length() int { return l } +func (p *RetryExperimentOApiRequest) field5Length() int { + l := 0 + if p.IsSetExpectedRunID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + +func (p *RetryExperimentOApiRequest) field6Length() int { + l := 0 + if p.IsSetRequestID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.StringLengthNocopy(*p.RequestID) + } + return l +} + func (p *RetryExperimentOApiRequest) field100Length() int { l := 0 if p.IsSetExt() { @@ -29619,6 +29715,19 @@ func (p *RetryExperimentOApiRequest) DeepCopy(s interface{}) error { } } + if src.ExpectedRunID != nil { + tmp := *src.ExpectedRunID + p.ExpectedRunID = &tmp + } + + if src.RequestID != nil { + var tmp string + if *src.RequestID != "" { + tmp = kutils.StringDeepCopy(*src.RequestID) + } + p.RequestID = &tmp + } + if src.Ext != nil { p.Ext = make(map[string]string, len(src.Ext)) for key, val := range src.Ext { @@ -29958,6 +30067,62 @@ func (p *RetryExperimentOpenAPIData) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 4: + if fieldTypeId == thrift.STRING { + l, err = p.FastReadField4(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 5: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField5(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 6: + if fieldTypeId == thrift.LIST { + l, err = p.FastReadField6(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 7: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField7(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } default: l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) offset += l @@ -29990,6 +30155,73 @@ func (p *RetryExperimentOpenAPIData) FastReadField3(buf []byte) (int, error) { return offset, nil } +func (p *RetryExperimentOpenAPIData) FastReadField4(buf []byte) (int, error) { + offset := 0 + + var _field *string + if v, l, err := thrift.Binary.ReadString(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.RequestID = _field + return offset, nil +} + +func (p *RetryExperimentOpenAPIData) FastReadField5(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.AcceptedCount = _field + return offset, nil +} + +func (p *RetryExperimentOpenAPIData) FastReadField6(buf []byte) (int, error) { + offset := 0 + + _, size, l, err := thrift.Binary.ReadListBegin(buf[offset:]) + offset += l + if err != nil { + return offset, err + } + _field := make([]*experiment.RetryRejectedItem, 0, size) + values := make([]experiment.RetryRejectedItem, size) + for i := 0; i < size; i++ { + _elem := &values[i] + _elem.InitDefault() + if l, err := _elem.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + + _field = append(_field, _elem) + } + p.RejectedItems = _field + return offset, nil +} + +func (p *RetryExperimentOpenAPIData) FastReadField7(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.SnapshotAtMs = _field + return offset, nil +} + func (p *RetryExperimentOpenAPIData) FastWrite(buf []byte) int { return p.FastWriteNocopy(buf, nil) } @@ -29998,6 +30230,10 @@ func (p *RetryExperimentOpenAPIData) FastWriteNocopy(buf []byte, w thrift.Nocopy offset := 0 if p != nil { offset += p.fastWriteField3(buf[offset:], w) + offset += p.fastWriteField5(buf[offset:], w) + offset += p.fastWriteField7(buf[offset:], w) + offset += p.fastWriteField4(buf[offset:], w) + offset += p.fastWriteField6(buf[offset:], w) } offset += thrift.Binary.WriteFieldStop(buf[offset:]) return offset @@ -30007,6 +30243,10 @@ func (p *RetryExperimentOpenAPIData) BLength() int { l := 0 if p != nil { l += p.field3Length() + l += p.field4Length() + l += p.field5Length() + l += p.field6Length() + l += p.field7Length() } l += thrift.Binary.FieldStopLength() return l @@ -30021,6 +30261,49 @@ func (p *RetryExperimentOpenAPIData) fastWriteField3(buf []byte, w thrift.Nocopy return offset } +func (p *RetryExperimentOpenAPIData) fastWriteField4(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetRequestID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRING, 4) + offset += thrift.Binary.WriteStringNocopy(buf[offset:], w, *p.RequestID) + } + return offset +} + +func (p *RetryExperimentOpenAPIData) fastWriteField5(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetAcceptedCount() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 5) + offset += thrift.Binary.WriteI32(buf[offset:], *p.AcceptedCount) + } + return offset +} + +func (p *RetryExperimentOpenAPIData) fastWriteField6(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetRejectedItems() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.LIST, 6) + listBeginOffset := offset + offset += thrift.Binary.ListBeginLength() + var length int + for _, v := range p.RejectedItems { + length++ + offset += v.FastWriteNocopy(buf[offset:], w) + } + thrift.Binary.WriteListBegin(buf[listBeginOffset:], thrift.STRUCT, length) + } + return offset +} + +func (p *RetryExperimentOpenAPIData) fastWriteField7(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetSnapshotAtMs() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 7) + offset += thrift.Binary.WriteI64(buf[offset:], *p.SnapshotAtMs) + } + return offset +} + func (p *RetryExperimentOpenAPIData) field3Length() int { l := 0 if p.IsSetRunID() { @@ -30030,6 +30313,46 @@ func (p *RetryExperimentOpenAPIData) field3Length() int { return l } +func (p *RetryExperimentOpenAPIData) field4Length() int { + l := 0 + if p.IsSetRequestID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.StringLengthNocopy(*p.RequestID) + } + return l +} + +func (p *RetryExperimentOpenAPIData) field5Length() int { + l := 0 + if p.IsSetAcceptedCount() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + +func (p *RetryExperimentOpenAPIData) field6Length() int { + l := 0 + if p.IsSetRejectedItems() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.ListBeginLength() + for _, v := range p.RejectedItems { + _ = v + l += v.BLength() + } + } + return l +} + +func (p *RetryExperimentOpenAPIData) field7Length() int { + l := 0 + if p.IsSetSnapshotAtMs() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + func (p *RetryExperimentOpenAPIData) DeepCopy(s interface{}) error { src, ok := s.(*RetryExperimentOpenAPIData) if !ok { @@ -30041,6 +30364,39 @@ func (p *RetryExperimentOpenAPIData) DeepCopy(s interface{}) error { p.RunID = &tmp } + if src.RequestID != nil { + var tmp string + if *src.RequestID != "" { + tmp = kutils.StringDeepCopy(*src.RequestID) + } + p.RequestID = &tmp + } + + if src.AcceptedCount != nil { + tmp := *src.AcceptedCount + p.AcceptedCount = &tmp + } + + if src.RejectedItems != nil { + p.RejectedItems = make([]*experiment.RetryRejectedItem, 0, len(src.RejectedItems)) + for _, elem := range src.RejectedItems { + var _elem *experiment.RetryRejectedItem + if elem != nil { + _elem = &experiment.RetryRejectedItem{} + if err := _elem.DeepCopy(elem); err != nil { + return err + } + } + + p.RejectedItems = append(p.RejectedItems, _elem) + } + } + + if src.SnapshotAtMs != nil { + tmp := *src.SnapshotAtMs + p.SnapshotAtMs = &tmp + } + return nil } diff --git a/backend/modules/evaluation/application/convertor/experiment/expt.go b/backend/modules/evaluation/application/convertor/experiment/expt.go index ced79c9d0a..7f72319378 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt.go @@ -63,7 +63,7 @@ func (e *EvalConfConvert) ConvertToEntity(cer *expt.CreateExperimentRequest, eva } ec.ConnectorConf.EvaluatorsConf = evalsConf } - if cer.GetItemRetryNum() > 0 { + if cer.IsSetItemRetryNum() && cer.GetItemRetryNum() >= 0 { ec.ItemRetryNum = gptr.Of(int(cer.GetItemRetryNum())) } if cer.IsSetEnableExtractTrajectory() { @@ -346,6 +346,7 @@ func ToExptDTO(experiment *entity.Experiment) *domain_expt.Experiment { res := &domain_expt.Experiment{ ID: gptr.Of(experiment.ID), + LatestRunID: gptr.Of(experiment.LatestRunID), Name: gptr.Of(experiment.Name), Desc: gptr.Of(experiment.Description), ExperimentGroupKey: gptr.Of(experiment.ExperimentGroupKey), @@ -1024,7 +1025,7 @@ func buildExptConfFromEvalSetConfigs(cer *expt.CreateExperimentRequest, runConfi Ext: cer.Ext, RunModeConfig: runModeConfigDTO2DO(cer.GetRunModeConfig()), } - if cer.GetItemRetryNum() > 0 { + if cer.IsSetItemRetryNum() && cer.GetItemRetryNum() >= 0 { ec.ItemRetryNum = gptr.Of(int(cer.GetItemRetryNum())) } if cer.IsSetEnableExtractTrajectory() { diff --git a/backend/modules/evaluation/application/convertor/experiment/expt_retry_num_test.go b/backend/modules/evaluation/application/convertor/experiment/expt_retry_num_test.go new file mode 100644 index 0000000000..da29052656 --- /dev/null +++ b/backend/modules/evaluation/application/convertor/experiment/expt_retry_num_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package experiment_test + +import ( + "encoding/json" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + + domainexpt "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/expt" + exptpb "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/expt" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/application/convertor/experiment" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + mysqlconvert "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" +) + +// Dropping explicit zero, accepting a negative value, or clamping a positive +// value during creation must change these independently specified expectations. +func TestConvertCreateReq_ItemRetryNum(t *testing.T) { + cases := []struct { + name string + input *int32 + want *int + }{ + {name: "unset"}, + {name: "explicit_zero", input: gptr.Of(int32(0)), want: gptr.Of(0)}, + {name: "positive", input: gptr.Of(int32(3)), want: gptr.Of(3)}, + {name: "negative", input: gptr.Of(int32(-1))}, + {name: "int32_max", input: gptr.Of(int32(2147483647)), want: gptr.Of(2147483647)}, + } + for _, multi := range []bool{false, true} { + branch := "single_set" + if multi { + branch = "multi_set_config" + } + t.Run(branch, func(t *testing.T) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := &exptpb.CreateExperimentRequest{ + WorkspaceID: 1001, + ItemRetryNum: tc.input, + } + if multi { + req.EvalSetSourceType = gptr.Of(domainexpt.ExptEvalSetSourceType_MultiSetConfig) + req.EvalSetConfigs = []*domainexpt.EvalSetConfig{{EvalSetID: 111, EvalSetVersionID: 222}} + } + require.NoError(t, req.IsValid()) + param, err := experiment.ConvertCreateReq(req, nil) + require.NoError(t, err) + require.NotNil(t, param) + require.NotNil(t, param.ExptConf) + require.Equal(t, tc.want, param.ExptConf.ItemRetryNum) + + // CreateExpt freezes req.ExptConf as Experiment.EvalConf unchanged. + // Exercise its actual persistence serializer, without a DB or platform call. + frozen := &entity.Experiment{EvalConf: param.ExptConf} + po, err := mysqlconvert.NewExptConverter().DO2PO(frozen) + require.NoError(t, err) + require.NotNil(t, po.EvalConf) + t.Logf("FREEZE_JSON branch=%s case=%s json=%s", branch, tc.name, string(*po.EvalConf)) + var fields map[string]json.RawMessage + require.NoError(t, json.Unmarshal(*po.EvalConf, &fields)) + raw, present := fields["ItemRetryNum"] + require.True(t, present) + if tc.want == nil { + require.JSONEq(t, "null", string(raw)) + } else { + var retryNum int + require.NoError(t, json.Unmarshal(raw, &retryNum)) + require.Equal(t, *tc.want, retryNum) + } + }) + } + }) + } +} diff --git a/backend/modules/evaluation/application/convertor/experiment/expt_test.go b/backend/modules/evaluation/application/convertor/experiment/expt_test.go index 04c65d188c..0ceb943798 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt_test.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt_test.go @@ -4,6 +4,8 @@ package experiment import ( + stdjson "encoding/json" + "fmt" "reflect" "testing" "time" @@ -2713,3 +2715,76 @@ func TestToExptDTO_EvalSetByNilCheck(t *testing.T) { assert.Nil(t, result.EvalSet) }) } + +func TestToExptDTO_LatestRunID(t *testing.T) { + t.Run("old read response leaves latest run absent", func(t *testing.T) { + var experiment domain_expt.Experiment + require.NoError(t, stdjson.Unmarshal([]byte(`{"id":42}`), &experiment)) + assert.Nil(t, experiment.LatestRunID) + assert.Zero(t, experiment.GetLatestRunID()) + }) + + for _, runID := range []int64{0, 9007199254740993} { + t.Run(fmt.Sprint(runID), func(t *testing.T) { + result := ToExptDTO(&entity.Experiment{ID: 42, LatestRunID: runID}) + field := reflect.ValueOf(result).Elem().FieldByName("LatestRunID") + require.True(t, field.IsValid(), "Experiment DTO must expose the authoritative latest_run_id") + require.False(t, field.IsNil(), "zero must remain zero instead of inferring another run") + assert.Equal(t, runID, field.Elem().Int()) + }) + } +} + +func TestRetryExperimentContractRoundTrip(t *testing.T) { + t.Run("current run request survives JSON and Thrift", func(t *testing.T) { + input := `{"retry_mode":3,"workspace_id":7,"expt_id":42,"item_ids":[701],"expected_run_id":9007199254740993,"request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05"}` + var request expt.RetryExperimentRequest + require.NoError(t, stdjson.Unmarshal([]byte(input), &request)) + buf := make([]byte, request.BLength()) + require.Equal(t, len(buf), request.FastWrite(buf)) + var decoded expt.RetryExperimentRequest + _, err := decoded.FastRead(buf) + require.NoError(t, err) + body, err := stdjson.Marshal(&decoded) + require.NoError(t, err) + assert.JSONEq(t, input, string(body)) + assert.Contains(t, string(body), `"expected_run_id":9007199254740993`) + }) + + t.Run("legacy request keeps new fields absent", func(t *testing.T) { + input := `{"retry_mode":2,"workspace_id":7,"expt_id":42,"item_ids":null}` + var request expt.RetryExperimentRequest + require.NoError(t, stdjson.Unmarshal([]byte(input), &request)) + body, err := stdjson.Marshal(&request) + require.NoError(t, err) + assert.JSONEq(t, input, string(body)) + }) + + t.Run("legacy response keeps new fields absent", func(t *testing.T) { + input := `{"run_id":7,"BaseResp":{"StatusCode":0,"StatusMessage":""}}` + var response expt.RetryExperimentResponse + require.NoError(t, stdjson.Unmarshal([]byte(input), &response)) + buf := make([]byte, response.BLength()) + require.Equal(t, len(buf), response.FastWrite(buf)) + var decoded expt.RetryExperimentResponse + _, err := decoded.FastRead(buf) + require.NoError(t, err) + body, err := stdjson.Marshal(&decoded) + require.NoError(t, err) + assert.JSONEq(t, input, string(body)) + }) + + t.Run("receipt preserves zero acceptance and rejected items", func(t *testing.T) { + input := `{"run_id":9007199254740993,"request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","accepted_count":0,"rejected_items":[{"item_id":701,"reason":"busy"}],"snapshot_at_ms":1792497600000,"BaseResp":{"StatusCode":0,"StatusMessage":""}}` + var response expt.RetryExperimentResponse + require.NoError(t, stdjson.Unmarshal([]byte(input), &response)) + buf := make([]byte, response.BLength()) + require.Equal(t, len(buf), response.FastWrite(buf)) + var decoded expt.RetryExperimentResponse + _, err := decoded.FastRead(buf) + require.NoError(t, err) + body, err := stdjson.Marshal(&decoded) + require.NoError(t, err) + assert.JSONEq(t, input, string(body)) + }) +} diff --git a/backend/modules/evaluation/application/convertor/experiment/openapi.go b/backend/modules/evaluation/application/convertor/experiment/openapi.go index ef9b06b3d6..d91d1875fa 100644 --- a/backend/modules/evaluation/application/convertor/experiment/openapi.go +++ b/backend/modules/evaluation/application/convertor/experiment/openapi.go @@ -386,6 +386,7 @@ func DomainExperimentDTO2OpenAPI(dto *domainExpt.Experiment) *openapiExperiment. result := &openapiExperiment.Experiment{ ID: dto.ID, + LatestRunID: dto.LatestRunID, Name: dto.Name, Description: dto.Desc, ExperimentGroupKey: dto.ExperimentGroupKey, @@ -707,6 +708,7 @@ func OpenAPIExptDO2DTO(experiment *entity.Experiment) *openapiExperiment.Experim result := &openapiExperiment.Experiment{ ID: gptr.Of(experiment.ID), + LatestRunID: gptr.Of(experiment.LatestRunID), Name: gptr.Of(experiment.Name), ExperimentGroupKey: gptr.Of(experiment.ExperimentGroupKey), ExptStats: openAPIExperimentStatsDO2DTO(experiment.Stats), diff --git a/backend/modules/evaluation/application/convertor/experiment/openapi_running_retry_test.go b/backend/modules/evaluation/application/convertor/experiment/openapi_running_retry_test.go new file mode 100644 index 0000000000..92fcea4987 --- /dev/null +++ b/backend/modules/evaluation/application/convertor/experiment/openapi_running_retry_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package experiment + +import ( + "reflect" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + domainexpt "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/expt" + "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/openapi" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/pkg/lang/js_conv" +) + +func TestOpenAPIRunningRetryLatestRun(t *testing.T) { + assert.Nil(t, DomainExperimentDTO2OpenAPI(nil)) + assert.Nil(t, OpenAPIExptDO2DTO(nil)) + for _, runID := range []int64{0, 9007199254740993, 9223372036854775807} { + for _, dto := range []interface{}{ + DomainExperimentDTO2OpenAPI(&domainexpt.Experiment{ID: gptr.Of(int64(7)), LatestRunID: gptr.Of(runID)}), + OpenAPIExptDO2DTO(&entity.Experiment{ID: 7, LatestRunID: runID}), + } { + field := reflect.ValueOf(dto).Elem().FieldByName("LatestRunID") + require.True(t, field.IsValid(), "OpenAPI experiment must expose the authoritative latest_run_id") + require.False(t, field.IsNil()) + assert.Equal(t, runID, field.Elem().Int()) + } + } +} + +func TestOpenAPIRunningRetryContractRoundTrip(t *testing.T) { + t.Run("request JSON and Thrift preserve identity", func(t *testing.T) { + input := `{"workspace_id":"9","experiment_id":"7","retry_mode":"retry_target_items","item_ids":["9223372036854775806"],"expected_run_id":"9223372036854775807","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05"}` + var req openapi.RetryExperimentOApiRequest + require.NoError(t, js_conv.GetUnmarshaler()([]byte(input), &req)) + buf := make([]byte, req.BLength()) + require.Equal(t, len(buf), req.FastWrite(buf)) + var decoded openapi.RetryExperimentOApiRequest + _, err := decoded.FastRead(buf) + require.NoError(t, err) + body, err := js_conv.GetMarshaler()(&decoded) + require.NoError(t, err) + assert.JSONEq(t, input, string(body)) + }) + for _, input := range []string{ + `{"run_id":"9223372036854775807","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","accepted_count":0,"rejected_items":[],"snapshot_at_ms":"1793275200123"}`, + `{"run_id":"9223372036854775807","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","accepted_count":1,"rejected_items":[{"item_id":"9223372036854775806","reason":"busy"}],"snapshot_at_ms":"1793275200123"}`, + } { + t.Run(input, func(t *testing.T) { + var receipt openapi.RetryExperimentOpenAPIData + require.NoError(t, js_conv.GetUnmarshaler()([]byte(input), &receipt)) + buf := make([]byte, receipt.BLength()) + require.Equal(t, len(buf), receipt.FastWrite(buf)) + var decoded openapi.RetryExperimentOpenAPIData + _, err := decoded.FastRead(buf) + require.NoError(t, err) + body, err := js_conv.GetMarshaler()(&decoded) + require.NoError(t, err) + assert.JSONEq(t, input, string(body)) + }) + } +} diff --git a/backend/modules/evaluation/application/current_retry_locker.go b/backend/modules/evaluation/application/current_retry_locker.go new file mode 100644 index 0000000000..c2047a2363 --- /dev/null +++ b/backend/modules/evaluation/application/current_retry_locker.go @@ -0,0 +1,14 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package application + +import ( + "github.com/coze-dev/coze-loop/backend/infra/lock" + "github.com/coze-dev/coze-loop/backend/infra/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service" +) + +func NewManualRetryLockerFactory(client redis.Cmdable) service.ManualRetryLockerFactory { + return func() lock.ILocker { return lock.NewRedisLocker(client) } +} diff --git a/backend/modules/evaluation/application/eval_openapi_app.go b/backend/modules/evaluation/application/eval_openapi_app.go index 26ccd43ea5..e59740ba7f 100644 --- a/backend/modules/evaluation/application/eval_openapi_app.go +++ b/backend/modules/evaluation/application/eval_openapi_app.go @@ -40,6 +40,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" @@ -1312,6 +1313,7 @@ func (e *EvalOpenAPIApplication) ReportEvalTargetInvokeResult_(ctx context.Conte logs.CtxWarn(ctx, "report target record, actx missing, invoke_id: %v, space_id: %v", req.GetInvokeID(), req.GetWorkspaceID()) return nil, errorx.NewByCode(errno.ResourceNotFoundCode, errorx.WithExtraMsg(fmt.Sprintf("eval async context not found, invoke_id: %v", req.GetInvokeID()))) } + actx = actx.Clone() // 调试场景（actx.Event == nil）：无论成功失败都 best-effort 销毁沙箱执行。 // @@ -1341,6 +1343,19 @@ func (e *EvalOpenAPIApplication) ReportEvalTargetInvokeResult_(ctx context.Conte } else { logs.CtxInfo(ctx, "report target record, record_id: %v, space_id: %v, expt_id: %v, expt_run_id: %v, item_id: %v", req.GetInvokeID(), req.GetWorkspaceID(), actx.Event.ExptID, actx.Event.ExptRunID, actx.Event.EvalSetItemID) } + var resumeEvent *entity.ExptItemEvalEvent + if actx.Event != nil { + // Prepare the original source before the terminal write, so a read failure + // can be retried while ReportInvokeRecords still accepts this invocation. + record, err := e.targetSvc.GetRecordByID(contexts.WithCtxWriteDB(ctx), req.GetWorkspaceID(), req.GetInvokeID()) + if err != nil { + return nil, err + } + if record == nil || record.ID != req.GetInvokeID() || record.TurnID < 0 { + return nil, errorx.New("target resume source unavailable, invoke_id: %v", req.GetInvokeID()) + } + resumeEvent = actx.Event.CloneWithResumeSource("target", record.ID, record.TurnID) + } outputData := target.ToInvokeOutputDataDO(req) outputData.TimeConsumingMS = gptr.Of(time.Now().UnixMilli() - actx.AsyncUnixMS) if err := e.targetSvc.ReportInvokeRecords(ctx, &entity.ReportTargetRecordParam{ @@ -1361,8 +1376,8 @@ func (e *EvalOpenAPIApplication) ReportEvalTargetInvokeResult_(ctx context.Conte // 用 AsyncCtx.AsyncUnixMS 与当前时间差计算端到端异步耗时。仅对沙箱 agent 目标上报的调用生效。 e.emitSandboxAgentInvokeFinished(ctx, req, actx) - if actx.Event != nil { - if err := e.publisher.PublishExptRecordEvalEvent(ctx, actx.Event, gptr.Of(e.configer.GetTargetTrajectoryConf(ctx).GetExtractInterval(req.GetWorkspaceID())+time.Second*35), + if resumeEvent != nil { + if err := e.publisher.PublishExptRecordEvalEvent(ctx, resumeEvent, gptr.Of(e.configer.GetTargetTrajectoryConf(ctx).GetExtractInterval(req.GetWorkspaceID())+time.Second*35), func(event *entity.ExptItemEvalEvent) { event.AsyncReportTrigger = true }); err != nil { @@ -2343,11 +2358,13 @@ func (e *EvalOpenAPIApplication) RetryExperimentOApi(ctx context.Context, req *o } retryReq := &exptpb.RetryExperimentRequest{ - RetryMode: domain_expt.ExptRetryModePtr(retryMode), - WorkspaceID: req.WorkspaceID, - ExptID: req.ExperimentID, - ItemIds: req.ItemIds, - Ext: req.Ext, + RetryMode: domain_expt.ExptRetryModePtr(retryMode), + WorkspaceID: req.WorkspaceID, + ExptID: req.ExperimentID, + ItemIds: req.ItemIds, + ExpectedRunID: req.ExpectedRunID, + RequestID: req.RequestID, + Ext: req.Ext, } resp, err := e.experimentApp.RetryExperiment(ctx, retryReq) @@ -2355,9 +2372,21 @@ func (e *EvalOpenAPIApplication) RetryExperimentOApi(ctx context.Context, req *o return nil, err } + var rejected []*experiment.RetryRejectedItem + if resp.RejectedItems != nil { + rejected = make([]*experiment.RetryRejectedItem, 0, len(resp.RejectedItems)) + for _, item := range resp.RejectedItems { + rejected = append(rejected, &experiment.RetryRejectedItem{ItemID: item.ItemID, Reason: item.Reason}) + } + } + return &openapi.RetryExperimentOApiResponse{ Data: &openapi.RetryExperimentOpenAPIData{ - RunID: resp.RunID, + RunID: resp.RunID, + RequestID: resp.RequestID, + AcceptedCount: resp.AcceptedCount, + RejectedItems: rejected, + SnapshotAtMs: resp.SnapshotAtMs, }, }, nil } @@ -3775,6 +3804,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex logs.CtxWarn(ctx, "[ExptTurnEval] report evaluator record, actx missing, invoke_id: %v, space_id: %v", req.GetInvokeID(), req.GetWorkspaceID()) return nil, errorx.New("eval async context not found, invoke_id: %v", req.GetInvokeID()) } + actx = actx.Clone() logs.CtxInfo(ctx, "[ExptTurnEval] report evaluator record, invoke_id: %v, evaluator_version_id: %v, space_id: %v, expt_id: %v, expt_run_id: %v, item_id: %v， status: %v", req.GetInvokeID(), actx.EvaluatorVersionID, req.GetWorkspaceID(), actx.Event.GetExptID(), actx.Event.GetExptRunID(), actx.Event.GetEvalSetItemID(), req.GetStatus()) @@ -3796,26 +3826,37 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex } var resumeErr error if actx.Event != nil { + // Refresh readiness alone; source and notification settings stay frozen. if !actx.ResumeReady { latestCtx, ctxErr := e.asyncRepo.GetEvalAsyncCtxStrong(ctx, asyncCtxKey) if ctxErr != nil { - return nil, ctxErr - } - if latestCtx != nil { - actx = latestCtx + resumeErr = ctxErr + } else if latestCtx != nil { + actx.ResumeReady = latestCtx.ResumeReady } } - if !actx.ResumeReady { + if resumeErr == nil && !actx.ResumeReady { resumeErr = errorx.New("evaluator resume is not ready, invoke_id: %v", req.GetInvokeID()) - } else { - if err := e.publisher.PublishExptRecordEvalEvent(ctx, actx.Event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { - event.AsyncEvaluatorReportTrigger = true - }); err != nil { - return nil, err + } + if resumeErr == nil { + record, err := e.evaluatorRecordService.GetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), req.GetInvokeID(), false) + if err != nil { + resumeErr = err + } else if record == nil || record.ID != req.GetInvokeID() || record.TurnID < 0 { + resumeErr = errorx.New("evaluator resume source unavailable, invoke_id: %v", req.GetInvokeID()) + } else if record.Status == entity.EvaluatorRunStatusAsyncInvoking { + resumeErr = errorx.New("evaluator record is still invoking, invoke_id: %v", req.GetInvokeID()) + } else { + event := actx.Event.CloneWithResumeSource("evaluator", record.ID, record.TurnID) + resumeErr = e.publisher.PublishExptRecordEvalEvent(ctx, event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { + event.AsyncEvaluatorReportTrigger = true + }) } } } + // An Applied result only occurs once; deliver its best-effort notification + // before returning a recovery error that the provider may retry as Duplicate. if outcome == entity.ReportEvaluatorResultApplied && actx.CallbackURL != "" { payload := &openapi.EvaluatorCallbackPayloadOApi{ InvokeID: gptr.Of(req.GetInvokeID()), diff --git a/backend/modules/evaluation/application/eval_openapi_app_skip_convert_test.go b/backend/modules/evaluation/application/eval_openapi_app_skip_convert_test.go index f96f6a21cf..f72945126c 100644 --- a/backend/modules/evaluation/application/eval_openapi_app_skip_convert_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_skip_convert_test.go @@ -50,6 +50,7 @@ func TestReportEvalTargetInvokeResult_SandboxCallee_SetsSkipErrMsgConvert(t *tes assert.True(t, param.SkipErrMsgConvert, "sandbox callee should skip err msg convert") return nil }) + targetSvc.EXPECT().GetRecordByID(gomock.Any(), req.GetWorkspaceID(), req.GetInvokeID()).Return(&entity.EvalTargetRecord{ID: req.GetInvokeID(), TurnID: 909}, nil) configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(&entity.TargetTrajectoryConf{}) publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) @@ -93,6 +94,7 @@ func TestReportEvalTargetInvokeResult_NonSandboxCallee_KeepsConvert(t *testing.T assert.False(t, param.SkipErrMsgConvert, "non-sandbox callee should not skip err msg convert") return nil }) + targetSvc.EXPECT().GetRecordByID(gomock.Any(), req.GetWorkspaceID(), req.GetInvokeID()).Return(&entity.EvalTargetRecord{ID: req.GetInvokeID(), TurnID: 909}, nil) configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(&entity.TargetTrajectoryConf{}) publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) diff --git a/backend/modules/evaluation/application/eval_openapi_app_test.go b/backend/modules/evaluation/application/eval_openapi_app_test.go index ccb6b48730..843f5d3e99 100755 --- a/backend/modules/evaluation/application/eval_openapi_app_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_test.go @@ -2366,14 +2366,16 @@ func TestEvalOpenAPIApplication_ReportEvalTargetInvokeResult(t *testing.T) { event := &entity.ExptItemEvalEvent{} actx := &entity.EvalAsyncCtx{AsyncUnixMS: time.Now().Add(-150 * time.Millisecond).UnixMilli(), Event: event, Session: session} asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), strconv.FormatInt(publisherErrorReq.GetInvokeID(), 10)).Return(actx, nil) + targetSvc.EXPECT().GetRecordByID(gomock.Any(), publisherErrorReq.GetWorkspaceID(), publisherErrorReq.GetInvokeID()).Return(&entity.EvalTargetRecord{ID: publisherErrorReq.GetInvokeID(), TurnID: 909}, nil) targetSvc.EXPECT().ReportInvokeRecords(gomock.Any(), gomock.AssignableToTypeOf(&entity.ReportTargetRecordParam{})).DoAndReturn(func(_ context.Context, param *entity.ReportTargetRecordParam) error { assert.Equal(t, session, param.Session) return nil }) conf := &entity.TargetTrajectoryConf{} configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(conf) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, evt *entity.ExptItemEvalEvent, duration *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { - assert.Equal(t, event, evt) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, evt *entity.ExptItemEvalEvent, duration *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + assert.NotSame(t, event, evt) + assertCallbackResumeSource(t, evt, "target", publisherErrorReq.GetInvokeID(), 909) if assert.NotNil(t, duration) { assert.Equal(t, 50*time.Second, *duration) } @@ -2403,18 +2405,23 @@ func TestEvalOpenAPIApplication_ReportEvalTargetInvokeResult(t *testing.T) { event := &entity.ExptItemEvalEvent{SpaceID: 44, ExptID: 1, ExptRunID: 2, EvalSetItemID: 3} actx := &entity.EvalAsyncCtx{AsyncUnixMS: time.Now().Add(-100 * time.Millisecond).UnixMilli(), Event: event} asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "404").Return(actx, nil) + read := targetSvc.EXPECT().GetRecordByID(gomock.Any(), int64(44), int64(404)).Return(&entity.EvalTargetRecord{ID: 404, TurnID: 0}, nil) report := targetSvc.EXPECT().ReportInvokeRecords(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, param *entity.ReportTargetRecordParam) error { assert.Equal(t, int64(44), param.SpaceID) assert.Equal(t, int64(404), param.RecordID) assert.Equal(t, entity.EvalTargetRunStatusSuccess, param.Status) return nil - }) + }).After(read) configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(&entity.TargetTrajectoryConf{}) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).After(report).DoAndReturn(func(_ context.Context, evt *entity.ExptItemEvalEvent, duration *time.Duration, option func(*entity.ExptItemEvalEvent)) error { + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).After(report).DoAndReturn(func(_ context.Context, evt *entity.ExptItemEvalEvent, duration *time.Duration, option func(*entity.ExptItemEvalEvent)) error { + assert.NotSame(t, event, evt) + assert.Equal(t, event.ExptRunID, evt.ExptRunID) + assertCallbackResumeSource(t, evt, "target", 404, 0) require.NotNil(t, duration) assert.Equal(t, 50*time.Second, *duration) option(evt) assert.True(t, evt.AsyncReportTrigger) + assert.False(t, event.AsyncReportTrigger) return nil }) }, @@ -2427,6 +2434,7 @@ func TestEvalOpenAPIApplication_ReportEvalTargetInvokeResult(t *testing.T) { event := &entity.ExptItemEvalEvent{} actx := &entity.EvalAsyncCtx{AsyncUnixMS: time.Now().Add(-120 * time.Millisecond).UnixMilli(), Event: event, Session: session} asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), strconv.FormatInt(failedReq.GetInvokeID(), 10)).Return(actx, nil) + targetSvc.EXPECT().GetRecordByID(gomock.Any(), failedReq.GetWorkspaceID(), failedReq.GetInvokeID()).Return(&entity.EvalTargetRecord{ID: failedReq.GetInvokeID(), TurnID: 909}, nil) targetSvc.EXPECT().ReportInvokeRecords(gomock.Any(), gomock.AssignableToTypeOf(&entity.ReportTargetRecordParam{})).DoAndReturn(func(_ context.Context, param *entity.ReportTargetRecordParam) error { assert.Equal(t, entity.EvalTargetRunStatusFail, param.Status) if assert.NotNil(t, param.OutputData) { @@ -2440,8 +2448,9 @@ func TestEvalOpenAPIApplication_ReportEvalTargetInvokeResult(t *testing.T) { }) conf := &entity.TargetTrajectoryConf{} configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(conf) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, evt *entity.ExptItemEvalEvent, duration *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { - assert.Equal(t, event, evt) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, evt *entity.ExptItemEvalEvent, duration *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + assert.NotSame(t, event, evt) + assertCallbackResumeSource(t, evt, "target", failedReq.GetInvokeID(), 909) if assert.NotNil(t, duration) { assert.Equal(t, 50*time.Second, *duration) } @@ -6816,7 +6825,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { } return entity.ReportEvaluatorResultApplied, nil }) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), resumeEvent, gomock.Not(gomock.Nil()), gomock.Any()).DoAndReturn( + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Not(gomock.Nil()), gomock.Any()).DoAndReturn( func(_ context.Context, ev *entity.ExptItemEvalEvent, _ *time.Duration, modifyFunc func(*entity.ExptItemEvalEvent)) error { modifyFunc(ev) assert.True(t, ev.AsyncEvaluatorReportTrigger) @@ -7055,7 +7064,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { EvaluatorVersionID: 9, }, nil), ) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Not(gomock.Nil()), gomock.Any()).Return(nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Not(gomock.Nil()), gomock.Any()).Return(nil) }, }, { @@ -7095,7 +7104,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { Event: event, ResumeReady: true, EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", }, nil) evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultDuplicate, nil) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) dispatcher.EXPECT().Dispatch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) }, }, @@ -7271,17 +7280,20 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { auth := rpcmocks.NewMockIAuthProvider(ctrl) asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), invokeID, false).Return(&entity.EvaluatorRecord{ID: invokeID, TurnID: 909, Status: entity.EvaluatorRunStatusSuccess}, nil).AnyTimes() publisher := eventmocks.NewMockExptEventPublisher(ctrl) callbackDispatcher := servicemocks.NewMockIEvaluatorCallbackDispatcher(ctrl) metric := &fakeOpenAPIMetric{} app := &EvalOpenAPIApplication{ - auth: auth, - asyncRepo: asyncRepo, - evaluatorService: evaluatorSvc, - publisher: publisher, - callbackDispatcher: callbackDispatcher, - metric: metric, + auth: auth, + asyncRepo: asyncRepo, + evaluatorService: evaluatorSvc, + evaluatorRecordService: recordSvc, + publisher: publisher, + callbackDispatcher: callbackDispatcher, + metric: metric, } tc.setup(auth, asyncRepo, evaluatorSvc, publisher, callbackDispatcher) diff --git a/backend/modules/evaluation/application/eval_openapi_async_source_test.go b/backend/modules/evaluation/application/eval_openapi_async_source_test.go new file mode 100644 index 0000000000..d01d0737a0 --- /dev/null +++ b/backend/modules/evaluation/application/eval_openapi_async_source_test.go @@ -0,0 +1,464 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package application + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + datasetdto "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/data/dataset" + datasetentity "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/data/domain/dataset" + "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/openapi" + "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/spi" + configmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc" + rpcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + servicemocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + datasetadapter "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/rpc/data" + datasetmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/rpc/data/mocks" +) + +// Removing the real entrypoint's record-derived source or Event copy must fail these assertions. +func assertCallbackResumeSource(t *testing.T, event *entity.ExptItemEvalEvent, kind string, recordID, turnID int64) { + t.Helper() + encoded, err := json.Marshal(event) + require.NoError(t, err) + var wire struct { + ResumeSource *struct { + Kind string + TurnID int64 + InvokeID string + RecordID int64 + } + } + require.NoError(t, json.Unmarshal(encoded, &wire)) + require.NotNil(t, wire.ResumeSource, "real callback must publish the original record source") + assert.Equal(t, kind, wire.ResumeSource.Kind) + assert.Equal(t, recordID, wire.ResumeSource.RecordID) + assert.Equal(t, strconv.FormatInt(recordID, 10), wire.ResumeSource.InvokeID) + assert.Equal(t, turnID, wire.ResumeSource.TurnID) +} + +func callbackDatasetTurn0(t *testing.T, ctrl *gomock.Controller) *entity.Turn { + t.Helper() + client := datasetmocks.NewMockClient(ctrl) + client.EXPECT().BatchGetDatasetItems(gomock.Any(), gomock.Any()).Return(&datasetdto.BatchGetDatasetItemsResponse{Items: []*datasetentity.DatasetItem{{ + ItemID: gptr.Of(int64(4)), DatasetID: gptr.Of(int64(9)), Data: []*datasetentity.FieldData{{Name: gptr.Of("input"), Content: gptr.Of("single turn")}}, + }}}, nil) + items, err := datasetadapter.NewDatasetRPCAdapter(client).BatchGetDatasetItems(context.Background(), &rpc.BatchGetDatasetItemsParam{}) + require.NoError(t, err) + require.Len(t, items, 1) + require.Len(t, items[0].Turns, 1) + require.Zero(t, items[0].Turns[0].ID, "production dataset conversion supplies the legitimate single-turn identity") + return items[0].Turns[0] +} + +func TestAsyncSourceFixTargetTurn0(t *testing.T) { + ctrl := gomock.NewController(t) + turn := callbackDatasetTurn0(t, ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + targetSvc := servicemocks.NewMockIEvalTargetService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + configer := configmocks.NewMockIConfiger(ctrl) + original := &entity.ExptItemEvalEvent{ExptID: 2, ExptRunID: 3, EvalSetItemID: turn.ItemID, RequestID: "original-rq"} + asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "100").Return(&entity.EvalAsyncCtx{Event: original}, nil) + targetSvc.EXPECT().GetRecordByID(gomock.Any(), int64(1), int64(100)).Return(&entity.EvalTargetRecord{ID: 100, TurnID: turn.ID}, nil) + written, published := 0, 0 + targetSvc.EXPECT().ReportInvokeRecords(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.ReportTargetRecordParam) error { written++; return nil }).MaxTimes(1) + configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(&entity.TargetTrajectoryConf{}).MaxTimes(1) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + assert.Equal(t, 1, written) + assertCallbackResumeSource(t, event, "target", 100, turn.ID) + assert.Equal(t, original.RequestID, event.RequestID) + published++ + return nil + }).MaxTimes(1) + app := &EvalOpenAPIApplication{asyncRepo: asyncRepo, targetSvc: targetSvc, publisher: publisher, configer: configer} + _, err := app.ReportEvalTargetInvokeResult_(context.Background(), newSuccessInvokeResultReq(1, 100)) + require.NoError(t, err) + assert.Equal(t, 1, published) +} + +func TestAsyncSourceFixEvaluatorTurn0(t *testing.T) { + ctrl := gomock.NewController(t) + turn := callbackDatasetTurn0(t, ctrl) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + svc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + original := &entity.ExptItemEvalEvent{ExptID: 2, ExptRunID: 3, EvalSetItemID: turn.ItemID, RequestID: "original-rq"} + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: original, ResumeReady: true}, nil) + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, TurnID: turn.ID, Status: entity.EvaluatorRunStatusSuccess}, nil) + published := 0 + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + assertCallbackResumeSource(t, event, "evaluator", 100, turn.ID) + assert.Equal(t, original.RequestID, event.RequestID) + published++ + return nil + }).MaxTimes(1) + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: svc, evaluatorRecordService: recordSvc, publisher: publisher} + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), sourceFixEvaluatorRequest()) + require.NoError(t, err) + assert.Equal(t, 1, published) +} + +func sourceFixEvaluatorRequest() *openapi.ReportEvaluatorInvokeResultRequest { + return &openapi.ReportEvaluatorInvokeResultRequest{WorkspaceID: gptr.Of(int64(1)), InvokeID: gptr.Of(int64(100)), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{}}} +} + +func TestAsyncSourceFixAppliedReadFailurePreservesCallback(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + svc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + dispatcher := servicemocks.NewMockIEvaluatorCallbackDispatcher(ctrl) + original := &entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{RequestID: "original-rq"}, ResumeReady: true, CallbackURL: "https://original.example/callback", EvaluatorVersionID: 9, AsyncUnixMS: time.Now().UnixMilli() - 1000} + cause := errors.New("source read failed") + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil).Times(2) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(original, nil).Times(2) + gomock.InOrder( + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultDuplicate, nil), + ) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(nil, cause).Times(2) + dispatched := 0 + dispatcher.EXPECT().Dispatch(gomock.Any(), int64(1), original.CallbackURL, gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ string, payload *openapi.EvaluatorCallbackPayloadOApi) error { + assert.Equal(t, original.EvaluatorVersionID, payload.GetEvaluatorVersionID()) + dispatched++ + return errors.New("best-effort notification failed") + }).MaxTimes(1) + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: svc, evaluatorRecordService: recordSvc, publisher: publisher, callbackDispatcher: dispatcher} + for i := 0; i < 2; i++ { + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), sourceFixEvaluatorRequest()) + assert.Same(t, cause, err, "resume failure must retain its original cause") + assert.Equal(t, 1, dispatched, "Applied notifies once before returning source failure; Duplicate never notifies again") + } +} + +func TestAsyncSourceFixReadinessFreezesNotification(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + svc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + dispatcher := servicemocks.NewMockIEvaluatorCallbackDispatcher(ctrl) + original := &entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{ExptID: 2, ExptRunID: 3, EvalSetItemID: 4, RequestID: "original-rq"}, CallbackURL: "https://original.example/callback", EvaluatorVersionID: 9, AsyncUnixMS: time.Now().UnixMilli() - 1000, Session: &entity.Session{UserID: "original"}} + latest := &entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{ExptID: 20, ExptRunID: 30, EvalSetItemID: 40, RequestID: "other-rq"}, ResumeReady: true, CallbackURL: "https://other.example/callback", EvaluatorVersionID: 99, AsyncUnixMS: time.Now().UnixMilli() + 100000, Session: &entity.Session{UserID: "other"}} + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + gomock.InOrder( + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(original, nil), + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(latest, nil), + ) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, TurnID: 5, Status: entity.EvaluatorRunStatusSuccess}, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + assert.Equal(t, original.Event.RequestID, event.RequestID) + assert.Equal(t, original.Event.ExptRunID, event.ExptRunID) + return nil + }) + dispatched := 0 + dispatcher.EXPECT().Dispatch(gomock.Any(), int64(1), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, url string, payload *openapi.EvaluatorCallbackPayloadOApi) error { + assert.Equal(t, original.CallbackURL, url) + assert.Equal(t, original.EvaluatorVersionID, payload.GetEvaluatorVersionID()) + assert.GreaterOrEqual(t, payload.GetTimeConsumingMs(), int64(1000)) + dispatched++ + return nil + }) + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: svc, evaluatorRecordService: recordSvc, publisher: publisher, callbackDispatcher: dispatcher} + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), sourceFixEvaluatorRequest()) + require.NoError(t, err) + assert.Equal(t, 1, dispatched) + assert.False(t, original.ResumeReady, "readiness refresh must not mutate cached original context") +} + +func TestAsyncSourceFixTargetReadFailureBeforeRecordWrite(t *testing.T) { + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + targetSvc := servicemocks.NewMockIEvalTargetService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + configer := configmocks.NewMockIConfiger(ctrl) + asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "100").Return(&entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{RequestID: "original-rq", ExptRunID: 3}}, nil).Times(2) + cause := errors.New("source DB read failed") + gomock.InOrder( + targetSvc.EXPECT().GetRecordByID(gomock.Any(), int64(1), int64(100)).Return(nil, cause), + targetSvc.EXPECT().GetRecordByID(gomock.Any(), int64(1), int64(100)).Return(&entity.EvalTargetRecord{ID: 100, TurnID: 5}, nil), + ) + written, published := 0, 0 + targetSvc.EXPECT().ReportInvokeRecords(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.ReportTargetRecordParam) error { written++; return nil }).AnyTimes() + configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(&entity.TargetTrajectoryConf{}).AnyTimes() + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + assert.Equal(t, 1, written, "record write still precedes Publish after a retryable source read") + assertCallbackResumeSource(t, event, "target", 100, 5) + published++ + return nil + }).AnyTimes() + app := &EvalOpenAPIApplication{asyncRepo: asyncRepo, targetSvc: targetSvc, publisher: publisher, configer: configer} + _, err := app.ReportEvalTargetInvokeResult_(context.Background(), newSuccessInvokeResultReq(1, 100)) + assert.Same(t, cause, err) + assert.Zero(t, written, "source read failure must not make the old record terminal") + assert.Zero(t, published) + _, err = app.ReportEvalTargetInvokeResult_(context.Background(), newSuccessInvokeResultReq(1, 100)) + require.NoError(t, err) + assert.Equal(t, 1, written) + assert.Equal(t, 1, published) +} + +func TestAsyncSourceFixAppliedRecoveryErrorsNotify(t *testing.T) { + for _, sourceCase := range []string{"missing record", "negative turn", "wrong record", "still invoking", "readiness read failure", "publish failure"} { + t.Run(sourceCase, func(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + svc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + dispatcher := servicemocks.NewMockIEvaluatorCallbackDispatcher(ctrl) + original := &entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{RequestID: "original-rq"}, ResumeReady: sourceCase != "readiness read failure", CallbackURL: "https://original.example/callback"} + cause := errors.New("recovery failed") + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(original, nil) + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) + record := &entity.EvaluatorRecord{ID: 100, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess} + switch sourceCase { + case "missing record": + record = nil + case "negative turn": + record.TurnID = -1 + case "wrong record": + record.ID = 200 + case "still invoking": + record.Status = entity.EvaluatorRunStatusAsyncInvoking + } + if sourceCase == "readiness read failure" { + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(nil, cause) + } else { + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(record, nil) + } + if sourceCase == "publish failure" { + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(cause) + } + dispatched := 0 + dispatcher.EXPECT().Dispatch(gomock.Any(), int64(1), original.CallbackURL, gomock.Any()).DoAndReturn(func(context.Context, int64, string, *openapi.EvaluatorCallbackPayloadOApi) error { + dispatched++ + return nil + }) + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: svc, evaluatorRecordService: recordSvc, publisher: publisher, callbackDispatcher: dispatcher} + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), sourceFixEvaluatorRequest()) + require.Error(t, err) + if sourceCase == "readiness read failure" || sourceCase == "publish failure" { + assert.Same(t, cause, err) + } + assert.Equal(t, 1, dispatched) + }) + } +} + +func TestAsyncSourceTargetCallback(t *testing.T) { + for _, readCase := range []string{"success", "legacy empty request", "read failure", "missing record", "negative turn", "wrong record"} { + t.Run(readCase, func(t *testing.T) { + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + targetSvc := servicemocks.NewMockIEvalTargetService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + configer := configmocks.NewMockIConfiger(ctrl) + const recordID, turnID = int64(7384603622000000101), int64(7384603622000000102) + original := &entity.ExptItemEvalEvent{SpaceID: 1, ExptID: 2, ExptRunID: 3, EvalSetItemID: 4, RequestID: "original-rq", Ext: map[string]string{"inputExpr": "original"}, Session: &entity.Session{UserID: "owner"}} + if readCase == "legacy empty request" { + original.RequestID = "" + } + actx := &entity.EvalAsyncCtx{Event: original, RecordID: recordID, Session: original.Session} + asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), strconv.FormatInt(recordID, 10)).Return(actx, nil).Times(2) + written := 0 + targetSvc.EXPECT().ReportInvokeRecords(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, p *entity.ReportTargetRecordParam) error { + assert.Equal(t, recordID, p.RecordID) + written++ + return nil + }).MaxTimes(2) + record := &entity.EvalTargetRecord{ID: recordID, TurnID: turnID, ExperimentRunID: 3, ItemID: 4} + var readErr error + if readCase == "read failure" { + readErr = errors.New("source storage unavailable") + } + if readCase == "missing record" { + record = nil + } + if readCase == "negative turn" { + record.TurnID = -1 + } + if readCase == "wrong record" { + record.ID++ + } + // Optional before implementation, so the red result is a source assertion, not an unmet mock. + targetSvc.EXPECT().GetRecordByID(gomock.Any(), int64(1), recordID).Return(record, readErr).AnyTimes() + configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).Return(&entity.TargetTrajectoryConf{}).AnyTimes() + var published []*entity.ExptItemEvalEvent + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, modify func(*entity.ExptItemEvalEvent)) error { + assert.Greater(t, written, len(published), "record write must precede Publish") + assert.NotSame(t, original, event) + assert.Equal(t, original.RequestID, event.RequestID) + assert.Equal(t, "original", event.Ext["inputExpr"]) + assert.Equal(t, "owner", event.Session.UserID) + assert.Equal(t, int64(2), event.ExptID) + assert.Equal(t, int64(3), event.ExptRunID) + assert.Equal(t, int64(4), event.EvalSetItemID) + assertCallbackResumeSource(t, event, "target", recordID, turnID) + modify(event) + published = append(published, event) + event.Ext["inputExpr"] = "publisher mutation" + event.Session.UserID = "publisher mutation" + return nil + }).AnyTimes() + app := &EvalOpenAPIApplication{asyncRepo: asyncRepo, targetSvc: targetSvc, publisher: publisher, configer: configer} + for i := 0; i < 2; i++ { + _, err := app.ReportEvalTargetInvokeResult_(context.Background(), newSuccessInvokeResultReq(1, recordID)) + if readCase == "success" || readCase == "legacy empty request" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + } + if readCase == "success" || readCase == "legacy empty request" { + assert.Equal(t, 2, written, "valid source allows the original record write") + } else { + assert.Zero(t, written, "invalid or unreadable source leaves the original record retryable") + } + assert.False(t, original.AsyncReportTrigger) + assert.Equal(t, "original", original.Ext["inputExpr"]) + assert.Equal(t, "owner", original.Session.UserID) + if readCase == "success" || readCase == "legacy empty request" { + require.Len(t, published, 2) + assert.NotSame(t, published[0], published[1]) + } else { + assert.Empty(t, published) + } + }) + } +} + +func TestAsyncSourceEvaluatorCallback(t *testing.T) { + for _, outcome := range []entity.ReportEvaluatorResultOutcome{entity.ReportEvaluatorResultApplied, entity.ReportEvaluatorResultDuplicate, entity.ReportEvaluatorResultConflict} { + t.Run(strconv.Itoa(int(outcome)), func(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + svc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + const recordID, turnID = int64(7384603622000000201), int64(7384603622000000202) + original := &entity.ExptItemEvalEvent{SpaceID: 1, ExptID: 2, ExptRunID: 3, EvalSetItemID: 4, RequestID: "original-rq", Ext: map[string]string{"inputExpr": "original"}, Session: &entity.Session{UserID: "owner"}} + actx := &entity.EvalAsyncCtx{Event: original, RecordID: recordID, ResumeReady: true} + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:"+strconv.FormatInt(recordID, 10)).Return(actx, nil) + written := false + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, p *entity.ReportEvaluatorRecordParam) (entity.ReportEvaluatorResultOutcome, error) { + assert.Equal(t, recordID, p.RecordID) + written = true + return outcome, nil + }) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), recordID, false).Return(&entity.EvaluatorRecord{ID: recordID, TurnID: turnID, Status: entity.EvaluatorRunStatusSuccess}, nil).AnyTimes() + published := false + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, modify func(*entity.ExptItemEvalEvent)) error { + require.True(t, written) + assert.NotSame(t, original, event) + assert.Equal(t, "original-rq", event.RequestID) + assert.Equal(t, original.ExptID, event.ExptID) + assert.Equal(t, original.ExptRunID, event.ExptRunID) + assert.Equal(t, original.EvalSetItemID, event.EvalSetItemID) + assertCallbackResumeSource(t, event, "evaluator", recordID, turnID) + modify(event) + event.Ext["inputExpr"] = "mutation" + event.Session.UserID = "mutation" + published = true + return nil + }).AnyTimes() + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: svc, evaluatorRecordService: recordSvc, publisher: publisher} + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), &openapi.ReportEvaluatorInvokeResultRequest{WorkspaceID: gptr.Of(int64(1)), InvokeID: gptr.Of(recordID), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{}}}) + require.NoError(t, err) + assert.Equal(t, outcome != entity.ReportEvaluatorResultConflict, published) + assert.False(t, original.AsyncEvaluatorReportTrigger) + assert.Equal(t, "original", original.Ext["inputExpr"]) + assert.Equal(t, "owner", original.Session.UserID) + }) + } +} + +func TestAsyncSourceEvaluatorReadinessKeepsOriginalEvent(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + svc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + original := &entity.ExptItemEvalEvent{ExptID: 2, ExptRunID: 3, EvalSetItemID: 4, RequestID: "original-rq"} + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + gomock.InOrder( + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: original}, nil), + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultDuplicate, nil), + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{ResumeReady: true, Event: &entity.ExptItemEvalEvent{ExptID: 20, ExptRunID: 30, EvalSetItemID: 40, RequestID: "other-rq"}}, nil), + ) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, TurnID: 5, Status: entity.EvaluatorRunStatusSuccess}, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, ev *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + assert.Equal(t, "original-rq", ev.RequestID, "readiness reread cannot relabel a callback") + assert.Equal(t, original.ExptID, ev.ExptID) + assert.Equal(t, original.ExptRunID, ev.ExptRunID) + assert.Equal(t, original.EvalSetItemID, ev.EvalSetItemID) + assertCallbackResumeSource(t, ev, "evaluator", 100, 5) + return nil + }) + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: svc, evaluatorRecordService: recordSvc, publisher: publisher} + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), &openapi.ReportEvaluatorInvokeResultRequest{WorkspaceID: gptr.Of(int64(1)), InvokeID: gptr.Of(int64(100)), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{}}}) + require.NoError(t, err) +} + +func TestAsyncSourceEvaluatorReadFailures(t *testing.T) { + for _, sourceCase := range []string{"read failure", "missing", "negative turn", "wrong record", "still invoking"} { + t.Run(sourceCase, func(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + svc := servicemocks.NewMockEvaluatorService(ctrl) + recordSvc := servicemocks.NewMockEvaluatorRecordService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{RequestID: "original-rq"}, ResumeReady: true}, nil) + svc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultDuplicate, nil) + record := &entity.EvaluatorRecord{ID: 100, TurnID: 5, Status: entity.EvaluatorRunStatusSuccess} + var readErr error + switch sourceCase { + case "read failure": + readErr = errors.New("source storage unavailable") + case "missing": + record = nil + case "negative turn": + record.TurnID = -1 + case "wrong record": + record.ID = 200 + case "still invoking": + record.Status = entity.EvaluatorRunStatusAsyncInvoking + } + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(record, readErr) + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: svc, evaluatorRecordService: recordSvc, publisher: publisher} + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), &openapi.ReportEvaluatorInvokeResultRequest{WorkspaceID: gptr.Of(int64(1)), InvokeID: gptr.Of(int64(100)), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{}}}) + require.Error(t, err) + }) + } +} diff --git a/backend/modules/evaluation/application/eval_openapi_early_sandbox_callback_test.go b/backend/modules/evaluation/application/eval_openapi_early_sandbox_callback_test.go index 9250bcc9fc..02257d574e 100644 --- a/backend/modules/evaluation/application/eval_openapi_early_sandbox_callback_test.go +++ b/backend/modules/evaluation/application/eval_openapi_early_sandbox_callback_test.go @@ -35,6 +35,7 @@ func TestSandboxEarlyCallbackUsesPreRegisteredRedisContext(t *testing.T) { event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 21, ExptRunID: 22, EvalSetItemID: 23} prepared := &entity.EvalAsyncCtx{RecordID: 707, Event: event, Callee: "sandbox_agent", AsyncUnixMS: time.Now().Add(-time.Second).UnixMilli()} require.NoError(t, service.PrepareSandboxAsyncContext(ctx, repository, prepared)) + target.EXPECT().GetRecordByID(gomock.Any(), int64(2), int64(707)).Return(&entity.EvalTargetRecord{ID: 707, TurnID: 0}, nil) target.EXPECT().ReportInvokeRecords(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, p *entity.ReportTargetRecordParam) error { require.Equal(t, int64(2), p.SpaceID) require.Equal(t, int64(707), p.RecordID) @@ -49,6 +50,8 @@ func TestSandboxEarlyCallbackUsesPreRegisteredRedisContext(t *testing.T) { require.Equal(t, int64(21), e.ExptID) require.Equal(t, int64(22), e.ExptRunID) require.Equal(t, int64(23), e.EvalSetItemID) + assertCallbackResumeSource(t, e, "target", 707, 0) + require.NotSame(t, event, e) return nil }) response, err := app.ReportEvalTargetInvokeResult_(ctx, request) @@ -65,6 +68,7 @@ func TestSandboxEarlyCallbackUsesPreRegisteredRedisContext(t *testing.T) { require.NoError(t, err) require.Equal(t, prepared.AsyncUnixMS, actual.AsyncUnixMS) require.True(t, actual.ResumeReady) + require.Nil(t, actual.Event.ResumeSource) for _, field := range []string{"space", "experiment", "run", "item", "record", "callee"} { t.Run(field, func(t *testing.T) { wrong := *prepared diff --git a/backend/modules/evaluation/application/eval_openapi_running_retry_test.go b/backend/modules/evaluation/application/eval_openapi_running_retry_test.go new file mode 100644 index 0000000000..fdf53a4225 --- /dev/null +++ b/backend/modules/evaluation/application/eval_openapi_running_retry_test.go @@ -0,0 +1,165 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package application + +import ( + "context" + "errors" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + domainexpt "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/expt" + exptpb "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/expt" + "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/openapi" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc" + rpcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/pkg/lang/js_conv" +) + +func TestOpenAPIRunningRetryIdentityAndReceipt(t *testing.T) { + for _, tc := range []struct { + name string + accepted int32 + rejected []*exptpb.RetryRejectedItem + wantJSON string + }{ + { + name: "partial acceptance", + accepted: 1, + rejected: []*exptpb.RetryRejectedItem{{ItemID: gptr.Of(int64(9223372036854775806)), Reason: gptr.Of("busy")}}, + wantJSON: `{"run_id":"9223372036854775807","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","accepted_count":1,"rejected_items":[{"item_id":"9223372036854775806","reason":"busy"}],"snapshot_at_ms":"1793275200123"}`, + }, + { + name: "zero acceptance with rejection", + rejected: []*exptpb.RetryRejectedItem{{ItemID: gptr.Of(int64(9223372036854775806)), Reason: gptr.Of("not_found")}}, + wantJSON: `{"run_id":"9223372036854775807","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","accepted_count":0,"rejected_items":[{"item_id":"9223372036854775806","reason":"not_found"}],"snapshot_at_ms":"1793275200123"}`, + }, + { + name: "zero acceptance with empty rejection list", + rejected: []*exptpb.RetryRejectedItem{}, + wantJSON: `{"run_id":"9223372036854775807","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","accepted_count":0,"rejected_items":[],"snapshot_at_ms":"1793275200123"}`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + fakeApp := &fakeExperimentApp{retryResp: &exptpb.RetryExperimentResponse{ + RunID: gptr.Of(int64(9223372036854775807)), RequestID: gptr.Of("72cc8921-40f4-4a43-b772-aaf2b5dcdf05"), + AcceptedCount: gptr.Of(tc.accepted), RejectedItems: tc.rejected, SnapshotAtMs: gptr.Of(int64(1793275200123)), + }} + app := &EvalOpenAPIApplication{auth: auth, experimentApp: fakeApp, metric: &fakeOpenAPIMetric{}} + var req openapi.RetryExperimentOApiRequest + require.NoError(t, js_conv.GetUnmarshaler()([]byte(`{"workspace_id":"9","experiment_id":"7","retry_mode":"retry_target_items","item_ids":["9223372036854775806"],"expected_run_id":"9223372036854775807","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","ext":{"caller":"cli"}}`), &req)) + resp, err := app.RetryExperimentOApi(context.Background(), &req) + require.NoError(t, err) + require.NotNil(t, fakeApp.lastRetryReq) + assert.Equal(t, int64(9223372036854775807), fakeApp.lastRetryReq.GetExpectedRunID()) + assert.Equal(t, "72cc8921-40f4-4a43-b772-aaf2b5dcdf05", fakeApp.lastRetryReq.GetRequestID()) + assert.Equal(t, domainexpt.ExptRetryMode_RetryTargetItems, fakeApp.lastRetryReq.GetRetryMode()) + assert.Equal(t, []int64{9223372036854775806}, fakeApp.lastRetryReq.ItemIds) + assert.Equal(t, map[string]string{"caller": "cli"}, fakeApp.lastRetryReq.Ext) + body, err := js_conv.GetMarshaler()(resp.Data) + require.NoError(t, err) + assert.JSONEq(t, tc.wantJSON, string(body)) + }) + } +} + +func TestOpenAPIRunningRetryLegacyAndError(t *testing.T) { + for _, current := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy request", true: "current run error"}[current], func(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + fakeApp := &fakeExperimentApp{retryResp: &exptpb.RetryExperimentResponse{RunID: gptr.Of(int64(18))}} + input := `{"workspace_id":"9","experiment_id":"7"}` + if current { + input = `{"workspace_id":"9","experiment_id":"7","expected_run_id":"8","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05"}` + fakeApp.retryResp = nil + fakeApp.retryErr = errors.New("request fingerprint conflict") + } + var req openapi.RetryExperimentOApiRequest + require.NoError(t, js_conv.GetUnmarshaler()([]byte(input), &req)) + app := &EvalOpenAPIApplication{auth: auth, experimentApp: fakeApp, metric: &fakeOpenAPIMetric{}} + resp, err := app.RetryExperimentOApi(context.Background(), &req) + if current { + require.ErrorIs(t, err, fakeApp.retryErr) + require.Nil(t, resp) + assert.Equal(t, int64(8), fakeApp.lastRetryReq.GetExpectedRunID()) + assert.Equal(t, "72cc8921-40f4-4a43-b772-aaf2b5dcdf05", fakeApp.lastRetryReq.GetRequestID()) + return + } + require.NoError(t, err) + assert.Nil(t, fakeApp.lastRetryReq.ExpectedRunID) + assert.Nil(t, fakeApp.lastRetryReq.RequestID) + assert.Equal(t, domainexpt.ExptRetryMode_RetryFailure, fakeApp.lastRetryReq.GetRetryMode()) + assert.Equal(t, int64(18), resp.Data.GetRunID()) + }) + } +} + +func TestOpenAPIRunningRetryRealExperimentApplication(t *testing.T) { + for _, kind := range []string{"receipt", "manager error", "missing request_id", "zero expected_run_id"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, param *rpc.AuthorizationWithoutSPIParam) error { + assert.Equal(t, "7", param.ObjectID) + assert.Equal(t, int64(9), param.ResourceSpaceID) + assert.Equal(t, "owner", *param.OwnerID) + return nil + }) + manager.EXPECT().Get(gomock.Any(), int64(7), int64(9), gomock.Any()).Return(&entity.Experiment{ + ID: 7, SpaceID: 9, LatestRunID: 8, CreatedBy: "owner", Status: entity.ExptStatus_Processing, + }, nil) + input := `{"workspace_id":"9","experiment_id":"7","expected_run_id":"8","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","retry_mode":"retry_target_items","item_ids":["11","12"]}` + var req openapi.RetryExperimentOApiRequest + require.NoError(t, js_conv.GetUnmarshaler()([]byte(input), &req)) + failure := errors.New("current run mismatch") + switch kind { + case "missing request_id": + req.RequestID = nil + case "zero expected_run_id": + req.ExpectedRunID = gptr.Of(int64(0)) + default: + manager.EXPECT().RetryInCurrentRun(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RetryInCurrentRunRequest) (*entity.ManualRetryReceipt, error) { + assert.Equal(t, int64(8), r.ExpectedRunID) + assert.Equal(t, "72cc8921-40f4-4a43-b772-aaf2b5dcdf05", r.RequestID) + assert.Equal(t, entity.EvaluationModeRetryItems, r.Mode) + assert.Equal(t, []int64{11, 12}, r.ItemIDs) + if kind == "manager error" { + return nil, failure + } + return &entity.ManualRetryReceipt{ + RequestID: r.RequestID, AcceptedCount: 1, + RejectedItems: []*entity.ManualRetryRejectedItem{{ItemID: 12, Reason: "busy"}}, SnapshotAtMS: 1793275200123, + }, nil + }) + } + app := &EvalOpenAPIApplication{auth: auth, metric: &fakeOpenAPIMetric{}, experimentApp: &experimentApplication{manager: manager, auth: auth}} + resp, err := app.RetryExperimentOApi(context.Background(), &req) + if kind != "receipt" { + require.Error(t, err) + require.Nil(t, resp) + if kind == "manager error" { + assert.ErrorIs(t, err, failure) + } + return + } + require.NoError(t, err) + body, err := js_conv.GetMarshaler()(resp.Data) + require.NoError(t, err) + assert.JSONEq(t, `{"run_id":"8","request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","accepted_count":1,"rejected_items":[{"item_id":"12","reason":"busy"}],"snapshot_at_ms":"1793275200123"}`, string(body)) + }) + } +} diff --git a/backend/modules/evaluation/application/experiment_app.go b/backend/modules/evaluation/application/experiment_app.go index 21ec5ea01a..0803359fbb 100644 --- a/backend/modules/evaluation/application/experiment_app.go +++ b/backend/modules/evaluation/application/experiment_app.go @@ -1763,6 +1763,28 @@ func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.R return nil, err } + if req.ExpectedRunID != nil { + if req.GetExpectedRunID() <= 0 || req.GetRequestID() == "" { + return nil, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("invalid current run retry identity")) + } + receipt, err := e.manager.RetryInCurrentRun(ctx, &entity.RetryInCurrentRunRequest{SpaceID: req.GetWorkspaceID(), ExptID: req.GetExptID(), ExpectedRunID: req.GetExpectedRunID(), RequestID: req.GetRequestID(), Mode: runMode, ItemIDs: req.GetItemIds(), Ext: req.GetExt(), Session: session}) + if err != nil { + return nil, err + } + rejected := make([]*expt.RetryRejectedItem, 0, len(receipt.RejectedItems)) + for _, item := range receipt.RejectedItems { + rejected = append(rejected, &expt.RetryRejectedItem{ItemID: gptr.Of(item.ItemID), Reason: gptr.Of(item.Reason)}) + } + return &expt.RetryExperimentResponse{RunID: req.ExpectedRunID, RequestID: gptr.Of(receipt.RequestID), AcceptedCount: gptr.Of(receipt.AcceptedCount), RejectedItems: rejected, SnapshotAtMs: gptr.Of(receipt.SnapshotAtMS), BaseResp: base.NewBaseResp()}, nil + } + + // 只有「离线 + 运行中」才有同 run 重排（validateCurrentRetryExperiment 也只受理这一种）。 + // 其余状态（Pending / Finishing / 在线实验）若也强制身份，旧调用方会 400、带身份又 4102 run_finished， + // 两条路都不通；这里放过它们，保持开关打开前的旧重试语义。 + if got.ExptType == entity.ExptType_Offline && got.Status == entity.ExptStatus_Processing && e.configer != nil && e.configer.GetExptExecConf(ctx, req.GetWorkspaceID()).IsRunningRetryEnabled() { + return nil, errorx.NewByCode(errno.ManualRetryIdentityRequiredCode) + } + // SandboxAgent 评测对象：重试前重新初始化沙箱任务。 // 首次运行结束后每条 execute 已被 destroy，沙箱侧任务不再持有可用 execute，Retry 必须重跑一次 Init 才能继续 SandboxRun。 // 与 Submit 保持一致：并发度先归一化，再按实际 task 的 execution 数分别计算。 @@ -2137,6 +2159,10 @@ func (e *experimentApplication) KillExperiment(ctx context.Context, req *expt.Ki } kill := func(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { + if terminate, ok := e.manager.(service.IExptCurrentRunTerminator); ok && got.ExptType == entity.ExptType_Offline { + _, err := terminate.TerminateCurrentRun(ctx, exptID, exptRunID, spaceID, session, nil, entity.WithStatus(entity.ExptStatus_Terminated), entity.NoAggrCalculate()) + return err + } if err := e.manager.CompleteRun(ctx, exptID, exptRunID, spaceID, session, entity.WithStatus(entity.ExptStatus_Terminated)); err != nil { return err } diff --git a/backend/modules/evaluation/application/experiment_running_retry_mysql_test.go b/backend/modules/evaluation/application/experiment_running_retry_mysql_test.go new file mode 100644 index 0000000000..2ed7714a8a --- /dev/null +++ b/backend/modules/evaluation/application/experiment_running_retry_mysql_test.go @@ -0,0 +1,121 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package application + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + mysqlDriver "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/db" + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + lwtmocks "github.com/coze-dev/coze-loop/backend/infra/platestwrite/mocks" + dexpt "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/expt" + exptpb "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/expt" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + confmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + rpcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service" + exptrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + exptmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +func TestRunningRetryMySQLApplicationAdmission(t *testing.T) { + dsn := os.Getenv("EXPT_RETRY_MYSQL_DSN") + if dsn == "" { + t.Skip("EXPT_RETRY_MYSQL_DSN not configured") + } + provider, err := db.NewDB(mysqlDriver.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + conn := provider.NewSession(context.Background(), db.WithMaster()) + var identity struct{ Version, Schema string } + require.NoError(t, conn.Raw("SELECT VERSION() AS version, DATABASE() AS `schema`").Scan(&identity).Error) + require.Equal(t, "expt_running_retry_test", identity.Schema) + t.Logf("REAL_MYSQL_APPLICATION_EXECUTED version=%s schema=%s", identity.Version, identity.Schema) + base := int64(870000000000000000) + time.Now().UnixNano()%1000000000000 + space, expt, run, item := base+1, base+2, base+3, base+11 + t.Cleanup(func() { + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_item_ref", "expt_run_log", "expt_stats"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=?", space, expt).Error) + } + require.NoError(t, conn.Exec("DELETE FROM experiment WHERE space_id=? AND id=?", space, expt).Error) + sqlDB, err := conn.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + }) + require.NoError(t, conn.Exec("INSERT INTO experiment(id,space_id,latest_run_id,status,expt_type,created_by,eval_conf,eval_set_id,eval_set_version_id) VALUES(?,?,?,?,?,?,?,?,?)", expt, space, run, entity.ExptStatus_Processing, entity.ExptType_Offline, "original", `{"item_retry_num":3,"item_concur_num":4}`, base+80, base+81).Error) + raw, err := json.Marshal([]entity.ExptRunLogItems{{ItemIDs: []int64{item}}}) + require.NoError(t, err) + require.NoError(t, conn.Create(&model.ExptRunLog{ID: run, SpaceID: space, ExptID: expt, ExptRunID: run, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit)), CreatedBy: "original", ItemIds: &raw, FailCnt: 1}).Error) + require.NoError(t, conn.Create(&model.ExptStats{ID: base + 4, SpaceID: space, ExptID: expt, FailCnt: 1}).Error) + require.NoError(t, conn.Create(&model.ExptItemResult{ID: base + 12, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: base + 91, Status: 3, Ext: gptr.Of([]byte(`{"business":"keep"}`))}).Error) + require.NoError(t, conn.Create(&model.ExptItemResultRunLog{ID: base + 13, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: base + 91, Status: 3, ResultState: gptr.Of(int32(1)), RetryTimes: 3}).Error) + require.NoError(t, conn.Create(&model.ExptItemRef{ID: base + 14, SpaceID: space, ExptID: expt, ItemID: item, ItemVersionID: base + 91, ItemConfig: gptr.Of([]byte(`{}`))}).Error) + require.NoError(t, conn.Create(&model.ExptTurnResult{ID: base + 15, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: base + 91, TurnID: base + 21, Status: 2, TargetResultID: 123, WeightedScore: gptr.Of(0.0)}).Error) + require.NoError(t, conn.Create(&model.ExptTurnResultRunLog{ID: base + 16, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: base + 91, TurnID: base + 21, Status: 2, TargetResultID: 123, EvaluatorResultIds: gptr.Of([]byte(`null`)), Ext: []byte(`{"other":"keep"}`)}).Error) + ctrl := gomock.NewController(t) + config := confmocks.NewMockIConfiger(ctrl) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + generator := idmocks.NewMockIIDGenerator(ctrl) + lwt := lwtmocks.NewMockILatestWriteTracker(ctrl) + mutex := lockmocks.NewMockILocker(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + enabled := true + config.EXPECT().GetExptExecConf(gomock.Any(), space).DoAndReturn(func(context.Context, int64) *entity.ExptExecConf { + return &entity.ExptExecConf{RunningRetryEnabled: &enabled} + }).AnyTimes() + lwt.EXPECT().CheckWriteFlagByID(gomock.Any(), gomock.Any(), expt).Return(false).AnyTimes() + auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mutex.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), 5*time.Second, 5*time.Minute).DoAndReturn(func(ctx context.Context, _ string, _ time.Duration, _ time.Duration) (bool, context.Context, func(), error) { + c, cancel := context.WithCancel(ctx) + return true, c, cancel, nil + }).Times(2) + publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + manager := service.NewExptManager(nil, + exptrepo.NewExptRepo(exptmysql.NewExptDAO(provider), exptmysql.NewExptEvaluatorRefDAO(provider), generator), + exptrepo.NewExptRunLogRepo(exptmysql.NewExptRunLogDAO(provider)), exptrepo.NewExptStatsRepo(exptmysql.NewExptStatsDAO(provider)), exptrepo.NewExptItemResultRepo(exptmysql.NewExptItemResultDAO(provider)), exptrepo.NewExptItemRefRepo(exptmysql.NewExptItemRefDAO(provider)), exptrepo.NewExptTurnResultRepo(generator, exptmysql.NewExptTurnResultDAO(provider), exptmysql.NewExptTurnEvaluatorResultRefDAO(provider)), config, nil, mutex, nil, publisher, nil, generator, nil, lwt, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, component.NewNoopCentralSchedulerScopeProvider(), component.NewNoopCentralAdmissionPolicy(), component.NewNoopCentralReservationGuard(), exptrepo.NewExptManualRetryRepo(provider, generator), nil, component.NewNoopCurrentRunSchedulerWaker(), func() lock.ILocker { return mutex }) + app := &experimentApplication{manager: manager, auth: auth, idgen: generator, configer: config} + req := &exptpb.RetryExperimentRequest{WorkspaceID: &space, ExptID: &expt, ExpectedRunID: &run, RequestID: gptr.Of("72cc8921-40f4-4a43-b772-aaf2b5dcdf05"), RetryMode: gptr.Of(dexpt.ExptRetryMode_RetryTargetItems), ItemIds: []int64{item, item}, Ext: map[string]string{"caller": "identity"}} + response, err := app.RetryExperiment(context.Background(), req) + require.NoError(t, err) + require.Equal(t, int32(1), response.GetAcceptedCount()) + require.Equal(t, run, response.GetRunID()) + var canonical model.ExptItemResult + require.NoError(t, conn.First(&canonical, "id=?", base+12).Error) + require.Zero(t, canonical.Status) + ext, err := entity.ParseExptItemResultExt(*canonical.Ext) + require.NoError(t, err) + require.Equal(t, "keep", ext["business"]) + require.NotContains(t, ext, "caller") + manual, err := entity.ParseManualRetryContext(ext) + require.NoError(t, err) + require.Equal(t, req.GetRequestID(), manual.RequestID) + var log model.ExptItemResultRunLog + require.NoError(t, conn.First(&log, "id=?", base+13).Error) + require.Zero(t, *log.ResultState) + require.Equal(t, int32(3), log.RetryTimes) + enabled = false + require.NoError(t, conn.Model(&model.Experiment{}).Where("id=?", expt).Update("status", entity.ExptStatus_Success).Error) + again, err := app.RetryExperiment(context.Background(), req) + require.NoError(t, err) + require.Equal(t, response, again) + req.Ext = map[string]string{"caller": "changed"} + _, err = app.RetryExperiment(context.Background(), req) + require.ErrorContains(t, err, "request_conflict") + t.Log("REAL_MYSQL_APPLICATION_RECEIPT_VERIFIED same-run reset, budget preserved, ended/off replay, Ext conflict") +} diff --git a/backend/modules/evaluation/application/experiment_running_retry_test.go b/backend/modules/evaluation/application/experiment_running_retry_test.go new file mode 100644 index 0000000000..597d0eeb65 --- /dev/null +++ b/backend/modules/evaluation/application/experiment_running_retry_test.go @@ -0,0 +1,142 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package application + +import ( + "context" + "errors" + "strings" + "testing" + + domain_expt "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/expt" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + idgenmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + exptpb "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/expt" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc" + rpcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/errorx" +) + +func TestRunningRetryExplicitNeverFallsBack(t *testing.T) { + for _, kind := range []string{"zero_expected", "missing_request", "permission_denied"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + generator := idgenmocks.NewMockIIDGenerator(ctrl) + got := &entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, CreatedBy: "owner", Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline, EvalConf: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3)}} + manager.EXPECT().Get(gomock.Any(), int64(7), int64(9), gomock.Any()).Return(got, nil) + var denied error + if kind == "permission_denied" { + denied = errors.New("permission denied") + } + auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, p *rpc.AuthorizationWithoutSPIParam) error { + require.Equal(t, "7", p.ObjectID) + require.Equal(t, int64(9), p.ResourceSpaceID) + require.Equal(t, "owner", *p.OwnerID) + return denied + }) + req := &exptpb.RetryExperimentRequest{ExptID: gptr.Of(int64(7)), WorkspaceID: gptr.Of(int64(9)), ExpectedRunID: gptr.Of(int64(8))} + if kind == "zero_expected" { + req.ExpectedRunID = gptr.Of(int64(0)) + req.RequestID = gptr.Of("72cc8921-40f4-4a43-b772-aaf2b5dcdf05") + } + app := &experimentApplication{manager: manager, auth: auth, idgen: generator} + response, err := app.RetryExperiment(context.Background(), req) + require.Error(t, err) + require.Nil(t, response) + if denied != nil { + require.ErrorIs(t, err, denied) + } + }) + } +} + +func TestRunningRetryExplicitReceiptAndLegacySwitch(t *testing.T) { + for _, kind := range []string{"explicit_sandbox", "old_live_join", "old_terminal_failure", "old_terminal_all", "old_terminal_items", "enabled_missing_identity", "enabled_online_live", "enabled_pending_offline", "enabled_finishing_offline"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + generator := idgenmocks.NewMockIIDGenerator(ctrl) + sandbox := rpcmocks.NewMockISandboxSchedulerAdapter(ctrl) + got := &entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, CreatedBy: "owner", Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline, EvalConf: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3)}} + req := &exptpb.RetryExperimentRequest{WorkspaceID: gptr.Of(int64(9)), ExptID: gptr.Of(int64(7)), ItemIds: []int64{11}} + switch kind { + case "explicit_sandbox": + req.ExpectedRunID = gptr.Of(int64(8)) + req.RequestID = gptr.Of("72cc8921-40f4-4a43-b772-aaf2b5dcdf05") + req.Ext = map[string]string{"caller": "keep"} + got.Target = &entity.EvalTarget{EvalTargetType: entity.EvalTargetTypeSandboxAgent} + case "old_live_join", "old_terminal_items": + req.RetryMode = gptr.Of(domain_expt.ExptRetryMode_RetryTargetItems) + case "old_terminal_all": + req.RetryMode = gptr.Of(domain_expt.ExptRetryMode_RetryAll) + } + if strings.HasPrefix(kind, "old_terminal") { + got.Status = entity.ExptStatus_Success + } + switch kind { + case "enabled_online_live": + got.ExptType = entity.ExptType_Online + case "enabled_pending_offline": + got.Status = entity.ExptStatus_Pending + case "enabled_finishing_offline": + got.Status = entity.ExptStatus_Terminating + } + manager.EXPECT().Get(gomock.Any(), int64(7), int64(9), gomock.Any()).Return(got, nil) + auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).Return(nil) + switch kind { + case "explicit_sandbox": + manager.EXPECT().RetryInCurrentRun(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RetryInCurrentRunRequest) (*entity.ManualRetryReceipt, error) { + require.Equal(t, int64(8), r.ExpectedRunID) + require.Equal(t, map[string]string{"caller": "keep"}, r.Ext) + return &entity.ManualRetryReceipt{RequestID: r.RequestID, AcceptedCount: 1, RejectedItems: []*entity.ManualRetryRejectedItem{{ItemID: 12, Reason: "busy"}}, SnapshotAtMS: 1700000000123}, nil + }) + case "enabled_missing_identity": + config.EXPECT().GetExptExecConf(gomock.Any(), int64(9)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}) + case "old_live_join": + config.EXPECT().GetExptExecConf(gomock.Any(), int64(9)).Return(&entity.ExptExecConf{}) + manager.EXPECT().LogRetryItemsRun(gomock.Any(), int64(7), entity.EvaluationModeRetryItems, int64(9), []int64{11}, gomock.Any()).Return(int64(8), true, nil) + case "old_terminal_items": + manager.EXPECT().LogRetryItemsRun(gomock.Any(), int64(7), entity.EvaluationModeRetryItems, int64(9), []int64{11}, gomock.Any()).Return(int64(18), false, nil) + manager.EXPECT().RetryItems(gomock.Any(), int64(7), int64(18), int64(9), 3, []int64{11}, gomock.Any(), gomock.Any()).Return(nil) + default: + mode := entity.EvaluationModeFailRetry + if kind == "old_terminal_all" { + mode = entity.EvaluationModeRetryAll + } + generator.EXPECT().GenID(gomock.Any()).Return(int64(18), nil) + manager.EXPECT().LogRun(gomock.Any(), int64(7), int64(18), mode, int64(9), gomock.Any(), gomock.Any()).Return(nil) + manager.EXPECT().Run(gomock.Any(), int64(7), int64(18), int64(9), 3, gomock.Any(), mode, gomock.Any()).Return(nil) + } + app := &experimentApplication{manager: manager, auth: auth, idgen: generator, configer: config, sandboxSchedulerAdapter: sandbox} + resp, err := app.RetryExperiment(context.Background(), req) + if kind == "enabled_missing_identity" { + statusErr, ok := errorx.FromStatusError(err) + require.True(t, ok) + require.EqualValues(t, errno.ManualRetryIdentityRequiredCode, statusErr.Code()) + require.Nil(t, resp) + return + } + require.NoError(t, err) + if kind == "explicit_sandbox" { + require.Equal(t, int64(8), resp.GetRunID()) + require.Equal(t, req.GetRequestID(), resp.GetRequestID()) + require.Equal(t, int32(1), resp.GetAcceptedCount()) + require.Equal(t, int64(1700000000123), resp.GetSnapshotAtMs()) + require.Equal(t, "busy", resp.RejectedItems[0].GetReason()) + } + }) + } +} diff --git a/backend/modules/evaluation/application/standard_eval_output_test.go b/backend/modules/evaluation/application/standard_eval_output_test.go index 9caf85ca69..308193bb6b 100644 --- a/backend/modules/evaluation/application/standard_eval_output_test.go +++ b/backend/modules/evaluation/application/standard_eval_output_test.go @@ -5,6 +5,7 @@ package application import ( "context" + standardjson "encoding/json" "errors" "testing" "time" @@ -1080,3 +1081,41 @@ func TestBuildItemStandardEvalOutput_FornaxSandboxLogURL(t *testing.T) { assert.Empty(t, sandboxLogURLFromItem(nil, 20)) }) } + +func TestExperimentApplication_StandardOutputIgnoresRemovedSourceProtocol(t *testing.T) { + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + results := servicemocks.NewMockExptResultService(ctrl) + manager := servicemocks.NewMockIExptManager(ctrl) + target := servicemocks.NewMockIEvalTargetService(ctrl) + app := &experimentApplication{auth: auth, resultSvc: results, manager: manager, evalTargetService: target} + target.EXPECT().GetEvalTarget(gomock.Any(), int64(200)).Return(&entity.EvalTarget{ID: 200, SpaceID: 1, SourceTargetID: "src-200"}, nil).Times(2) + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil).Times(2) + manager.EXPECT().GetDetail(gomock.Any(), int64(2), int64(1), gomock.Any()).Return(makeStandardEvalOutputExpt(2, 1), nil).Times(2) + results.EXPECT().MGetExperimentResult(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, param *entity.MGetExperimentResultParam) (*entity.MGetExperimentReportResult, error) { + require.Equal(t, int64(1), param.SpaceID) + require.Equal(t, []int64{2}, param.ExptIDs) + require.Equal(t, []int64{4}, param.ItemIDs) + require.False(t, param.UseAccelerator) + require.True(t, param.FullTrajectory) + require.True(t, gptr.Indirect(param.LoadEvaluatorFullContent)) + require.True(t, gptr.Indirect(param.LoadEvalTargetFullContent)) + return makeStandardEvalOutputReportResult(2, 3, 4, 5, 0), nil + }).Times(2) + var original, extra exptpb.MGetExperimentStandardEvalOutputsRequest + require.NoError(t, json.Unmarshal([]byte(`{"workspace_id":1,"expt_id":2,"item_ids":[4]}`), &original)) + require.NoError(t, json.Unmarshal([]byte(`{"workspace_id":1,"expt_id":2,"item_ids":[4],"expected_expt_run_id":99,"expected_request_id":"wrong-request","expected_source_turn_records":[{"turn_id":"0","target_record_id":"99"}],"source_only":true}`), &extra)) + require.Equal(t, original, extra, "removed source fields cannot select a different standard result path") + expected, err := app.MGetExperimentStandardEvalOutputs(context.Background(), &original) + require.NoError(t, err) + actual, err := app.MGetExperimentStandardEvalOutputs(context.Background(), &extra) + require.NoError(t, err) + require.Equal(t, expected, actual) + require.Len(t, actual.Items, 1) + body, err := json.Marshal(actual.Items[0]) + require.NoError(t, err) + var payload map[string]standardjson.RawMessage + require.NoError(t, json.Unmarshal(body, &payload)) + require.NotContains(t, payload, "request_id") + require.NotContains(t, payload, "source_turn_records") +} diff --git a/backend/modules/evaluation/application/wire.go b/backend/modules/evaluation/application/wire.go index 2cdbebc296..c52c6b230e 100644 --- a/backend/modules/evaluation/application/wire.go +++ b/backend/modules/evaluation/application/wire.go @@ -81,6 +81,7 @@ var ( notifyrpc.NotifyRPCSet, userinfo.NewUserInfoServiceImpl, NewLock, + NewManualRetryLockerFactory, flagSet, domainservice.NewDefaultURLProcessor, storage.StorageSet, @@ -92,8 +93,9 @@ var ( domainservice.EvaluatorDomainServiceSet, domainservice.EvaluationSetDomainServiceSet, domainservice.TargetDomainServiceSet, - domainservice.NewExptResultService, + domainservice.NewExptResultServiceWithPublication, domainservice.ProvideExptItemRefRepos, + NewManualRetryLockerFactory, domainservice.NewEvaluationAnalysisService, // Infrastructure Sets infrahttp.NewHTTPClient, diff --git a/backend/modules/evaluation/application/wire_gen.go b/backend/modules/evaluation/application/wire_gen.go index 45238d462a..a88cf62d7f 100644 --- a/backend/modules/evaluation/application/wire_gen.go +++ b/backend/modules/evaluation/application/wire_gen.go @@ -157,10 +157,12 @@ func InitExperimentApplication(ctx context.Context, idgen2 idgen.IIDGenerator, d evaluationSetItemService := service.NewEvaluationSetItemServiceImpl(iDatasetRPCAdapter) iEvaluationAnalysisService := service.NewEvaluationAnalysisService() iFileProvider := foundation.NewFileRPCProvider(fileClient) + iExptResultPublishRepo := experiment.NewExptResultPublishRepo(db2) + manualRetryLockerFactory := NewManualRetryLockerFactory(cmdable) iExptItemRefDAO := mysql.NewExptItemRefDAO(db2) iExptItemRefRepo := experiment.NewExptItemRefRepo(iExptItemRefDAO) v3 := service.ProvideExptItemRefRepos(iExptItemRefRepo) - exptResultService := service.NewExptResultService(iExptItemResultRepo, iExptTurnResultRepo, iExptAnnotateRepo, iExptStatsRepo, iExperimentRepo, exptMetric, iLatestWriteTracker, idgen2, iExptTurnResultFilterRepo, serviceEvaluatorService, iEvalTargetService, iEvalTargetRepo, evaluationSetVersionService, iEvaluationSetService, evaluatorRecordService, evaluationSetItemService, exptEventPublisher, iTagRPCAdapter, iEvaluationAnalysisService, iFileProvider, iEvaluatorScoreCalculator, v3...) + exptResultService := service.NewExptResultServiceWithPublication(iExptItemResultRepo, iExptTurnResultRepo, iExptAnnotateRepo, iExptStatsRepo, iExperimentRepo, exptMetric, iLatestWriteTracker, idgen2, iExptTurnResultFilterRepo, serviceEvaluatorService, iEvalTargetService, iEvalTargetRepo, evaluationSetVersionService, iEvaluationSetService, evaluatorRecordService, evaluationSetItemService, exptEventPublisher, iTagRPCAdapter, iEvaluationAnalysisService, iFileProvider, iEvaluatorScoreCalculator, iExptResultPublishRepo, manualRetryLockerFactory, v3...) iQuotaDAO := dao.NewQuotaDAO(cmdable) quotaRepo := experiment.NewQuotaService(iQuotaDAO, iLocker) iExptTemplateDAO := mysql.NewExptTemplateDAO(db2) @@ -179,7 +181,10 @@ func InitExperimentApplication(ctx context.Context, idgen2 idgen.IIDGenerator, d iCentralSchedulerScopeProvider := component.NewNoopCentralSchedulerScopeProvider() iCentralAdmissionPolicy := component.NewNoopCentralAdmissionPolicy() iCentralReservationGuard := component.NewNoopCentralReservationGuard() - iExptManager := service.NewExptManager(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptItemRefRepo, iExptTurnResultRepo, componentIConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, serviceEvaluatorService, benefitSvc, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider, pipelineListAdapter, resourceAccessAuthorizer, sandboxAgentMetrics, iCentralSchedulerScopeProvider, iCentralAdmissionPolicy, iCentralReservationGuard) + iExptManualRetryRepo := experiment.NewExptManualRetryRepo(db2, idgen2) + iCurrentRunSchedulerWaker := component.NewNoopCurrentRunSchedulerWaker() + iExptRunFinishRepo := experiment.NewExptRunFinishRepo(db2) + iExptManager := service.NewExptManagerWithRunFinish(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptItemRefRepo, iExptTurnResultRepo, componentIConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, serviceEvaluatorService, benefitSvc, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider, pipelineListAdapter, resourceAccessAuthorizer, sandboxAgentMetrics, iCentralSchedulerScopeProvider, iCentralAdmissionPolicy, iCentralReservationGuard, iExptManualRetryRepo, evaluatorRecordService, iCurrentRunSchedulerWaker, manualRetryLockerFactory, iExptRunFinishRepo) v4 := service.ProvideNoSandboxAgentNotifiers() schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, iExptItemRefRepo, idempotentService, componentIConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager, iExptRunLogRepo, iLocker, v4...) iItemCompletePublisher := service.ProvideNilItemCompletePublisher() @@ -290,10 +295,12 @@ func InitEvaluatorApplication(ctx context.Context, idgen2 idgen.IIDGenerator, au evaluationSetItemService := service.NewEvaluationSetItemServiceImpl(iDatasetRPCAdapter) iTagRPCAdapter := tag.NewTagRPCProvider(tagClient) iEvaluationAnalysisService := service.NewEvaluationAnalysisService() + iExptResultPublishRepo := experiment.NewExptResultPublishRepo(db2) + manualRetryLockerFactory := NewManualRetryLockerFactory(cmdable) iExptItemRefDAO := mysql.NewExptItemRefDAO(db2) iExptItemRefRepo := experiment.NewExptItemRefRepo(iExptItemRefDAO) v3 := service.ProvideExptItemRefRepos(iExptItemRefRepo) - exptResultService := service.NewExptResultService(iExptItemResultRepo, iExptTurnResultRepo, iExptAnnotateRepo, iExptStatsRepo, iExperimentRepo, exptMetric, iLatestWriteTracker, idgen2, iExptTurnResultFilterRepo, evaluatorService, iEvalTargetService, iEvalTargetRepo, evaluationSetVersionService, iEvaluationSetService, evaluatorRecordService, evaluationSetItemService, exptEventPublisher, iTagRPCAdapter, iEvaluationAnalysisService, iFileProvider, iEvaluatorScoreCalculator, v3...) + exptResultService := service.NewExptResultServiceWithPublication(iExptItemResultRepo, iExptTurnResultRepo, iExptAnnotateRepo, iExptStatsRepo, iExperimentRepo, exptMetric, iLatestWriteTracker, idgen2, iExptTurnResultFilterRepo, evaluatorService, iEvalTargetService, iEvalTargetRepo, evaluationSetVersionService, iEvaluationSetService, evaluatorRecordService, evaluationSetItemService, exptEventPublisher, iTagRPCAdapter, iEvaluationAnalysisService, iFileProvider, iEvaluatorScoreCalculator, iExptResultPublishRepo, manualRetryLockerFactory, v3...) evaluationEvaluatorService := NewEvaluatorHandlerImpl(idgen2, iConfiger, iAuthProvider, evaluatorService, evaluatorRecordService, evaluatorTemplateService, evaluatorExecMetrics, userInfoService, auditClient, benefitSvc, iFileProvider, v, exptResultService, iEvalAsyncRepo) return evaluationEvaluatorService, nil } @@ -348,7 +355,7 @@ func InitEvalTargetApplication(ctx context.Context, idgen2 idgen.IIDGenerator, d return evalTargetService, nil } -func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigLoaderFactory, rmqFactory mq.IFactory, cmdable redis.Cmdable, idgen2 idgen.IIDGenerator, db2 db.Provider, client promptmanageservice.Client, executeClient promptexecuteservice.Client, authClient authservice.Client, meter metrics.Meter, dataClient datasetservice.Client, userClient userservice.Client, llmClient llmruntimeservice.Client, tagClient tagservice.Client, limiterFactory limiter.IRateLimiterFactory, objectStorage fileserver.ObjectStorage, batchObjectStorage fileserver.BatchObjectStorage, auditClient audit.IAuditService, benefitService benefit.IBenefitService, ckProvider ck.Provider, plainLimiterFactory limiter.IPlainRateLimiterFactory, trajectoryAdapter rpc.ITrajectoryAdapter, fileClient fileservice.Client, taskClientFactory func() taskservice.Client, scheduleAdapter rpc.IExptScheduleAdapter) (evaluation.EvalOpenAPIService, error) { +func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigLoaderFactory, rmqFactory mq.IFactory, cmdable redis.Cmdable, idgen2 idgen.IIDGenerator, db2 db.Provider, client promptmanageservice.Client, executeClient promptexecuteservice.Client, authClient authservice.Client, meter metrics.Meter, dataClient datasetservice.Client, userClient userservice.Client, llmClient llmruntimeservice.Client, tagClient tagservice.Client, limiterFactory limiter.IRateLimiterFactory, objectStorage fileserver.ObjectStorage, batchObjectStorage fileserver.BatchObjectStorage, auditClient audit.IAuditService, benefitService benefit.IBenefitService, ckProvider ck.Provider, plainLimiterFactory limiter.IPlainRateLimiterFactory, trajectoryAdapter rpc.ITrajectoryAdapter, fileClient fileservice.Client, taskClientFactory func() taskservice.Client, scheduleAdapter rpc.IExptScheduleAdapter) (IEvalOpenAPIApplication, error) { iConfiger, err := conf2.NewConfiger(configFactory) if err != nil { return nil, err @@ -435,10 +442,12 @@ func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigL iExptTurnResultFilterRepo := experiment.NewExptTurnResultFilterRepo(iExptTurnResultFilterDAO, iExptTurnResultFilterKeyMappingDAO) iEvaluationAnalysisService := service.NewEvaluationAnalysisService() iFileProvider := foundation.NewFileRPCProvider(fileClient) + iExptResultPublishRepo := experiment.NewExptResultPublishRepo(db2) + manualRetryLockerFactory := NewManualRetryLockerFactory(cmdable) iExptItemRefDAO := mysql.NewExptItemRefDAO(db2) iExptItemRefRepo := experiment.NewExptItemRefRepo(iExptItemRefDAO) v3 := service.ProvideExptItemRefRepos(iExptItemRefRepo) - exptResultService := service.NewExptResultService(iExptItemResultRepo, iExptTurnResultRepo, iExptAnnotateRepo, iExptStatsRepo, iExperimentRepo, exptMetric, iLatestWriteTracker, idgen2, iExptTurnResultFilterRepo, evaluatorService, iEvalTargetService, iEvalTargetRepo, evaluationSetVersionService, iEvaluationSetService, evaluatorRecordService, evaluationSetItemService, exptEventPublisher, iTagRPCAdapter, iEvaluationAnalysisService, iFileProvider, iEvaluatorScoreCalculator, v3...) + exptResultService := service.NewExptResultServiceWithPublication(iExptItemResultRepo, iExptTurnResultRepo, iExptAnnotateRepo, iExptStatsRepo, iExperimentRepo, exptMetric, iLatestWriteTracker, idgen2, iExptTurnResultFilterRepo, evaluatorService, iEvalTargetService, iEvalTargetRepo, evaluationSetVersionService, iEvaluationSetService, evaluatorRecordService, evaluationSetItemService, exptEventPublisher, iTagRPCAdapter, iEvaluationAnalysisService, iFileProvider, iEvaluatorScoreCalculator, iExptResultPublishRepo, manualRetryLockerFactory, v3...) iQuotaDAO := dao.NewQuotaDAO(cmdable) quotaRepo := experiment.NewQuotaService(iQuotaDAO, iLocker) iExptTemplateDAO := mysql.NewExptTemplateDAO(db2) @@ -456,7 +465,10 @@ func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigL iCentralSchedulerScopeProvider := component.NewNoopCentralSchedulerScopeProvider() iCentralAdmissionPolicy := component.NewNoopCentralAdmissionPolicy() iCentralReservationGuard := component.NewNoopCentralReservationGuard() - iExptManager := service.NewExptManager(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptItemRefRepo, iExptTurnResultRepo, iConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, evaluatorService, benefitService, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider, pipelineListAdapter, resourceAccessAuthorizer, sandboxAgentMetrics, iCentralSchedulerScopeProvider, iCentralAdmissionPolicy, iCentralReservationGuard) + iExptManualRetryRepo := experiment.NewExptManualRetryRepo(db2, idgen2) + iCurrentRunSchedulerWaker := component.NewNoopCurrentRunSchedulerWaker() + iExptRunFinishRepo := experiment.NewExptRunFinishRepo(db2) + iExptManager := service.NewExptManagerWithRunFinish(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptItemRefRepo, iExptTurnResultRepo, iConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, evaluatorService, benefitService, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider, pipelineListAdapter, resourceAccessAuthorizer, sandboxAgentMetrics, iCentralSchedulerScopeProvider, iCentralAdmissionPolicy, iCentralReservationGuard, iExptManualRetryRepo, evaluatorRecordService, iCurrentRunSchedulerWaker, manualRetryLockerFactory, iExptRunFinishRepo) v4 := service.ProvideNoSandboxAgentNotifiers() schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, iExptItemRefRepo, idempotentService, iConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager, iExptRunLogRepo, iLocker, v4...) iItemCompletePublisher := service.ProvideNilItemCompletePublisher() @@ -481,8 +493,8 @@ func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigL exptLifecycleEventHandler := service.NewExptLifecycleEventHandler(iExperimentRepo, iNotifyRPCAdapter, iUserProvider, webhookDispatcher) iExperimentApplication := NewExperimentApplication(exptAggrResultService, exptResultService, iExptManager, exptSchedulerEvent, exptItemEvalEvent, idgen2, iConfiger, iAuthProvider, userInfoService, iEvalTargetService, evaluationSetItemService, iExptAnnotateService, iTagRPCAdapter, iExptResultExportService, iExptInsightAnalysisService, evaluatorService, iExptTemplateManager, iFileProvider, exptLifecycleEventHandler, sandboxSchedulerAdapter, sandboxAgentMetrics) evaluatorCallbackDispatcher := service.NewEvaluatorCallbackDispatcher(noopWebhookSecretProvider) - evalOpenAPIService := NewEvalOpenAPIApplication(iEvalAsyncRepo, exptEventPublisher, iEvalTargetService, iEvalTargetRepo, iAuthProvider, iEvaluationSetService, evaluationSetVersionService, evaluationSetItemService, evaluationSetSchemaService, openAPIEvaluationMetrics, sandboxAgentMetrics, userInfoService, iExperimentApplication, iExptManager, exptResultService, exptAggrResultService, evaluatorService, evaluatorRecordService, iExptTemplateManager, iConfiger, sandboxSchedulerAdapter, iFileProvider, evaluatorCallbackDispatcher, resourceAccessAuthorizer) - return evalOpenAPIService, nil + v5 := NewEvalOpenAPIApplication(iEvalAsyncRepo, exptEventPublisher, iEvalTargetService, iEvalTargetRepo, iAuthProvider, iEvaluationSetService, evaluationSetVersionService, evaluationSetItemService, evaluationSetSchemaService, openAPIEvaluationMetrics, sandboxAgentMetrics, userInfoService, iExperimentApplication, iExptManager, exptResultService, exptAggrResultService, evaluatorService, evaluatorRecordService, iExptTemplateManager, iConfiger, sandboxSchedulerAdapter, iFileProvider, evaluatorCallbackDispatcher, resourceAccessAuthorizer) + return v5, nil } // wire.go: @@ -492,11 +504,12 @@ var ( experimentSet = wire.NewSet( NewExperimentApplication, service.ExperimentDomainServiceSet, service.EvaluationSetDomainServiceSet, service.TargetDomainServiceSet, service.EvaluatorDomainServiceSet, conf2.NewSharedResourceConfigProvider, service.NewResourceAccessAuthorizer, metrics2.ExperimentMetricsSet, metrics3.EvalTargetMetricsSet, sandbox_agent.SandboxAgentMetricsSet, http.NewHTTPClient, foundation.FoundationRPCSet, tag.TagRPCSet, agent.AgentRPCSet, notify.NotifyRPCSet, userinfo.NewUserInfoServiceImpl, NewLock, + NewManualRetryLockerFactory, flagSet, service.NewDefaultURLProcessor, storage.StorageSet, ) evaluatorSet = wire.NewSet( - NewEvaluatorHandlerImpl, service.EvaluatorDomainServiceSet, service.EvaluationSetDomainServiceSet, service.TargetDomainServiceSet, service.NewExptResultService, service.ProvideExptItemRefRepos, service.NewEvaluationAnalysisService, http.NewHTTPClient, foundation.FoundationRPCSet, tag.TagRPCSet, trajectory.TrajectoryRPCSet, userinfo.NewUserInfoServiceImpl, experiment.ExperimentRepoSet, metrics2.ExperimentMetricsSet, metrics3.EvalTargetMetricsSet, sandbox_agent.SandboxAgentMetricsSet, conf2.NewConfiger, flagSet, storage.StorageSet, + NewEvaluatorHandlerImpl, service.EvaluatorDomainServiceSet, service.EvaluationSetDomainServiceSet, service.TargetDomainServiceSet, service.NewExptResultServiceWithPublication, service.ProvideExptItemRefRepos, NewManualRetryLockerFactory, service.NewEvaluationAnalysisService, http.NewHTTPClient, foundation.FoundationRPCSet, tag.TagRPCSet, trajectory.TrajectoryRPCSet, userinfo.NewUserInfoServiceImpl, experiment.ExperimentRepoSet, metrics2.ExperimentMetricsSet, metrics3.EvalTargetMetricsSet, sandbox_agent.SandboxAgentMetricsSet, conf2.NewConfiger, flagSet, storage.StorageSet, ) evaluationSetSet = wire.NewSet( diff --git a/backend/modules/evaluation/domain/component/central_reservation_guard.go b/backend/modules/evaluation/domain/component/central_reservation_guard.go index 555f27acaa..7976a95e66 100644 --- a/backend/modules/evaluation/domain/component/central_reservation_guard.go +++ b/backend/modules/evaluation/domain/component/central_reservation_guard.go @@ -5,6 +5,7 @@ package component import ( "context" + "fmt" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" ) @@ -21,6 +22,7 @@ import ( // reservation 就执行，那么迟到的、重复的、账本已重建过的消息都会变成"无额度执行"，全局额度 // 保护即失效。 type ICentralReservationGuard interface { + ReleaseTerminalReservation(ctx context.Context, schedulerScope string, previousRunID, itemID int64, previousRequestID string) (TerminalReservationReleaseResult, error) // ConfirmRunning 确认该 item 持有额度，取得执行资格。 // // 注意它是**幂等**的而非"一次性"：实现只校验 reservation 存在、并把状态推进到 Running， @@ -34,11 +36,13 @@ type ICentralReservationGuard interface { // schedulerScope 是 Experiment 冻结的调度域，决定去哪本额度账本查这条 reservation。 // 由调用方从 DB 读出后传入，不由实现方按当前运行环境推断 —— 推断会让"实验属于哪本账" // 取决于谁在处理消息，而它本该只取决于数据本身。 - ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64) (bool, error) + // expectedRequestID is the original event/grant token; empty is never a wildcard. + ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID string) (bool, error) // Release 在 item 进入终态（成功/失败/终止/僵尸清理）时幂等释放额度。 // 重复调用为 no-op。 - Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, reason string) error + // Compare the original expectedRequestID atomically before refunding/deleting. + Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID, reason string) error // RepriceRunConsumption 把该 run 在飞的预占按新的单 item 消耗向量重新计价。 // @@ -93,13 +97,13 @@ type noopCentralReservationGuard struct{} // 选择 fail-closed 而非放行：本方法只会被判定为 enforce 的实验触达，而 enforce 意味着 // "该实验的额度由中心账本管控"。没有账本却放行，等于让实验在无任何额度约束下跑， // 这比让它停下来等待更危险 —— 后者可见（实验不动会被发现），前者静默（资源被打爆才发现）。 -func (noopCentralReservationGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64) (bool, error) { +func (noopCentralReservationGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID string) (bool, error) { return false, nil } // Release 无账本可释放，直接成功返回。 // 不返回错误：终态收口路径不应因为额度模块缺席而失败。 -func (noopCentralReservationGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, reason string) error { +func (noopCentralReservationGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID, reason string) error { return nil } @@ -251,3 +255,29 @@ type noopCentralAdmissionPolicy struct{} func (noopCentralAdmissionPolicy) AllowCentralScheduling(ctx context.Context, subject CentralAdmissionSubject) (CentralAdmissionDecision, error) { return CentralAdmissionDecision{Admitted: true}, nil } + +type TerminalReservationReleaseResult int + +const ( + TerminalReservationReleased TerminalReservationReleaseResult = iota + 1 + TerminalReservationAbsent + TerminalReservationMismatch +) + +func (noopCentralReservationGuard) ReleaseTerminalReservation(context.Context, string, int64, int64, string) (TerminalReservationReleaseResult, error) { + return 0, fmt.Errorf("terminal reservation release unsupported without token ledger") +} + +type ICurrentRunSchedulerWaker interface { + WakeCurrentRun(context.Context, string) error +} + +type noopCurrentRunSchedulerWaker struct{} + +func NewNoopCurrentRunSchedulerWaker() ICurrentRunSchedulerWaker { + return noopCurrentRunSchedulerWaker{} +} + +func (noopCurrentRunSchedulerWaker) WakeCurrentRun(context.Context, string) error { + return fmt.Errorf("central current run wake unsupported") +} diff --git a/backend/modules/evaluation/domain/component/central_reservation_guard_test.go b/backend/modules/evaluation/domain/component/central_reservation_guard_test.go index 2838de5a9f..c7e55bd482 100644 --- a/backend/modules/evaluation/domain/component/central_reservation_guard_test.go +++ b/backend/modules/evaluation/domain/component/central_reservation_guard_test.go @@ -18,7 +18,7 @@ func TestNoopCentralReservationGuard_ConfirmRunningFailsClosed(t *testing.T) { // 关键语义：无账本环境下一律拒绝执行。 // 放行会让 enforce 实验在零额度约束下跑（静默、事后才发现）；拒绝则可见（实验不动会被察觉）。 - ok, err := guard.ConfirmRunning(context.Background(), "fornax_cn_prod", 1, 2) + ok, err := guard.ConfirmRunning(context.Background(), "fornax_cn_prod", 1, 2, "") require.NoError(t, err) assert.False(t, ok, "noop guard 必须 fail-closed，不得放行 enforce item") } @@ -29,8 +29,8 @@ func TestNoopCentralReservationGuard_ReleaseNeverErrors(t *testing.T) { guard := NewNoopCentralReservationGuard() // 终态收口路径不应因为额度模块缺席而失败 - require.NoError(t, guard.Release(context.Background(), "fornax_cn_prod", 1, 2, "item success")) - require.NoError(t, guard.Release(context.Background(), "fornax_cn_prod", 1, 2, "duplicate terminal event")) + require.NoError(t, guard.Release(context.Background(), "fornax_cn_prod", 1, 2, "", "item success")) + require.NoError(t, guard.Release(context.Background(), "fornax_cn_prod", 1, 2, "", "duplicate terminal event")) } func TestNoopCentralSchedulerScopeOwner_AlwaysOwns(t *testing.T) { diff --git a/backend/modules/evaluation/domain/component/item_complete_publisher_test.go b/backend/modules/evaluation/domain/component/item_complete_publisher_test.go new file mode 100644 index 0000000000..6b20bb04a8 --- /dev/null +++ b/backend/modules/evaluation/domain/component/item_complete_publisher_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package component + +import ( + "testing" + + "github.com/coze-dev/coze-loop/backend/pkg/json" + "github.com/stretchr/testify/require" +) + +// This is the original consumer payload, with decimal IDs above JSON's exact integer range. +const originalItemCompletePayload = `{"eval_target_workspace_id":"7384603622000000001","eval_target_id":"7384603622000000002","source_target_id":"application","expt_workspace_id":"7384603622000000003","expt_id":"7384603622000000004","expt_run_id":"7384603622000000005","dataset_workspace_id":"7384603622000000006","dataset_id":"7384603622000000007","dataset_version_id":"7384603622000000008","dataset_key":"dataset","dataset_version_name":"0.0.1","experiment_group_key":"group","item_id":"7384603622000000009","item_key":"case","enable_analysis":true,"created_by":"user"}` + +func TestItemCompleteOriginalPayloadRoundTrip(t *testing.T) { + var event ItemCompleteEvent + require.NoError(t, json.Unmarshal([]byte(originalItemCompletePayload), &event)) + body, err := json.Marshal(&event) + require.NoError(t, err) + require.Equal(t, originalItemCompletePayload, string(body)) + // Repeated delivery preserves the existing consumer identity and exact payload. + var duplicate ItemCompleteEvent + require.NoError(t, json.Unmarshal(body, &duplicate)) + repeated, err := json.Marshal(&duplicate) + require.NoError(t, err) + require.Equal(t, body, repeated) +} + +func TestItemCompleteExtraProtocolDoesNotPollutePayload(t *testing.T) { + for _, request := range []string{`""`, `"manual-retry"`} { + extra := originalItemCompletePayload[:len(originalItemCompletePayload)-1] + `,"request_id":` + request + `,"source_turn_records":[{"turn_id":"0","target_record_id":"7384603622000000010","registered":[{"version_id":"11","alias":"a","record_id":"12"}],"inline":[{"inline_key":"key","record_id":"13"}]}]}` + var event ItemCompleteEvent + require.NoError(t, json.Unmarshal([]byte(extra), &event)) + body, err := json.Marshal(&event) + require.NoError(t, err) + require.Equal(t, originalItemCompletePayload, string(body), "extra source protocol must not enter the original notification") + } +} diff --git a/backend/modules/evaluation/domain/component/mocks/central_reservation_guard.go b/backend/modules/evaluation/domain/component/mocks/central_reservation_guard.go index 1d001a7912..087b691411 100644 --- a/backend/modules/evaluation/domain/component/mocks/central_reservation_guard.go +++ b/backend/modules/evaluation/domain/component/mocks/central_reservation_guard.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -destination=mocks/central_reservation_guard.go -package=mocks github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component ICentralReservationGuard +// mockgen -destination=mocks/central_reservation_guard.go -package=mocks . ICentralReservationGuard // // Package mocks is a generated GoMock package. @@ -13,6 +13,7 @@ import ( context "context" reflect "reflect" + component "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" gomock "go.uber.org/mock/gomock" ) @@ -42,32 +43,47 @@ func (m *MockICentralReservationGuard) EXPECT() *MockICentralReservationGuardMoc } // ConfirmRunning mocks base method. -func (m *MockICentralReservationGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64) (bool, error) { +func (m *MockICentralReservationGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ConfirmRunning", ctx, schedulerScope, exptRunID, itemID) + ret := m.ctrl.Call(m, "ConfirmRunning", ctx, schedulerScope, exptRunID, itemID, expectedRequestID) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // ConfirmRunning indicates an expected call of ConfirmRunning. -func (mr *MockICentralReservationGuardMockRecorder) ConfirmRunning(ctx, schedulerScope, exptRunID, itemID any) *gomock.Call { +func (mr *MockICentralReservationGuardMockRecorder) ConfirmRunning(ctx, schedulerScope, exptRunID, itemID, expectedRequestID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfirmRunning", reflect.TypeOf((*MockICentralReservationGuard)(nil).ConfirmRunning), ctx, schedulerScope, exptRunID, itemID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfirmRunning", reflect.TypeOf((*MockICentralReservationGuard)(nil).ConfirmRunning), ctx, schedulerScope, exptRunID, itemID, expectedRequestID) } // Release mocks base method. -func (m *MockICentralReservationGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, reason string) error { +func (m *MockICentralReservationGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID, reason string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Release", ctx, schedulerScope, exptRunID, itemID, reason) + ret := m.ctrl.Call(m, "Release", ctx, schedulerScope, exptRunID, itemID, expectedRequestID, reason) ret0, _ := ret[0].(error) return ret0 } // Release indicates an expected call of Release. -func (mr *MockICentralReservationGuardMockRecorder) Release(ctx, schedulerScope, exptRunID, itemID, reason any) *gomock.Call { +func (mr *MockICentralReservationGuardMockRecorder) Release(ctx, schedulerScope, exptRunID, itemID, expectedRequestID, reason any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockICentralReservationGuard)(nil).Release), ctx, schedulerScope, exptRunID, itemID, reason) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockICentralReservationGuard)(nil).Release), ctx, schedulerScope, exptRunID, itemID, expectedRequestID, reason) +} + +// ReleaseTerminalReservation mocks base method. +func (m *MockICentralReservationGuard) ReleaseTerminalReservation(ctx context.Context, schedulerScope string, previousRunID, itemID int64, previousRequestID string) (component.TerminalReservationReleaseResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReleaseTerminalReservation", ctx, schedulerScope, previousRunID, itemID, previousRequestID) + ret0, _ := ret[0].(component.TerminalReservationReleaseResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReleaseTerminalReservation indicates an expected call of ReleaseTerminalReservation. +func (mr *MockICentralReservationGuardMockRecorder) ReleaseTerminalReservation(ctx, schedulerScope, previousRunID, itemID, previousRequestID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseTerminalReservation", reflect.TypeOf((*MockICentralReservationGuard)(nil).ReleaseTerminalReservation), ctx, schedulerScope, previousRunID, itemID, previousRequestID) } // RepriceRunConsumption mocks base method. diff --git a/backend/modules/evaluation/domain/entity/event.go b/backend/modules/evaluation/domain/entity/event.go index 45a4435128..3eeedf5391 100644 --- a/backend/modules/evaluation/domain/entity/event.go +++ b/backend/modules/evaluation/domain/entity/event.go @@ -5,6 +5,8 @@ package entity import ( "context" + "maps" + "strconv" "github.com/coze-dev/coze-loop/backend/pkg/ctxcache" ) @@ -25,7 +27,7 @@ type ExptScheduleEvent struct { InfraErrorRetryTimes int } -type ctxTargetCalledCacheKey struct{} +type ctxTargetCalledCacheKey struct{ event *ExptItemEvalEvent } type ctxForceNoRetryKey struct{} @@ -35,11 +37,21 @@ type ctxCentralQuotaHeldKey struct{} // ctxCentralAdmittedExptKey 见 WithCtxCentralAdmittedExpt。 type ctxCentralAdmittedExptKey struct{} +// ExptResumeSource identifies the original invocation, independently of scheduling retries. +type ExptResumeSource struct { + Kind string + TurnID int64 + InvokeID string + RecordID int64 +} + type ExptItemEvalEvent struct { - SpaceID int64 - ExptID int64 - ExptRunID int64 - ExptRunMode ExptRunMode + RequestID string `json:",omitempty"` + ResumeSource *ExptResumeSource `json:",omitempty"` + SpaceID int64 + ExptID int64 + ExptRunID int64 + ExptRunMode ExptRunMode EvalSetItemID int64 AsyncReportTrigger bool @@ -52,6 +64,47 @@ type ExptItemEvalEvent struct { Session *Session } +// Clone preserves the event contract and copies every mutable field. Session and +// ResumeSource contain only scalar values; Ext is the event's only collection. +func (e *ExptItemEvalEvent) Clone() *ExptItemEvalEvent { + if e == nil { + return nil + } + cloned := *e + cloned.Ext = maps.Clone(e.Ext) + if e.Session != nil { + session := *e.Session + cloned.Session = &session + } + if e.ResumeSource != nil { + source := *e.ResumeSource + cloned.ResumeSource = &source + } + return &cloned +} + +func (e *ExptItemEvalEvent) CloneWithResumeSource(kind string, recordID, turnID int64) *ExptItemEvalEvent { + cloned := e.Clone() + if cloned != nil { + cloned.ResumeSource = &ExptResumeSource{ + Kind: kind, TurnID: turnID, InvokeID: strconv.FormatInt(recordID, 10), RecordID: recordID, + } + } + return cloned +} + +// CloneForTurn keeps callback flags on the original turn, including turn zero, +// while preserving the source that identifies this event as a continuation. +// Events without source metadata retain the legacy callback contract. +func (e *ExptItemEvalEvent) CloneForTurn(turnID int64) *ExptItemEvalEvent { + cloned := e.Clone() + if cloned != nil && cloned.ResumeSource != nil && cloned.ResumeSource.TurnID != turnID { + cloned.AsyncReportTrigger = false + cloned.AsyncEvaluatorReportTrigger = false + } + return cloned +} + func (e *ExptItemEvalEvent) WithCtxForceNoRetry(ctx context.Context) { ctxcache.Store(ctx, ctxForceNoRetryKey{}, struct{}{}) } @@ -113,11 +166,11 @@ func (e *ExptItemEvalEvent) CtxCentralQuotaHeldScope(ctx context.Context) (strin } func (e *ExptItemEvalEvent) WithCtxTargetCalled(ctx context.Context) { - ctxcache.Store(ctx, ctxTargetCalledCacheKey{}, struct{}{}) + ctxcache.Store(ctx, ctxTargetCalledCacheKey{event: e}, struct{}{}) } func (e *ExptItemEvalEvent) CtxTargetCalled(ctx context.Context) bool { - _, ok := ctxcache.Get[struct{}](ctx, ctxTargetCalledCacheKey{}) + _, ok := ctxcache.Get[struct{}](ctx, ctxTargetCalledCacheKey{event: e}) return ok } @@ -129,6 +182,12 @@ func (e *ExptItemEvalEvent) IgnoreExistedEvaluatorResult(ctx context.Context) bo } func (e *ExptItemEvalEvent) ignoreExistedResult() bool { + if e == nil { + return false + } + if e.ResumeSource != nil || e.AsyncReportTrigger || e.AsyncEvaluatorReportTrigger { + return false + } return (e.ExptRunMode == EvaluationModeRetryItems || e.ExptRunMode == EvaluationModeRetryAll) && e.RetryTimes == 0 } diff --git a/backend/modules/evaluation/domain/entity/event_test.go b/backend/modules/evaluation/domain/entity/event_test.go index ea87c3bc4e..3956b9f94c 100644 --- a/backend/modules/evaluation/domain/entity/event_test.go +++ b/backend/modules/evaluation/domain/entity/event_test.go @@ -18,7 +18,7 @@ func TestExptItemEvalEvent_WithCtxTargetCalled(t *testing.T) { e := &ExptItemEvalEvent{} e.WithCtxTargetCalled(ctx) - _, ok := ctxcache.Get[struct{}](ctx, ctxTargetCalledCacheKey{}) + _, ok := ctxcache.Get[struct{}](ctx, ctxTargetCalledCacheKey{event: e}) assert.False(t, ok) }) @@ -27,7 +27,7 @@ func TestExptItemEvalEvent_WithCtxTargetCalled(t *testing.T) { e := &ExptItemEvalEvent{} e.WithCtxTargetCalled(ctx) - _, ok := ctxcache.Get[struct{}](ctx, ctxTargetCalledCacheKey{}) + _, ok := ctxcache.Get[struct{}](ctx, ctxTargetCalledCacheKey{event: e}) assert.True(t, ok) }) } diff --git a/backend/modules/evaluation/domain/entity/expt.go b/backend/modules/evaluation/domain/entity/expt.go index 5d9cc05a3e..6b331254a1 100644 --- a/backend/modules/evaluation/domain/entity/expt.go +++ b/backend/modules/evaluation/domain/entity/expt.go @@ -5,6 +5,7 @@ package entity import ( "context" + stdjson "encoding/json" "fmt" "regexp" "time" @@ -963,8 +964,10 @@ type InvokeExptReq struct { } type ExptRunLogItems struct { - ItemIDs []int64 - CreateAt *int64 + ItemIDs []int64 + CreateAt *int64 + ManualRetryReceipt *ManualRetryReceipt `json:"manual_retry_receipt,omitempty"` + unknownFields map[string]stdjson.RawMessage } // ===================================================================================== diff --git a/backend/modules/evaluation/domain/entity/expt_item_current_request.go b/backend/modules/evaluation/domain/entity/expt_item_current_request.go new file mode 100644 index 0000000000..a8ef323caa --- /dev/null +++ b/backend/modules/evaluation/domain/entity/expt_item_current_request.go @@ -0,0 +1,110 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package entity + +import ( + "time" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +// ExptItemCurrentRequest is one primary database observation of the canonical item and run log. +type ExptItemCurrentRequest struct { + PublicationObserved bool + Publication *ExptItemResultPublishSource + Source *ExptCurrentItemRunSource + SpaceID, ExptID, RunID, ItemID, ItemVersionID int64 + RequestID string + Mode ExptRunMode + Status ItemRunState + ResultState ExptItemResultState + RetryTimes int32 + UpdatedAt *time.Time + QuotaReservationState QuotaReservationState +} + +func (s *ExptItemCurrentRequest) Matches(runID int64, requestID string) bool { + return s != nil && s.RunID == runID && s.RequestID == requestID && !IsItemRunFinished(s.Status) +} + +// ExptCurrentItemRunSource freezes the item and invocation references before execution. +// It is internal and is never serialized into provider requests or MQ messages. +type ExptCurrentItemRunSource struct { + CanonicalTurns []*ExptTurnResult + CurrentRefs []*ExptTurnEvaluatorResultRef + Current ExptItemCurrentRequest + Turns map[int64]*ExptTurnResultRunLog +} + +// ExptCurrentItemRunWrite describes execution logging, not result publication. +type ExptCurrentItemRunWrite struct { + Status ItemRunState + ErrMsg []byte + RetryTimes int32 + Yield bool + RetryOnly bool + FailPendingTurns bool + MissingTurnLogs []*ExptTurnResultRunLog + + // OnlyIfQueueingUnreserved rejects a superseded pre-execution observation. + // Its zero value preserves ordinary completion, yield and retry writes. + OnlyIfQueueingUnreserved bool +} + +// SameCurrentEvaluatorRefs compares invocation identity across JSON round trips. +// JSON omitempty makes an empty slice/map and nil the same persisted source. +func SameCurrentEvaluatorRefs(a, b *EvaluatorResults) bool { + if a == nil { + a = &EvaluatorResults{} + } + if b == nil { + b = &EvaluatorResults{} + } + if len(a.EvalVerIDToResID) != len(b.EvalVerIDToResID) || len(a.Registered) != len(b.Registered) || len(a.Inline) != len(b.Inline) { + return false + } + for version, id := range a.EvalVerIDToResID { + if b.EvalVerIDToResID[version] != id { + return false + } + } + for index, ref := range a.Registered { + other := b.Registered[index] + if ref == nil || other == nil { + if ref != other { + return false + } + continue + } + if *ref != *other { + return false + } + } + for index, ref := range a.Inline { + other := b.Inline[index] + if ref == nil || other == nil { + if ref != other { + return false + } + continue + } + if *ref != *other { + return false + } + } + return true +} + +// TurnError preserves the existing result reader's turn-level error contract. +// A yielding retry leaves prior error text unchanged; terminal completion replaces it. +func (w *ExptCurrentItemRunWrite) TurnError(status TurnRunState) []byte { + if status == TurnRunState_Terminal { + return []byte(errno.SerializeErr(errno.NewItemManuallyTerminatedErr())) + } + if status != TurnRunState_Fail { + return nil + } + reason := errno.DeserializeErr(w.ErrMsg) + return []byte(errno.SerializeErr(errno.NewTurnOtherErr(reason.Error(), reason))) +} diff --git a/backend/modules/evaluation/domain/entity/expt_manual_retry.go b/backend/modules/evaluation/domain/entity/expt_manual_retry.go new file mode 100644 index 0000000000..7b9867f270 --- /dev/null +++ b/backend/modules/evaluation/domain/entity/expt_manual_retry.go @@ -0,0 +1,202 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package entity + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "maps" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/errorx" +) + +const ManualRetryExtKey = "manual_retry" + +type ManualRetryContext struct { + RunID int64 `json:"run_id"` + RequestID string `json:"request_id"` + Mode ExptRunMode `json:"mode"` + Operator string `json:"operator"` + AcceptedAtMS int64 `json:"accepted_at_ms"` +} + +type ManualRetryReceipt struct { + RequestID string `json:"request_id"` + PayloadHash string `json:"payload_hash"` + AcceptedCount int32 `json:"accepted_count"` + RejectedItems []*ManualRetryRejectedItem `json:"rejected_items"` + SnapshotAtMS int64 `json:"snapshot_at_ms"` + unknownFields map[string]json.RawMessage +} + +type ManualRetryRejectedItem struct { + ItemID int64 `json:"item_id"` + Reason string `json:"reason"` + unknownFields map[string]json.RawMessage +} + +// ParseExptItemResultExt validates both JSON layers before any retry merge. +func ParseExptItemResultExt(raw []byte) (map[string]string, error) { + if len(raw) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil, nil + } + var values map[string]json.RawMessage + if err := json.Unmarshal(raw, &values); err != nil { + return nil, fmt.Errorf("parse experiment item ext: %w", err) + } + ext := make(map[string]string, len(values)) + for key, value := range values { + var str *string + if err := json.Unmarshal(value, &str); err != nil || str == nil { + return nil, fmt.Errorf("experiment item ext %q must be a string", key) + } + ext[key] = *str + } + if _, err := ParseManualRetryContext(ext); err != nil { + return nil, err + } + return ext, nil +} + +func ParseManualRetryContext(ext map[string]string) (*ManualRetryContext, error) { + raw, exists := ext[ManualRetryExtKey] + if !exists { + return nil, nil + } + var retry *ManualRetryContext + if err := json.Unmarshal([]byte(raw), &retry); err != nil { + return nil, fmt.Errorf("parse manual retry context: %w", err) + } + if retry == nil || retry.RunID <= 0 || retry.RequestID == "" || + (retry.Mode != EvaluationModeFailRetry && retry.Mode != EvaluationModeRetryItems) { + return nil, fmt.Errorf("invalid manual retry context") + } + return retry, nil +} + +func MergeManualRetryContext(raw []byte, retry *ManualRetryContext) ([]byte, error) { + ext, err := ParseExptItemResultExt(raw) + if err != nil { + return nil, err + } + encoded, err := json.Marshal(retry) + if err != nil { + return nil, err + } + if _, err := ParseManualRetryContext(map[string]string{ManualRetryExtKey: string(encoded)}); err != nil { + return nil, err + } + if ext == nil { + ext = make(map[string]string) + } + ext[ManualRetryExtKey] = string(encoded) + encoded, err = json.Marshal(ext) + if err == nil && len(encoded) > 65535 { + return nil, errorx.NewByCode(errno.ManualRetryCapacityCode, errorx.WithExtraMsg("manual retry ext exceeds blob capacity")) + } + return encoded, err +} + +func WithoutManualRetryExt(ext map[string]string) map[string]string { + result := maps.Clone(ext) + delete(result, ManualRetryExtKey) + return result +} + +// ErrExptRunLogItemsCapacity identifies BLOB overflow without assigning a business status. +var ErrExptRunLogItemsCapacity = errors.New("manual retry receipt exceeds blob capacity") + +func EncodeExptRunLogItems(items []ExptRunLogItems) ([]byte, error) { + encoded, err := json.Marshal(items) + if err == nil && len(encoded) > 65535 { + return nil, ErrExptRunLogItemsCapacity + } + return encoded, err +} + +// Raw unknown fields retain future metadata without rounding large numeric IDs. +func decodeRunMetadata(raw []byte, value any, known ...string) (map[string]json.RawMessage, error) { + if err := json.Unmarshal(raw, value); err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, err + } + for _, key := range known { + delete(fields, key) + } + return fields, nil +} + +func encodeRunMetadata(value any, unknown map[string]json.RawMessage) ([]byte, error) { + raw, err := json.Marshal(value) + if err != nil || len(unknown) == 0 { + return raw, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, err + } + for key, value := range unknown { + if _, exists := fields[key]; !exists { + fields[key] = value + } + } + return json.Marshal(fields) +} + +func (i *ExptRunLogItems) UnmarshalJSON(raw []byte) error { + type plain ExptRunLogItems + var value plain + unknown, err := decodeRunMetadata(raw, &value, "ItemIDs", "CreateAt", "manual_retry_receipt") + if err != nil { + return err + } + *i = ExptRunLogItems(value) + i.unknownFields = unknown + return nil +} + +func (i ExptRunLogItems) MarshalJSON() ([]byte, error) { + type plain ExptRunLogItems + return encodeRunMetadata(plain(i), i.unknownFields) +} + +func (r *ManualRetryReceipt) UnmarshalJSON(raw []byte) error { + type plain ManualRetryReceipt + var value plain + unknown, err := decodeRunMetadata(raw, &value, "request_id", "payload_hash", "accepted_count", "rejected_items", "snapshot_at_ms") + if err != nil { + return err + } + *r = ManualRetryReceipt(value) + r.unknownFields = unknown + return nil +} + +func (r ManualRetryReceipt) MarshalJSON() ([]byte, error) { + type plain ManualRetryReceipt + return encodeRunMetadata(plain(r), r.unknownFields) +} + +func (r *ManualRetryRejectedItem) UnmarshalJSON(raw []byte) error { + type plain ManualRetryRejectedItem + var value plain + unknown, err := decodeRunMetadata(raw, &value, "item_id", "reason") + if err != nil { + return err + } + *r = ManualRetryRejectedItem(value) + r.unknownFields = unknown + return nil +} + +func (r ManualRetryRejectedItem) MarshalJSON() ([]byte, error) { + type plain ManualRetryRejectedItem + return encodeRunMetadata(plain(r), r.unknownFields) +} diff --git a/backend/modules/evaluation/domain/entity/expt_manual_retry_boundary_test.go b/backend/modules/evaluation/domain/entity/expt_manual_retry_boundary_test.go new file mode 100644 index 0000000000..5d17883088 --- /dev/null +++ b/backend/modules/evaluation/domain/entity/expt_manual_retry_boundary_test.go @@ -0,0 +1,48 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package entity + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/cloudwego/kitex/pkg/kerrors" + "github.com/stretchr/testify/require" +) + +func TestManualRetryCapacityBizBoundary(t *testing.T) { + retry := &ManualRetryContext{RunID: 8, RequestID: "request", Mode: EvaluationModeRetryItems} + _, extErr := MergeManualRetryContext([]byte(`{"padding":"`+strings.Repeat("x", 65500)+`"}`), retry) + for _, err := range []error{extErr} { + require.Error(t, err) + biz, ok := kerrors.FromBizStatusError(err) + require.True(t, ok, "capacity must reach PacketAdapter as BizStatus") + require.Equal(t, int32(601204104), biz.BizStatusCode()) + } + _, err := MergeManualRetryContext([]byte(`{"manual_retry":"broken"}`), retry) + require.Error(t, err) + _, ok := kerrors.FromBizStatusError(err) + require.False(t, ok, "corrupt stored source is not a business refusal") +} + +func TestRunLogCapacityIsOrdinaryError(t *testing.T) { + ids := make([]int64, 4000) + for i := range ids { + ids[i] = 9223372036854700000 + int64(i) + } + _, err := EncodeExptRunLogItems([]ExptRunLogItems{{ItemIDs: ids}}) + require.ErrorIs(t, err, ErrExptRunLogItemsCapacity) + _, ok := kerrors.FromBizStatusError(err) + require.False(t, ok, "ordinary run encoding must retain the internal error category") +} + +func TestRunLogEncodingCauseIsOrdinaryError(t *testing.T) { + _, err := EncodeExptRunLogItems([]ExptRunLogItems{{unknownFields: map[string]json.RawMessage{"future": json.RawMessage(`{`)}}}) + require.Error(t, err) + var cause *json.MarshalerError + require.ErrorAs(t, err, &cause) + require.NotErrorIs(t, err, ErrExptRunLogItemsCapacity) + _, ok := kerrors.FromBizStatusError(err) + require.False(t, ok) +} diff --git a/backend/modules/evaluation/domain/entity/expt_manual_retry_prepared.go b/backend/modules/evaluation/domain/entity/expt_manual_retry_prepared.go new file mode 100644 index 0000000000..0e4ac2c604 --- /dev/null +++ b/backend/modules/evaluation/domain/entity/expt_manual_retry_prepared.go @@ -0,0 +1,70 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package entity + +type PreparedRetryItem struct { + PreviousRunID int64 + PreviousRequestID string + ItemVersionID int64 + SourceItem *ExptItemResult + CurrentItemLog *ExptItemResultRunLog + FrozenItemRef *ExptItemRef + SingleSetSource *PreparedRetrySingleSetSource + SourceTurnLogs []*ExptTurnResultRunLog + Turns []*PreparedRetryTurn +} + +// PreparedRetrySingleSetSource freezes the existing experiment binding for the +// single-set path, whose start does not materialize expt_item_ref rows. +// It is preparation metadata only and is never persisted as a synthetic ref. +type PreparedRetrySingleSetSource struct { + SourceType ExptEvalSetSourceType + EvalSetID int64 + EvalSetVersionID int64 + EvalSetSpaceID int64 + TargetID int64 + TargetVersionID int64 + TargetSpaceID int64 + TargetType EvalTargetType +} + +type PreparedRetryTurn struct { + SourceTurn *ExptTurnResult + CurrentRunLog *ExptTurnResultRunLog + SourceRefs []*ExptTurnEvaluatorResultRef + TargetResultID int64 + EvaluatorResults *EvaluatorResults + KeepRefIDs []int64 + PreserveSuccess bool +} + +type ManualRetryPreparation struct { + Items []*PreparedRetryItem + RejectedItems []*ManualRetryRejectedItem + SnapshotAtMS int64 +} + +type RequeueTerminalItemsRequest struct { + SpaceID int64 + ExptID int64 + RunID int64 + RequestID string + PayloadHash string + Operator string + Mode ExptRunMode + SnapshotAtMS int64 + Items []*PreparedRetryItem + RejectedItems []*ManualRetryRejectedItem +} + +type RetryInCurrentRunRequest struct { + SpaceID int64 + ExptID int64 + ExpectedRunID int64 + RequestID string + Mode ExptRunMode + ItemIDs []int64 + Ext map[string]string + Session *Session +} diff --git a/backend/modules/evaluation/domain/entity/expt_manual_retry_test.go b/backend/modules/evaluation/domain/entity/expt_manual_retry_test.go new file mode 100644 index 0000000000..0a4cff8195 --- /dev/null +++ b/backend/modules/evaluation/domain/entity/expt_manual_retry_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package entity + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExptRunLogItemsPreserveOptionalMetadata(t *testing.T) { + raw := `[{"ItemIDs":[11],"CreateAt":1700000000,"future":{"id":9223372036854775806}},{"ItemIDs":null,"CreateAt":1700000001,"manual_retry_receipt":{"request_id":"rq1","payload_hash":"hash","accepted_count":1,"rejected_items":[{"item_id":12,"reason":"processing","future_reason":{"id":9223372036854775806}}],"snapshot_at_ms":1700000000123,"future_receipt":true}}]` + var chunks []ExptRunLogItems + require.NoError(t, json.Unmarshal([]byte(raw), &chunks)) + run := &ExptRunLog{ItemIds: chunks} + require.Equal(t, []int64{11}, run.GetItemIDs()) + encoded, err := json.Marshal(run.ItemIds) + require.NoError(t, err) + require.JSONEq(t, raw, string(encoded)) + var fields []map[string]json.RawMessage + require.NoError(t, json.Unmarshal(encoded, &fields)) + require.Equal(t, `{"id":9223372036854775806}`, string(fields[0]["future"])) + require.NoError(t, run.AppendItemIDs([]int64{13})) + require.Equal(t, []int64{11, 13}, run.GetItemIDs()) + require.Error(t, run.AppendItemIDs([]int64{11})) +} + +func TestManualRetryExtStrictMerge(t *testing.T) { + ctx := &ManualRetryContext{RunID: 7, RequestID: "rq1", Mode: EvaluationModeFailRetry, Operator: "user", AcceptedAtMS: 1700000000123} + raw, err := MergeManualRetryContext([]byte(`{"other":"keep","retry_yield_enabled":"true"}`), ctx) + require.NoError(t, err) + ext, err := ParseExptItemResultExt(raw) + require.NoError(t, err) + require.Equal(t, "keep", ext["other"]) + got, err := ParseManualRetryContext(ext) + require.NoError(t, err) + require.Equal(t, ctx, got) + require.Equal(t, map[string]string{"other": "keep", "retry_yield_enabled": "true"}, WithoutManualRetryExt(ext)) + require.Contains(t, ext, "manual_retry") + for _, raw := range []string{`{`, `[]`, `{"other":1}`, `{"other":null}`, `{"manual_retry":"{"}`, `{"manual_retry":"null"}`, `{"manual_retry":"{}"}`, `{"manual_retry":"{\"run_id\":7,\"request_id\":\"\",\"mode\":2}"}`} { + _, err := ParseExptItemResultExt([]byte(raw)) + require.Error(t, err, raw) + _, err = MergeManualRetryContext([]byte(raw), ctx) + require.Error(t, err, raw) + } + for _, raw := range [][]byte{nil, []byte(`null`), []byte(`{}`)} { + ext, err := ParseExptItemResultExt(raw) + require.NoError(t, err) + got, err := ParseManualRetryContext(ext) + require.NoError(t, err) + require.Nil(t, got) + } + _, err = MergeManualRetryContext(nil, &ManualRetryContext{RunID: 7, RequestID: "rq1", Mode: EvaluationModeSubmit}) + require.Error(t, err) +} + +func TestRunLogMetadataCapacity(t *testing.T) { + var chunks []ExptRunLogItems + base := `[{"ItemIDs":null,"CreateAt":null,"future":"` + strings.Repeat("x", 65489) + `"}]` + require.NoError(t, json.Unmarshal([]byte(base), &chunks)) + encoded, err := EncodeExptRunLogItems(chunks) + require.NoError(t, err) + require.Len(t, encoded, 65535) + chunks = append(chunks, ExptRunLogItems{ManualRetryReceipt: &ManualRetryReceipt{RequestID: "rq2", PayloadHash: "hash", AcceptedCount: 0, RejectedItems: []*ManualRetryRejectedItem{{ItemID: 9223372036854775806, Reason: "reservation_mismatch"}}, SnapshotAtMS: 1700000000123}}) + _, err = EncodeExptRunLogItems(chunks) + require.Error(t, err) +} + +func TestExptTurnResultRunLogFreezesItemVersion(t *testing.T) { + result := &ExptTurnResult{ID: 1, ItemID: 2, ItemVersionID: 1234567890123456789, TurnID: 3} + require.Equal(t, int64(1234567890123456789), result.ToRunLogDO().ItemVersionID) +} diff --git a/backend/modules/evaluation/domain/entity/expt_result.go b/backend/modules/evaluation/domain/entity/expt_result.go index 88878921a9..dc2cd8f434 100644 --- a/backend/modules/evaluation/domain/entity/expt_result.go +++ b/backend/modules/evaluation/domain/entity/expt_result.go @@ -292,12 +292,17 @@ func (e ExptEvalItems) GetItemIDs() []int64 { } type ExptEvalItem struct { - ExptID int64 - EvalSetVersionID int64 - ItemID int64 - State ItemRunState - RetryTimes int32 // ★ 本轮 expt_run 内该 item 已被系统自动重试的次数; 供 handleToSubmits 回填事件 - UpdatedAt *time.Time + PublicationObserved bool + Publication *ExptItemResultPublishSource + Source *ExptCurrentItemRunSource + RequestID string + Mode ExptRunMode + ExptID int64 + EvalSetVersionID int64 + ItemID int64 + State ItemRunState + RetryTimes int32 // ★ 本轮 expt_run 内该 item 已被系统自动重试的次数; 供 handleToSubmits 回填事件 + UpdatedAt *time.Time } func (e *ExptEvalItem) SetState(state ItemRunState) *ExptEvalItem { @@ -357,6 +362,7 @@ func (tr *ExptTurnResult) ToRunLogDO() *ExptTurnResultRunLog { ExptID: tr.ExptID, ExptRunID: tr.ExptRunID, ItemID: tr.ItemID, + ItemVersionID: tr.ItemVersionID, TurnID: tr.TurnID, Status: TurnRunState(tr.Status), TraceID: tr.TraceID, diff --git a/backend/modules/evaluation/domain/entity/expt_result_publish.go b/backend/modules/evaluation/domain/entity/expt_result_publish.go new file mode 100644 index 0000000000..6e1ad0eeb3 --- /dev/null +++ b/backend/modules/evaluation/domain/entity/expt_result_publish.go @@ -0,0 +1,21 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package entity + +// ExptItemResultPublishSource is a single primary database observation, frozen +// before record reads and publication. It is never sent to a target/evaluator. +type ExptItemResultPublishSource struct { + Source *ExptCurrentItemRunSource + Item *ExptItemResult + ItemLog *ExptItemResultRunLog + ItemRef *ExptItemRef + ItemConfigJSON []byte +} + +// ExptItemResultPublication contains only the projection prepared from Source. +// IDs and scores are prepared before entering the repository transaction. +type ExptItemResultPublication struct { + Source *ExptItemResultPublishSource + Turns []*ExptTurnResult + Refs []*ExptTurnEvaluatorResultRef +} diff --git a/backend/modules/evaluation/domain/entity/expt_run.go b/backend/modules/evaluation/domain/entity/expt_run.go index 876718e17e..62db918226 100644 --- a/backend/modules/evaluation/domain/entity/expt_run.go +++ b/backend/modules/evaluation/domain/entity/expt_run.go @@ -217,9 +217,10 @@ func (s *SchedulerAbortCtrl) Abort(spaceID, exptID int64, userID string, exptTyp } type ExptExecConf struct { - DaemonIntervalSecond int `json:"daemon_interval_second" mapstructure:"daemon_interval_second"` - ZombieIntervalSecond int `json:"expt_zombie_second" mapstructure:"expt_zombie_second"` - SpaceExptConcurLimit int `json:"space_expt_concur_limit" mapstructure:"space_expt_concur_limit"` + RunningRetryEnabled *bool `json:"running_retry_enabled,omitempty" mapstructure:"running_retry_enabled"` + DaemonIntervalSecond int `json:"daemon_interval_second" mapstructure:"daemon_interval_second"` + ZombieIntervalSecond int `json:"expt_zombie_second" mapstructure:"expt_zombie_second"` + SpaceExptConcurLimit int `json:"space_expt_concur_limit" mapstructure:"space_expt_concur_limit"` ExptItemEvalConf *ExptItemEvalConf `json:"expt_item_eval_conf" mapstructure:"expt_item_eval_conf"` @@ -779,3 +780,26 @@ type EvalAsyncCtx struct { // ApplicationID 沙箱 agent 应用 id (即 EvalTarget.SourceTargetID, AgentKit application_id), 供 invoke_finished 打点复用。 ApplicationID string } + +// Clone isolates the saved event, reporting session and optional trajectory flag +// without serializing away any of the existing AsyncCtx fields. +func (e *EvalAsyncCtx) Clone() *EvalAsyncCtx { + if e == nil { + return nil + } + cloned := *e + cloned.Event = e.Event.Clone() + if e.Session != nil { + session := *e.Session + cloned.Session = &session + } + if e.EnableExtractTrajectory != nil { + enabled := *e.EnableExtractTrajectory + cloned.EnableExtractTrajectory = &enabled + } + return &cloned +} + +func (e *ExptExecConf) IsRunningRetryEnabled() bool { + return e != nil && e.RunningRetryEnabled != nil && *e.RunningRetryEnabled +} diff --git a/backend/modules/evaluation/domain/entity/expt_run_finish.go b/backend/modules/evaluation/domain/entity/expt_run_finish.go new file mode 100644 index 0000000000..c57d6fcd95 --- /dev/null +++ b/backend/modules/evaluation/domain/entity/expt_run_finish.go @@ -0,0 +1,39 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package entity + +type ExptRunFinishDisposition int + +const ( + ExptRunFinishFinished ExptRunFinishDisposition = iota + 1 + ExptRunFinishAlreadyFinished + ExptRunFinishPending + ExptRunFinishNonCurrent +) + +// ExptRunFinishRequest carries identities only; final counts are read in SQL. +type ExptRunFinishRequest struct{ SpaceID, ExptID, RunID int64 } + +type ExptRunFinishResult struct { + Disposition ExptRunFinishDisposition + Experiment *Experiment +} + +// ExptRunTerminationRequest is a frozen control operation, never an execution writer. +type ExptRunTerminationRequest struct { + SpaceID, ExptID, RunID int64 + Begin bool + Finalize bool + SetupAbort bool + Status ExptStatus + StatusMessage string + NoCompleteItemTurn bool + ExpectedSource *ExptCurrentItemRunSource + Items []*ExptCurrentItemRunSource + MissingTurnLogs []*ExptTurnResultRunLog +} +type ExptRunTerminationResult struct { + Applied bool + NonCurrent bool + Experiment *Experiment +} diff --git a/backend/modules/evaluation/domain/repo/expt.go b/backend/modules/evaluation/domain/repo/expt.go index 1cf56c4a85..76d038ada2 100644 --- a/backend/modules/evaluation/domain/repo/expt.go +++ b/backend/modules/evaluation/domain/repo/expt.go @@ -59,6 +59,11 @@ type IExptStatsRepo interface { } type IExptItemResultRepo interface { + // Resource cleanup uses the same strict primary dispatch observation, including terminal runs. + ScanIncompleteDispatchObservations(ctx context.Context, spaceID, exptID, runID int64) ([]*ExptDispatchObservation, error) + ScanCurrentRequests(ctx context.Context, spaceID, exptID, runID int64, retryFirst bool) ([]*entity.ExptItemCurrentRequest, error) + GetCurrentRequest(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptItemCurrentRequest, error) + MarkProcessingIfCurrent(ctx context.Context, spaceID, exptID, runID, itemID int64, requestID string) (bool, error) BatchGet(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptItemResult, error) BatchCreateNX(ctx context.Context, itemResults []*entity.ExptItemResult) error ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64) (results []*entity.ExptItemResult, ncursor int64, err error) @@ -118,6 +123,8 @@ type IExptRunLogRepo interface { Save(ctx context.Context, exptRunLog *entity.ExptRunLog) error Update(ctx context.Context, exptID, exptRunID, spaceID int64, ufields map[string]any) error Get(ctx context.Context, exptID, exptRunID int64) (*entity.ExptRunLog, error) + AppendRunItemIDs(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64) error + UpdateRunStats(ctx context.Context, exptID, exptRunID, spaceID int64, runLog *entity.ExptRunLog, stats *entity.ExptStats) error } type IExptAggrResultRepo interface { diff --git a/backend/modules/evaluation/domain/repo/expt_current_result.go b/backend/modules/evaluation/domain/repo/expt_current_result.go new file mode 100644 index 0000000000..9ed1c6b679 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/expt_current_result.go @@ -0,0 +1,20 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package repo + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +// IExptCurrentResultRepo is implemented by the existing item repository. +// Every write holds the canonical row lock and checks the frozen source in one transaction. +// +//go:generate mockgen -destination=mocks/expt_current_result.go -package=mocks . IExptCurrentResultRepo +type IExptCurrentResultRepo interface { + GetCurrentItemRunSource(context.Context, int64, int64, int64, int64) (*entity.ExptCurrentItemRunSource, error) + StoreCurrentTurnRunResult(context.Context, *entity.ExptCurrentItemRunSource, *entity.ExptTurnResultRunLog) (bool, error) + CompleteCurrentItemRun(context.Context, *entity.ExptCurrentItemRunSource, *entity.ExptCurrentItemRunWrite) (bool, error) +} diff --git a/backend/modules/evaluation/domain/repo/expt_item_dispatch.go b/backend/modules/evaluation/domain/repo/expt_item_dispatch.go index 60bc83ee30..bb719dee33 100644 --- a/backend/modules/evaluation/domain/repo/expt_item_dispatch.go +++ b/backend/modules/evaluation/domain/repo/expt_item_dispatch.go @@ -11,74 +11,37 @@ import ( //go:generate mockgen -destination=mocks/expt_item_dispatch.go -package=mocks . IExptItemDispatchRepo -// IExptItemDispatchRepo 中心化调度的 item 派发投影读写。 -// -// 与既有 IExptItemResultRepo 的分工：后者面向 item 执行结果的完整生命周期；本接口只关心 -// 「哪些 item 可被授予、哪些已占用并发」这一个问题，因此方法都围绕 run log 的 -// (status, quota_reservation_state) 二元组。 -// -// 索引：本接口刻意**不依赖新增索引**。所有方法的 WHERE 恒以 (space_id, expt_id, expt_run_id) -// 打头，该前缀已被既有 uk_expt_run_item_turn / idx_expt_run_result_state 覆盖； -// 单个 run 的 run log 仅数百行（内场实测最大 914），status 与预占态的过滤是内存操作。 -// 本表内场 7800 万行，为此加复合索引收益接近零而写放大长期存在。 -// 新增方法时请保持这个前缀形状；若确实需要跨 run 扫描（如未来的超时 reservation 对账）， -// 按那个查询的真实形状单独评估索引，不要沿用本接口的假设。 -// -// 为什么单独立一个窄接口而不是往 IExptItemResultRepo 加方法：那个接口已有 15+ 方法、被十余处 -// 依赖，调度只需其中极小一部分；混进去会让所有实现方和 mock 都被迫感知调度概念。 +// IExptItemDispatchRepo reads primary dispatch observations and applies each +// item transition in a canonical-first transaction. Expected request IDs always +// come from the original grant/event; an empty ID only matches an empty source. type IExptItemDispatchRepo interface { - // ClaimQuotaReserved 把一批 item 的 run log 从 Queueing/none CAS 成 Queueing/reserved。 - // - // 返回实际 claim 成功的 item ID。失败的 item 说明它已被别的路径改动（并发 tick、 - // consumer 抢先启动、或 item 已终态）—— 调用方必须只发布成功的那些，并释放失败项 - // 在 Redis 侧仍为 Reserved 的 reservation。 - // - // 这是 Redis 预占与 MQ 发布之间的必经关卡：没有它，同一 item 可能被两拍各自预占各自发布。 - ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) (claimed []int64, err error) - - // ResetQuotaReserved 把 Queueing/reserved 退回 Queueing/none，用于 MQ 明确发布失败的补偿。 - // - // 只处理仍为 Queueing/reserved 的记录：若 item 已被 consumer 推进到 Processing， - // 说明消息其实已投递成功，退回投影会让它被重复授予。 - ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) (reset []int64, err error) - - // LoadDispatchRuntime 一次查询取回该 run 的并发占用与可授予候选。 - // - // 并发占用 = status=Processing 的 item + status=Queueing 且 reserved 的 item； - // 可授予候选 = status=Queueing 且 none 的 item（按 id 升序，取前 limit 个）。 - // 两者由同一次查询产出，避免分两次查询之间状态漂移导致 deficit 算错。 + ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]ClaimedDispatchItem, error) + ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]int64, error) LoadDispatchRuntime(ctx context.Context, spaceID, exptID, exptRunID int64, candidateLimit int) (*ExptDispatchRuntime, error) + // CurrentRequest excludes mismatched/terminal sources. An already Processing + // source is current but not Started, allowing the existing recovery path. + StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (StartReservedItemResult, error) + RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (bool, error) + MGetDispatchObservations(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]*ExptDispatchObservation, error) +} - // StartReservedItem 由 consumer 在取得执行权后调用，把 run log 从 Queueing/reserved - // 推进到 Processing/none，表示「预占已兑现为实际执行」。 - // - // 返回 false 表示 CAS 未命中：可能是重复投递（已 Processing）或 reservation 已被回收。 - // 调用方据此决定是否继续执行 item。 - StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (started bool, err error) - - // RequeueProcessingItem 把 Processing/none 的 item 退回 Queueing/none，让它重新进入授予候选。 - // - // 唯一调用场景：consumer 取执行权时发现 reservation 已不存在（ConfirmRunning=false）， - // 而 run log 投影却已经是 Processing —— 该 item 既没有额度、也没有执行者，属于孤儿态。 - // 不退回的后果不是"少跑一个 item"，而是**整个实验停摆**：Processing 计入 item_concur_num， - // ScanEvalItems 与 LoadDispatchRuntime 都把它当"在跑"，于是既不派新 item 也等不到它完成， - // 直到异步僵尸阈值（默认 3h）才被判 Fail。实测踩过：两个实验各 20 个 item 卡满槽位 77 分钟。 - // - // 为什么退回 Queueing 而不是落 Fail：这类 item 是在创建 turn 之前被丢弃的（turn 表无记录）， - // 从未真正执行，判失败会凭空吃掉一道题；退回队列它能被重新授予并真正跑完。 - // - // 安全性依赖调用点在 item 锁内：锁保证同一 item 没有并发执行者， - // 加上「非终态 + 账本无 reservation」，可断定没有在途执行会被这次退回打断。 - RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (requeued bool, err error) +// ClaimedDispatchItem is captured in the successful Claim transaction. Callers +// must publish this payload without replacing its source with a later read. +type ClaimedDispatchItem struct { + ItemID int64 + RequestID string + Mode entity.ExptRunMode + RetryTimes int32 +} - // MGetDispatchObservations 供分钟级对账使用：批量读取指定 item 的 (status, 预占态)。 - // 对账据此与 Redis reservation 比对，识别四类漂移（Terminal 遗留、Processing 缺 reservation、 - // Queueing/reserved 超时未消费、Queueing/none 却有 reservation）。 - MGetDispatchObservations(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]*ExptDispatchObservation, error) +type StartReservedItemResult struct { + CurrentRequest bool + Started bool } // ExptDispatchRuntime 一个 run 的派发运行态。 type ExptDispatchRuntime struct { + ItemRequestIDs map[int64]string // OccupiedItemIDs 占用并发的 item（Processing ∪ Queueing/reserved）。 // 已在 DB 层去重 —— 同一 item 不可能同时是 Processing 和 Queueing。 OccupiedItemIDs []int64 @@ -96,6 +59,7 @@ func (r *ExptDispatchRuntime) OccupiedCount() int { // ExptDispatchObservation 单个 item 的派发投影观测，供对账使用。 type ExptDispatchObservation struct { + RequestID string ItemID int64 Status int32 QuotaReservationState entity.QuotaReservationState diff --git a/backend/modules/evaluation/domain/repo/expt_manual_retry.go b/backend/modules/evaluation/domain/repo/expt_manual_retry.go new file mode 100644 index 0000000000..3a35a7abc3 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/expt_manual_retry.go @@ -0,0 +1,17 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package repo + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +//go:generate mockgen -destination=mocks/expt_manual_retry.go -package=mocks . IExptManualRetryRepo +type IExptManualRetryRepo interface { + PreflightTerminalItems(context.Context, *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) + FindReceipt(context.Context, int64, int64, int64, string, string) (*entity.ManualRetryReceipt, error) + RequeueTerminalItems(context.Context, *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryReceipt, error) +} diff --git a/backend/modules/evaluation/domain/repo/expt_manual_retry_filter.go b/backend/modules/evaluation/domain/repo/expt_manual_retry_filter.go new file mode 100644 index 0000000000..2ccf403017 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/expt_manual_retry_filter.go @@ -0,0 +1,31 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package repo + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +// IExptManualRetryFilterRepo enumerates canonical manual retries independently +// of the Logged-only scheduler scan. Returned IDs are candidates, never payloads. +// +//go:generate mockgen -destination=mocks/expt_manual_retry_filter.go -package=mocks . IExptManualRetryFilterRepo +type IExptManualRetryFilterRepo interface { + ScanManualRetryFilterItems(context.Context, int64, int64, int64, int64, int) ([]int64, int64, error) +} + +// IExptMaterializedFilterRepo reads the complete projection at the OSS sink. +// An async repository without this port can observe its existing reader but +// remains pending until full projection materialization can be established. +// +//go:generate mockgen -destination=mocks/expt_materialized_filter.go -package=mocks . IExptMaterializedFilterRepo +type IExptMaterializedFilterRepo interface { + GetMaterializedByExptIDItemIDs(context.Context, string, string, string, []string) ([]*entity.ExptTurnResultFilterEntity, error) +} + +// Repositories without this optional declaration must compare item versions. +type IExptMaterializedItemVersionRepo interface { + PersistsMaterializedItemVersion() bool +} diff --git a/backend/modules/evaluation/domain/repo/expt_result_publish.go b/backend/modules/evaluation/domain/repo/expt_result_publish.go new file mode 100644 index 0000000000..7d8f84008f --- /dev/null +++ b/backend/modules/evaluation/domain/repo/expt_result_publish.go @@ -0,0 +1,18 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package repo + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +// IExptResultPublishRepo owns the canonical-first publication transaction. +// +//go:generate mockgen -destination=mocks/expt_result_publish.go -package=mocks . IExptResultPublishRepo +type IExptResultPublishRepo interface { + GetItemResultPublishSource(context.Context, int64, int64, int64, int64) (*entity.ExptItemResultPublishSource, error) + IsItemResultPublishSourceCurrent(context.Context, *entity.ExptItemResultPublishSource) (bool, error) + CommitItemResult(context.Context, *entity.ExptItemResultPublication) (bool, error) +} diff --git a/backend/modules/evaluation/domain/repo/expt_run_finish.go b/backend/modules/evaluation/domain/repo/expt_run_finish.go new file mode 100644 index 0000000000..43d38e4d32 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/expt_run_finish.go @@ -0,0 +1,25 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package repo + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +// IExptRunFinishRepo atomically closes a current run after primary rechecks. +// +//go:generate mockgen -destination=mocks/expt_run_finish.go -package=mocks . IExptRunFinishRepo +type IExptRunFinishRepo interface { + FinishCurrentRun(context.Context, *entity.ExptRunFinishRequest) (*entity.ExptRunFinishResult, error) +} + +// IExptRunTerminationRepo shares the production finish provider without +// widening legacy finish-only implementations. +// +//go:generate mockgen -destination=mocks/expt_run_termination.go -package=mocks . IExptRunTerminationRepo +type IExptRunTerminationRepo interface { + TerminateCurrentRun(context.Context, *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) + GetTerminationItemSource(context.Context, int64, int64, int64, int64) (*entity.ExptCurrentItemRunSource, error) +} diff --git a/backend/modules/evaluation/domain/repo/mocks/expt.go b/backend/modules/evaluation/domain/repo/mocks/expt.go index b74da85171..b7930f82a7 100644 --- a/backend/modules/evaluation/domain/repo/mocks/expt.go +++ b/backend/modules/evaluation/domain/repo/mocks/expt.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -destination ./mocks/expt.go --package mocks github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo IExperimentRepo,IExptStatsRepo,IExptItemResultRepo,IExptTurnResultRepo,IExptRunLogRepo,IExptAggrResultRepo,QuotaRepo,IExptTurnResultFilterRepo,IExptAnnotateRepo,IExptResultExportRecordRepo,IEvalAsyncRepo,IExptInsightAnalysisRecordRepo,IExptItemRefRepo +// mockgen -destination ./mocks/expt.go --package mocks . IExperimentRepo,IExptStatsRepo,IExptItemResultRepo,IExptTurnResultRepo,IExptRunLogRepo,IExptAggrResultRepo,QuotaRepo,IExptTurnResultFilterRepo,IExptAnnotateRepo,IExptResultExportRecordRepo,IEvalAsyncRepo,IExptInsightAnalysisRecordRepo,IExptItemRefRepo // // Package mocks is a generated GoMock package. @@ -15,6 +15,7 @@ import ( db "github.com/coze-dev/coze-loop/backend/infra/db" entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repo "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" gomock "go.uber.org/mock/gomock" ) @@ -427,6 +428,21 @@ func (mr *MockIExptItemResultRepoMockRecorder) CountItemsByStatus(ctx, spaceID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountItemsByStatus", reflect.TypeOf((*MockIExptItemResultRepo)(nil).CountItemsByStatus), ctx, spaceID, exptID) } +// GetCurrentRequest mocks base method. +func (m *MockIExptItemResultRepo) GetCurrentRequest(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptItemCurrentRequest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCurrentRequest", ctx, spaceID, exptID, runID, itemID) + ret0, _ := ret[0].(*entity.ExptItemCurrentRequest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCurrentRequest indicates an expected call of GetCurrentRequest. +func (mr *MockIExptItemResultRepoMockRecorder) GetCurrentRequest(ctx, spaceID, exptID, runID, itemID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentRequest", reflect.TypeOf((*MockIExptItemResultRepo)(nil).GetCurrentRequest), ctx, spaceID, exptID, runID, itemID) +} + // GetItemIDListByExptID mocks base method. func (m *MockIExptItemResultRepo) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) ([]int64, error) { m.ctrl.T.Helper() @@ -548,6 +564,21 @@ func (mr *MockIExptItemResultRepoMockRecorder) MGetItemTurnResults(ctx, spaceID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemTurnResults", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MGetItemTurnResults), ctx, spaceID, exptID, itemIDs) } +// MarkProcessingIfCurrent mocks base method. +func (m *MockIExptItemResultRepo) MarkProcessingIfCurrent(ctx context.Context, spaceID, exptID, runID, itemID int64, requestID string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkProcessingIfCurrent", ctx, spaceID, exptID, runID, itemID, requestID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MarkProcessingIfCurrent indicates an expected call of MarkProcessingIfCurrent. +func (mr *MockIExptItemResultRepoMockRecorder) MarkProcessingIfCurrent(ctx, spaceID, exptID, runID, itemID, requestID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkProcessingIfCurrent", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MarkProcessingIfCurrent), ctx, spaceID, exptID, runID, itemID, requestID) +} + // SaveItemResults mocks base method. func (m *MockIExptItemResultRepo) SaveItemResults(ctx context.Context, itemResults []*entity.ExptItemResult) error { m.ctrl.T.Helper() @@ -562,6 +593,36 @@ func (mr *MockIExptItemResultRepoMockRecorder) SaveItemResults(ctx, itemResults return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveItemResults", reflect.TypeOf((*MockIExptItemResultRepo)(nil).SaveItemResults), ctx, itemResults) } +// ScanCurrentRequests mocks base method. +func (m *MockIExptItemResultRepo) ScanCurrentRequests(ctx context.Context, spaceID, exptID, runID int64, retryFirst bool) ([]*entity.ExptItemCurrentRequest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanCurrentRequests", ctx, spaceID, exptID, runID, retryFirst) + ret0, _ := ret[0].([]*entity.ExptItemCurrentRequest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ScanCurrentRequests indicates an expected call of ScanCurrentRequests. +func (mr *MockIExptItemResultRepoMockRecorder) ScanCurrentRequests(ctx, spaceID, exptID, runID, retryFirst any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanCurrentRequests", reflect.TypeOf((*MockIExptItemResultRepo)(nil).ScanCurrentRequests), ctx, spaceID, exptID, runID, retryFirst) +} + +// ScanIncompleteDispatchObservations mocks base method. +func (m *MockIExptItemResultRepo) ScanIncompleteDispatchObservations(ctx context.Context, spaceID, exptID, runID int64) ([]*repo.ExptDispatchObservation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanIncompleteDispatchObservations", ctx, spaceID, exptID, runID) + ret0, _ := ret[0].([]*repo.ExptDispatchObservation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ScanIncompleteDispatchObservations indicates an expected call of ScanIncompleteDispatchObservations. +func (mr *MockIExptItemResultRepoMockRecorder) ScanIncompleteDispatchObservations(ctx, spaceID, exptID, runID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanIncompleteDispatchObservations", reflect.TypeOf((*MockIExptItemResultRepo)(nil).ScanIncompleteDispatchObservations), ctx, spaceID, exptID, runID) +} + // ScanItemResults mocks base method. func (m *MockIExptItemResultRepo) ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64) ([]*entity.ExptItemResult, int64, error) { m.ctrl.T.Helper() @@ -609,17 +670,17 @@ func (mr *MockIExptItemResultRepoMockRecorder) UpdateItemRunLog(ctx, exptID, exp } // UpdateItemRunLogIfNotTerminal mocks base method. -func (m *MockIExptItemResultRepo) UpdateItemRunLogIfNotTerminal(arg0 context.Context, arg1, arg2 int64, arg3 []int64, arg4 map[string]any, arg5 int64) error { +func (m *MockIExptItemResultRepo) UpdateItemRunLogIfNotTerminal(ctx context.Context, exptID, exptRunID int64, itemID []int64, ufields map[string]any, spaceID int64) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateItemRunLogIfNotTerminal", arg0, arg1, arg2, arg3, arg4, arg5) + ret := m.ctrl.Call(m, "UpdateItemRunLogIfNotTerminal", ctx, exptID, exptRunID, itemID, ufields, spaceID) ret0, _ := ret[0].(error) return ret0 } // UpdateItemRunLogIfNotTerminal indicates an expected call of UpdateItemRunLogIfNotTerminal. -func (mr *MockIExptItemResultRepoMockRecorder) UpdateItemRunLogIfNotTerminal(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { +func (mr *MockIExptItemResultRepoMockRecorder) UpdateItemRunLogIfNotTerminal(ctx, exptID, exptRunID, itemID, ufields, spaceID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateItemRunLogIfNotTerminal", reflect.TypeOf((*MockIExptItemResultRepo)(nil).UpdateItemRunLogIfNotTerminal), arg0, arg1, arg2, arg3, arg4, arg5) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateItemRunLogIfNotTerminal", reflect.TypeOf((*MockIExptItemResultRepo)(nil).UpdateItemRunLogIfNotTerminal), ctx, exptID, exptRunID, itemID, ufields, spaceID) } // UpdateItemsResult mocks base method. @@ -1011,6 +1072,20 @@ func (m *MockIExptRunLogRepo) EXPECT() *MockIExptRunLogRepoMockRecorder { return m.recorder } +// AppendRunItemIDs mocks base method. +func (m *MockIExptRunLogRepo) AppendRunItemIDs(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendRunItemIDs", ctx, exptID, exptRunID, spaceID, itemIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// AppendRunItemIDs indicates an expected call of AppendRunItemIDs. +func (mr *MockIExptRunLogRepoMockRecorder) AppendRunItemIDs(ctx, exptID, exptRunID, spaceID, itemIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendRunItemIDs", reflect.TypeOf((*MockIExptRunLogRepo)(nil).AppendRunItemIDs), ctx, exptID, exptRunID, spaceID, itemIDs) +} + // Create mocks base method. func (m *MockIExptRunLogRepo) Create(ctx context.Context, exptRunLog *entity.ExptRunLog) error { m.ctrl.T.Helper() @@ -1068,6 +1143,20 @@ func (mr *MockIExptRunLogRepoMockRecorder) Update(ctx, exptID, exptRunID, spaceI return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptRunLogRepo)(nil).Update), ctx, exptID, exptRunID, spaceID, ufields) } +// UpdateRunStats mocks base method. +func (m *MockIExptRunLogRepo) UpdateRunStats(ctx context.Context, exptID, exptRunID, spaceID int64, runLog *entity.ExptRunLog, stats *entity.ExptStats) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateRunStats", ctx, exptID, exptRunID, spaceID, runLog, stats) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateRunStats indicates an expected call of UpdateRunStats. +func (mr *MockIExptRunLogRepoMockRecorder) UpdateRunStats(ctx, exptID, exptRunID, spaceID, runLog, stats any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRunStats", reflect.TypeOf((*MockIExptRunLogRepo)(nil).UpdateRunStats), ctx, exptID, exptRunID, spaceID, runLog, stats) +} + // MockIExptAggrResultRepo is a mock of IExptAggrResultRepo interface. type MockIExptAggrResultRepo struct { ctrl *gomock.Controller diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_current_result.go b/backend/modules/evaluation/domain/repo/mocks/expt_current_result.go new file mode 100644 index 0000000000..ae84ef8f38 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/mocks/expt_current_result.go @@ -0,0 +1,87 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptCurrentResultRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_current_result.go -package=mocks . IExptCurrentResultRepo +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptCurrentResultRepo is a mock of IExptCurrentResultRepo interface. +type MockIExptCurrentResultRepo struct { + ctrl *gomock.Controller + recorder *MockIExptCurrentResultRepoMockRecorder + isgomock struct{} +} + +// MockIExptCurrentResultRepoMockRecorder is the mock recorder for MockIExptCurrentResultRepo. +type MockIExptCurrentResultRepoMockRecorder struct { + mock *MockIExptCurrentResultRepo +} + +// NewMockIExptCurrentResultRepo creates a new mock instance. +func NewMockIExptCurrentResultRepo(ctrl *gomock.Controller) *MockIExptCurrentResultRepo { + mock := &MockIExptCurrentResultRepo{ctrl: ctrl} + mock.recorder = &MockIExptCurrentResultRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptCurrentResultRepo) EXPECT() *MockIExptCurrentResultRepoMockRecorder { + return m.recorder +} + +// CompleteCurrentItemRun mocks base method. +func (m *MockIExptCurrentResultRepo) CompleteCurrentItemRun(arg0 context.Context, arg1 *entity.ExptCurrentItemRunSource, arg2 *entity.ExptCurrentItemRunWrite) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteCurrentItemRun", arg0, arg1, arg2) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompleteCurrentItemRun indicates an expected call of CompleteCurrentItemRun. +func (mr *MockIExptCurrentResultRepoMockRecorder) CompleteCurrentItemRun(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteCurrentItemRun", reflect.TypeOf((*MockIExptCurrentResultRepo)(nil).CompleteCurrentItemRun), arg0, arg1, arg2) +} + +// GetCurrentItemRunSource mocks base method. +func (m *MockIExptCurrentResultRepo) GetCurrentItemRunSource(arg0 context.Context, arg1, arg2, arg3, arg4 int64) (*entity.ExptCurrentItemRunSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCurrentItemRunSource", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(*entity.ExptCurrentItemRunSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCurrentItemRunSource indicates an expected call of GetCurrentItemRunSource. +func (mr *MockIExptCurrentResultRepoMockRecorder) GetCurrentItemRunSource(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentItemRunSource", reflect.TypeOf((*MockIExptCurrentResultRepo)(nil).GetCurrentItemRunSource), arg0, arg1, arg2, arg3, arg4) +} + +// StoreCurrentTurnRunResult mocks base method. +func (m *MockIExptCurrentResultRepo) StoreCurrentTurnRunResult(arg0 context.Context, arg1 *entity.ExptCurrentItemRunSource, arg2 *entity.ExptTurnResultRunLog) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StoreCurrentTurnRunResult", arg0, arg1, arg2) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StoreCurrentTurnRunResult indicates an expected call of StoreCurrentTurnRunResult. +func (mr *MockIExptCurrentResultRepoMockRecorder) StoreCurrentTurnRunResult(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreCurrentTurnRunResult", reflect.TypeOf((*MockIExptCurrentResultRepo)(nil).StoreCurrentTurnRunResult), arg0, arg1, arg2) +} diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_item_dispatch.go b/backend/modules/evaluation/domain/repo/mocks/expt_item_dispatch.go index cb772ebd89..fdd817b906 100644 --- a/backend/modules/evaluation/domain/repo/mocks/expt_item_dispatch.go +++ b/backend/modules/evaluation/domain/repo/mocks/expt_item_dispatch.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptItemDispatchRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_item_dispatch.go -package=mocks . IExptItemDispatchRepo +// // Package mocks is a generated GoMock package. package mocks @@ -16,6 +21,7 @@ import ( type MockIExptItemDispatchRepo struct { ctrl *gomock.Controller recorder *MockIExptItemDispatchRepoMockRecorder + isgomock struct{} } // MockIExptItemDispatchRepoMockRecorder is the mock recorder for MockIExptItemDispatchRepo. @@ -36,91 +42,91 @@ func (m *MockIExptItemDispatchRepo) EXPECT() *MockIExptItemDispatchRepoMockRecor } // ClaimQuotaReserved mocks base method. -func (m *MockIExptItemDispatchRepo) ClaimQuotaReserved(arg0 context.Context, arg1, arg2, arg3 int64, arg4 []int64) ([]int64, error) { +func (m *MockIExptItemDispatchRepo) ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]repo.ClaimedDispatchItem, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClaimQuotaReserved", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].([]int64) + ret := m.ctrl.Call(m, "ClaimQuotaReserved", ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) + ret0, _ := ret[0].([]repo.ClaimedDispatchItem) ret1, _ := ret[1].(error) return ret0, ret1 } // ClaimQuotaReserved indicates an expected call of ClaimQuotaReserved. -func (mr *MockIExptItemDispatchRepoMockRecorder) ClaimQuotaReserved(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptItemDispatchRepoMockRecorder) ClaimQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClaimQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).ClaimQuotaReserved), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClaimQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).ClaimQuotaReserved), ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) } // LoadDispatchRuntime mocks base method. -func (m *MockIExptItemDispatchRepo) LoadDispatchRuntime(arg0 context.Context, arg1, arg2, arg3 int64, arg4 int) (*repo.ExptDispatchRuntime, error) { +func (m *MockIExptItemDispatchRepo) LoadDispatchRuntime(ctx context.Context, spaceID, exptID, exptRunID int64, candidateLimit int) (*repo.ExptDispatchRuntime, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadDispatchRuntime", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "LoadDispatchRuntime", ctx, spaceID, exptID, exptRunID, candidateLimit) ret0, _ := ret[0].(*repo.ExptDispatchRuntime) ret1, _ := ret[1].(error) return ret0, ret1 } // LoadDispatchRuntime indicates an expected call of LoadDispatchRuntime. -func (mr *MockIExptItemDispatchRepoMockRecorder) LoadDispatchRuntime(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptItemDispatchRepoMockRecorder) LoadDispatchRuntime(ctx, spaceID, exptID, exptRunID, candidateLimit any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadDispatchRuntime", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).LoadDispatchRuntime), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadDispatchRuntime", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).LoadDispatchRuntime), ctx, spaceID, exptID, exptRunID, candidateLimit) } // MGetDispatchObservations mocks base method. -func (m *MockIExptItemDispatchRepo) MGetDispatchObservations(arg0 context.Context, arg1, arg2, arg3 int64, arg4 []int64) ([]*repo.ExptDispatchObservation, error) { +func (m *MockIExptItemDispatchRepo) MGetDispatchObservations(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]*repo.ExptDispatchObservation, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MGetDispatchObservations", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "MGetDispatchObservations", ctx, spaceID, exptID, exptRunID, itemIDs) ret0, _ := ret[0].([]*repo.ExptDispatchObservation) ret1, _ := ret[1].(error) return ret0, ret1 } // MGetDispatchObservations indicates an expected call of MGetDispatchObservations. -func (mr *MockIExptItemDispatchRepoMockRecorder) MGetDispatchObservations(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptItemDispatchRepoMockRecorder) MGetDispatchObservations(ctx, spaceID, exptID, exptRunID, itemIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetDispatchObservations", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).MGetDispatchObservations), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetDispatchObservations", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).MGetDispatchObservations), ctx, spaceID, exptID, exptRunID, itemIDs) } -// ResetQuotaReserved mocks base method. -func (m *MockIExptItemDispatchRepo) ResetQuotaReserved(arg0 context.Context, arg1, arg2, arg3 int64, arg4 []int64) ([]int64, error) { +// RequeueProcessingItem mocks base method. +func (m *MockIExptItemDispatchRepo) RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResetQuotaReserved", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].([]int64) + ret := m.ctrl.Call(m, "RequeueProcessingItem", ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) + ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } -// ResetQuotaReserved indicates an expected call of ResetQuotaReserved. -func (mr *MockIExptItemDispatchRepoMockRecorder) ResetQuotaReserved(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +// RequeueProcessingItem indicates an expected call of RequeueProcessingItem. +func (mr *MockIExptItemDispatchRepoMockRecorder) RequeueProcessingItem(ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).ResetQuotaReserved), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequeueProcessingItem", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).RequeueProcessingItem), ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) } -// RequeueProcessingItem mocks base method. -func (m *MockIExptItemDispatchRepo) RequeueProcessingItem(arg0 context.Context, arg1, arg2, arg3, arg4 int64) (bool, error) { +// ResetQuotaReserved mocks base method. +func (m *MockIExptItemDispatchRepo) ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RequeueProcessingItem", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "ResetQuotaReserved", ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) + ret0, _ := ret[0].([]int64) ret1, _ := ret[1].(error) return ret0, ret1 } -// RequeueProcessingItem indicates an expected call of RequeueProcessingItem. -func (mr *MockIExptItemDispatchRepoMockRecorder) RequeueProcessingItem(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +// ResetQuotaReserved indicates an expected call of ResetQuotaReserved. +func (mr *MockIExptItemDispatchRepoMockRecorder) ResetQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequeueProcessingItem", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).RequeueProcessingItem), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).ResetQuotaReserved), ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) } // StartReservedItem mocks base method. -func (m *MockIExptItemDispatchRepo) StartReservedItem(arg0 context.Context, arg1, arg2, arg3, arg4 int64) (bool, error) { +func (m *MockIExptItemDispatchRepo) StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (repo.StartReservedItemResult, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StartReservedItem", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "StartReservedItem", ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) + ret0, _ := ret[0].(repo.StartReservedItemResult) ret1, _ := ret[1].(error) return ret0, ret1 } // StartReservedItem indicates an expected call of StartReservedItem. -func (mr *MockIExptItemDispatchRepoMockRecorder) StartReservedItem(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptItemDispatchRepoMockRecorder) StartReservedItem(ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartReservedItem", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).StartReservedItem), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartReservedItem", reflect.TypeOf((*MockIExptItemDispatchRepo)(nil).StartReservedItem), ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) } diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_manual_retry.go b/backend/modules/evaluation/domain/repo/mocks/expt_manual_retry.go new file mode 100644 index 0000000000..c3bb28718e --- /dev/null +++ b/backend/modules/evaluation/domain/repo/mocks/expt_manual_retry.go @@ -0,0 +1,87 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptManualRetryRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_manual_retry.go -package=mocks . IExptManualRetryRepo +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptManualRetryRepo is a mock of IExptManualRetryRepo interface. +type MockIExptManualRetryRepo struct { + ctrl *gomock.Controller + recorder *MockIExptManualRetryRepoMockRecorder + isgomock struct{} +} + +// MockIExptManualRetryRepoMockRecorder is the mock recorder for MockIExptManualRetryRepo. +type MockIExptManualRetryRepoMockRecorder struct { + mock *MockIExptManualRetryRepo +} + +// NewMockIExptManualRetryRepo creates a new mock instance. +func NewMockIExptManualRetryRepo(ctrl *gomock.Controller) *MockIExptManualRetryRepo { + mock := &MockIExptManualRetryRepo{ctrl: ctrl} + mock.recorder = &MockIExptManualRetryRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptManualRetryRepo) EXPECT() *MockIExptManualRetryRepoMockRecorder { + return m.recorder +} + +// FindReceipt mocks base method. +func (m *MockIExptManualRetryRepo) FindReceipt(arg0 context.Context, arg1, arg2, arg3 int64, arg4, arg5 string) (*entity.ManualRetryReceipt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindReceipt", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(*entity.ManualRetryReceipt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindReceipt indicates an expected call of FindReceipt. +func (mr *MockIExptManualRetryRepoMockRecorder) FindReceipt(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindReceipt", reflect.TypeOf((*MockIExptManualRetryRepo)(nil).FindReceipt), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// PreflightTerminalItems mocks base method. +func (m *MockIExptManualRetryRepo) PreflightTerminalItems(arg0 context.Context, arg1 *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PreflightTerminalItems", arg0, arg1) + ret0, _ := ret[0].(*entity.ManualRetryPreparation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PreflightTerminalItems indicates an expected call of PreflightTerminalItems. +func (mr *MockIExptManualRetryRepoMockRecorder) PreflightTerminalItems(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreflightTerminalItems", reflect.TypeOf((*MockIExptManualRetryRepo)(nil).PreflightTerminalItems), arg0, arg1) +} + +// RequeueTerminalItems mocks base method. +func (m *MockIExptManualRetryRepo) RequeueTerminalItems(arg0 context.Context, arg1 *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryReceipt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RequeueTerminalItems", arg0, arg1) + ret0, _ := ret[0].(*entity.ManualRetryReceipt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RequeueTerminalItems indicates an expected call of RequeueTerminalItems. +func (mr *MockIExptManualRetryRepoMockRecorder) RequeueTerminalItems(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequeueTerminalItems", reflect.TypeOf((*MockIExptManualRetryRepo)(nil).RequeueTerminalItems), arg0, arg1) +} diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_manual_retry_filter.go b/backend/modules/evaluation/domain/repo/mocks/expt_manual_retry_filter.go new file mode 100644 index 0000000000..a6c0142f99 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/mocks/expt_manual_retry_filter.go @@ -0,0 +1,57 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptManualRetryFilterRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_manual_retry_filter.go -package=mocks . IExptManualRetryFilterRepo +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockIExptManualRetryFilterRepo is a mock of IExptManualRetryFilterRepo interface. +type MockIExptManualRetryFilterRepo struct { + ctrl *gomock.Controller + recorder *MockIExptManualRetryFilterRepoMockRecorder + isgomock struct{} +} + +// MockIExptManualRetryFilterRepoMockRecorder is the mock recorder for MockIExptManualRetryFilterRepo. +type MockIExptManualRetryFilterRepoMockRecorder struct { + mock *MockIExptManualRetryFilterRepo +} + +// NewMockIExptManualRetryFilterRepo creates a new mock instance. +func NewMockIExptManualRetryFilterRepo(ctrl *gomock.Controller) *MockIExptManualRetryFilterRepo { + mock := &MockIExptManualRetryFilterRepo{ctrl: ctrl} + mock.recorder = &MockIExptManualRetryFilterRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptManualRetryFilterRepo) EXPECT() *MockIExptManualRetryFilterRepoMockRecorder { + return m.recorder +} + +// ScanManualRetryFilterItems mocks base method. +func (m *MockIExptManualRetryFilterRepo) ScanManualRetryFilterItems(arg0 context.Context, arg1, arg2, arg3, arg4 int64, arg5 int) ([]int64, int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanManualRetryFilterItems", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].([]int64) + ret1, _ := ret[1].(int64) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ScanManualRetryFilterItems indicates an expected call of ScanManualRetryFilterItems. +func (mr *MockIExptManualRetryFilterRepoMockRecorder) ScanManualRetryFilterItems(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanManualRetryFilterItems", reflect.TypeOf((*MockIExptManualRetryFilterRepo)(nil).ScanManualRetryFilterItems), arg0, arg1, arg2, arg3, arg4, arg5) +} diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_materialized_filter.go b/backend/modules/evaluation/domain/repo/mocks/expt_materialized_filter.go new file mode 100644 index 0000000000..4cd3964c9c --- /dev/null +++ b/backend/modules/evaluation/domain/repo/mocks/expt_materialized_filter.go @@ -0,0 +1,57 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptMaterializedFilterRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_materialized_filter.go -package=mocks . IExptMaterializedFilterRepo +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptMaterializedFilterRepo is a mock of IExptMaterializedFilterRepo interface. +type MockIExptMaterializedFilterRepo struct { + ctrl *gomock.Controller + recorder *MockIExptMaterializedFilterRepoMockRecorder + isgomock struct{} +} + +// MockIExptMaterializedFilterRepoMockRecorder is the mock recorder for MockIExptMaterializedFilterRepo. +type MockIExptMaterializedFilterRepoMockRecorder struct { + mock *MockIExptMaterializedFilterRepo +} + +// NewMockIExptMaterializedFilterRepo creates a new mock instance. +func NewMockIExptMaterializedFilterRepo(ctrl *gomock.Controller) *MockIExptMaterializedFilterRepo { + mock := &MockIExptMaterializedFilterRepo{ctrl: ctrl} + mock.recorder = &MockIExptMaterializedFilterRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptMaterializedFilterRepo) EXPECT() *MockIExptMaterializedFilterRepoMockRecorder { + return m.recorder +} + +// GetMaterializedByExptIDItemIDs mocks base method. +func (m *MockIExptMaterializedFilterRepo) GetMaterializedByExptIDItemIDs(arg0 context.Context, arg1, arg2, arg3 string, arg4 []string) ([]*entity.ExptTurnResultFilterEntity, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaterializedByExptIDItemIDs", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].([]*entity.ExptTurnResultFilterEntity) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMaterializedByExptIDItemIDs indicates an expected call of GetMaterializedByExptIDItemIDs. +func (mr *MockIExptMaterializedFilterRepoMockRecorder) GetMaterializedByExptIDItemIDs(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaterializedByExptIDItemIDs", reflect.TypeOf((*MockIExptMaterializedFilterRepo)(nil).GetMaterializedByExptIDItemIDs), arg0, arg1, arg2, arg3, arg4) +} diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_result_publish.go b/backend/modules/evaluation/domain/repo/mocks/expt_result_publish.go new file mode 100644 index 0000000000..5863e73c47 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/mocks/expt_result_publish.go @@ -0,0 +1,87 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptResultPublishRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_result_publish.go -package=mocks . IExptResultPublishRepo +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptResultPublishRepo is a mock of IExptResultPublishRepo interface. +type MockIExptResultPublishRepo struct { + ctrl *gomock.Controller + recorder *MockIExptResultPublishRepoMockRecorder + isgomock struct{} +} + +// MockIExptResultPublishRepoMockRecorder is the mock recorder for MockIExptResultPublishRepo. +type MockIExptResultPublishRepoMockRecorder struct { + mock *MockIExptResultPublishRepo +} + +// NewMockIExptResultPublishRepo creates a new mock instance. +func NewMockIExptResultPublishRepo(ctrl *gomock.Controller) *MockIExptResultPublishRepo { + mock := &MockIExptResultPublishRepo{ctrl: ctrl} + mock.recorder = &MockIExptResultPublishRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptResultPublishRepo) EXPECT() *MockIExptResultPublishRepoMockRecorder { + return m.recorder +} + +// CommitItemResult mocks base method. +func (m *MockIExptResultPublishRepo) CommitItemResult(arg0 context.Context, arg1 *entity.ExptItemResultPublication) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CommitItemResult", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CommitItemResult indicates an expected call of CommitItemResult. +func (mr *MockIExptResultPublishRepoMockRecorder) CommitItemResult(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommitItemResult", reflect.TypeOf((*MockIExptResultPublishRepo)(nil).CommitItemResult), arg0, arg1) +} + +// GetItemResultPublishSource mocks base method. +func (m *MockIExptResultPublishRepo) GetItemResultPublishSource(arg0 context.Context, arg1, arg2, arg3, arg4 int64) (*entity.ExptItemResultPublishSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetItemResultPublishSource", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(*entity.ExptItemResultPublishSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetItemResultPublishSource indicates an expected call of GetItemResultPublishSource. +func (mr *MockIExptResultPublishRepoMockRecorder) GetItemResultPublishSource(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetItemResultPublishSource", reflect.TypeOf((*MockIExptResultPublishRepo)(nil).GetItemResultPublishSource), arg0, arg1, arg2, arg3, arg4) +} + +// IsItemResultPublishSourceCurrent mocks base method. +func (m *MockIExptResultPublishRepo) IsItemResultPublishSourceCurrent(arg0 context.Context, arg1 *entity.ExptItemResultPublishSource) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsItemResultPublishSourceCurrent", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsItemResultPublishSourceCurrent indicates an expected call of IsItemResultPublishSourceCurrent. +func (mr *MockIExptResultPublishRepoMockRecorder) IsItemResultPublishSourceCurrent(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsItemResultPublishSourceCurrent", reflect.TypeOf((*MockIExptResultPublishRepo)(nil).IsItemResultPublishSourceCurrent), arg0, arg1) +} diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_run_finish.go b/backend/modules/evaluation/domain/repo/mocks/expt_run_finish.go new file mode 100644 index 0000000000..0a68e247b6 --- /dev/null +++ b/backend/modules/evaluation/domain/repo/mocks/expt_run_finish.go @@ -0,0 +1,57 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptRunFinishRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_run_finish.go -package=mocks . IExptRunFinishRepo +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptRunFinishRepo is a mock of IExptRunFinishRepo interface. +type MockIExptRunFinishRepo struct { + ctrl *gomock.Controller + recorder *MockIExptRunFinishRepoMockRecorder + isgomock struct{} +} + +// MockIExptRunFinishRepoMockRecorder is the mock recorder for MockIExptRunFinishRepo. +type MockIExptRunFinishRepoMockRecorder struct { + mock *MockIExptRunFinishRepo +} + +// NewMockIExptRunFinishRepo creates a new mock instance. +func NewMockIExptRunFinishRepo(ctrl *gomock.Controller) *MockIExptRunFinishRepo { + mock := &MockIExptRunFinishRepo{ctrl: ctrl} + mock.recorder = &MockIExptRunFinishRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptRunFinishRepo) EXPECT() *MockIExptRunFinishRepoMockRecorder { + return m.recorder +} + +// FinishCurrentRun mocks base method. +func (m *MockIExptRunFinishRepo) FinishCurrentRun(arg0 context.Context, arg1 *entity.ExptRunFinishRequest) (*entity.ExptRunFinishResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinishCurrentRun", arg0, arg1) + ret0, _ := ret[0].(*entity.ExptRunFinishResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FinishCurrentRun indicates an expected call of FinishCurrentRun. +func (mr *MockIExptRunFinishRepoMockRecorder) FinishCurrentRun(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishCurrentRun", reflect.TypeOf((*MockIExptRunFinishRepo)(nil).FinishCurrentRun), arg0, arg1) +} diff --git a/backend/modules/evaluation/domain/repo/mocks/expt_run_termination.go b/backend/modules/evaluation/domain/repo/mocks/expt_run_termination.go new file mode 100644 index 0000000000..c611b10c6e --- /dev/null +++ b/backend/modules/evaluation/domain/repo/mocks/expt_run_termination.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo (interfaces: IExptRunTerminationRepo) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_run_termination.go -package=mocks . IExptRunTerminationRepo +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptRunTerminationRepo is a mock of IExptRunTerminationRepo interface. +type MockIExptRunTerminationRepo struct { + ctrl *gomock.Controller + recorder *MockIExptRunTerminationRepoMockRecorder + isgomock struct{} +} + +// MockIExptRunTerminationRepoMockRecorder is the mock recorder for MockIExptRunTerminationRepo. +type MockIExptRunTerminationRepoMockRecorder struct { + mock *MockIExptRunTerminationRepo +} + +// NewMockIExptRunTerminationRepo creates a new mock instance. +func NewMockIExptRunTerminationRepo(ctrl *gomock.Controller) *MockIExptRunTerminationRepo { + mock := &MockIExptRunTerminationRepo{ctrl: ctrl} + mock.recorder = &MockIExptRunTerminationRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptRunTerminationRepo) EXPECT() *MockIExptRunTerminationRepoMockRecorder { + return m.recorder +} + +// GetTerminationItemSource mocks base method. +func (m *MockIExptRunTerminationRepo) GetTerminationItemSource(arg0 context.Context, arg1, arg2, arg3, arg4 int64) (*entity.ExptCurrentItemRunSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTerminationItemSource", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(*entity.ExptCurrentItemRunSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTerminationItemSource indicates an expected call of GetTerminationItemSource. +func (mr *MockIExptRunTerminationRepoMockRecorder) GetTerminationItemSource(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTerminationItemSource", reflect.TypeOf((*MockIExptRunTerminationRepo)(nil).GetTerminationItemSource), arg0, arg1, arg2, arg3, arg4) +} + +// TerminateCurrentRun mocks base method. +func (m *MockIExptRunTerminationRepo) TerminateCurrentRun(arg0 context.Context, arg1 *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TerminateCurrentRun", arg0, arg1) + ret0, _ := ret[0].(*entity.ExptRunTerminationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TerminateCurrentRun indicates an expected call of TerminateCurrentRun. +func (mr *MockIExptRunTerminationRepoMockRecorder) TerminateCurrentRun(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateCurrentRun", reflect.TypeOf((*MockIExptRunTerminationRepo)(nil).TerminateCurrentRun), arg0, arg1) +} diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index 15c06dadfe..ad054fa517 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1371,10 +1371,13 @@ func (e *EvaluatorServiceImpl) ArmEvaluatorResume(ctx context.Context, recordID if record == nil { return errorx.NewByCode(errno.EvaluatorRecordNotFoundCode, errorx.WithExtraMsg("evaluator record not found")) } + if record.ID != recordID || record.TurnID < 0 { + return errorx.New("evaluator resume source unavailable, invoke_id: %v", recordID) + } if record.Status == entity.EvaluatorRunStatusAsyncInvoking { return nil } - return e.publishEvaluatorResumeEvent(ctx, actx.Event) + return e.publishEvaluatorResumeEvent(ctx, actx.Event.CloneWithResumeSource("evaluator", record.ID, record.TurnID)) } func (e *EvaluatorServiceImpl) publishEvaluatorResumeEvent(ctx context.Context, event *entity.ExptItemEvalEvent) error { @@ -1393,7 +1396,7 @@ func (e *EvaluatorServiceImpl) publishEvaluatorResumeEvent(ctx context.Context, case <-timer.C: } } - lastErr = e.exptEventPublisher.PublishExptRecordEvalEvent(ctx, event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { + lastErr = e.exptEventPublisher.PublishExptRecordEvalEvent(ctx, event.Clone(), gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { event.AsyncEvaluatorReportTrigger = true }) if lastErr == nil { diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index 6ed912b172..ba8afe68e1 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4975,12 +4975,15 @@ func TestEvaluatorServiceImpl_ArmEvaluatorResume(t *testing.T) { if tt.withEvent { recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).DoAndReturn(func(ctx context.Context, _ int64, _ bool, _ ...entity.GetEvaluatorRecordOptionFn) (*entity.EvaluatorRecord, error) { assert.True(t, contexts.CtxWriteDB(ctx)) - return &entity.EvaluatorRecord{ID: 100, Status: tt.recordStatus}, nil + return &entity.EvaluatorRecord{ID: 100, TurnID: 909, Status: tt.recordStatus}, nil }) } if tt.wantPublished { - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).DoAndReturn( + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, ev *entity.ExptItemEvalEvent, _ *time.Duration, modify func(*entity.ExptItemEvalEvent)) error { + assert.NotSame(t, event, ev) + require.NotNil(t, ev.ResumeSource) + assert.Equal(t, int64(909), ev.ResumeSource.TurnID) modify(ev) assert.True(t, ev.AsyncEvaluatorReportTrigger) return nil @@ -5001,10 +5004,10 @@ func TestEvaluatorServiceImpl_ArmEvaluatorResume_RetriesPublish(t *testing.T) { publisher := eventmocks.NewMockExptEventPublisher(ctrl) event := &entity.ExptItemEvalEvent{ExptID: 1} asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: event, ResumeReady: true}, nil) - recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, Status: entity.EvaluatorRunStatusSuccess}, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, TurnID: 909, Status: entity.EvaluatorRunStatusSuccess}, nil) gomock.InOrder( - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(errors.New("mq unavailable")), - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(nil), + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("mq unavailable")), + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), ) s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo, exptEventPublisher: publisher} require.NoError(t, s.ArmEvaluatorResume(context.Background(), 100)) diff --git a/backend/modules/evaluation/domain/service/expt_cancel_boundary_test.go b/backend/modules/evaluation/domain/service/expt_cancel_boundary_test.go new file mode 100644 index 0000000000..59d1976b91 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_cancel_boundary_test.go @@ -0,0 +1,572 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/infra/lock" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + confmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "go.uber.org/mock/gomock" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + exptrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + exptmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestRunCancelAtomicBeginRollsBackBothStates(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + before := f.snapshot() + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("cancel_experiment_fault", func(tx *gorm.DB) { + if tx.Statement.Table == "experiment" { + _ = tx.AddError(errors.New("cancel experiment write fault")) + } + })) + err := p.manager.SetExptTerminating(context.Background(), f.expt, f.run, f.space, p.event.Session) + require.Error(t, err) + require.Equal(t, before, f.snapshot(), "run and experiment must roll back together") +} + +func TestRunCancelOldRunCannotTerminateReplacement(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + replacement := f.run + 100 + require.NoError(t, f.conn.Create(&model.ExptRunLog{ID: replacement, SpaceID: f.space, ExptID: f.expt, ExptRunID: replacement, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit))}).Error) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("latest_run_id", replacement).Error) + before := f.snapshot() + require.Error(t, p.manager.SetExptTerminating(context.Background(), f.expt, f.run, f.space, p.event.Session)) + require.Equal(t, before, f.snapshot(), "an old cancellation cannot change R2 or R1") +} + +func TestTerminateRowAtomicLogsRollback(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + p.manager.turnResultRepo = exptrepo.NewExptTurnResultRepo(p.manager.idgenerator, exptmysql.NewExptTurnResultDAO(p.f.provider), exptmysql.NewExptTurnEvaluatorResultRefDAO(p.f.provider)) + f := p.f + before := f.snapshot() + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("cancel_item_fault", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_result_run_log" { + _ = tx.AddError(errors.New("cancel item write fault")) + } + })) + err := p.manager.TerminateItems(context.Background(), f.expt, f.run, f.space, []int64{f.item}, p.event.Session) + require.Error(t, err) + require.Equal(t, before, f.snapshot(), "turn and item logs must commit as a pair") +} + +func TestRunCancelAcceptanceReceiptSurvivesAndReplayDoesNotRewrite(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + p.publishSuccess(t, false) + receipt, err := p.manager.RetryInCurrentRun(context.Background(), &entity.RetryInCurrentRunRequest{SpaceID: p.f.space, ExptID: p.f.expt, ExpectedRunID: p.f.run, RequestID: "9f6fe7ea-7db6-4d16-8f5b-2d1344556677", Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{p.f.item}, Session: p.event.Session}) + require.NoError(t, err) + require.Equal(t, int32(1), receipt.AcceptedCount) + require.NoError(t, p.manager.SetExptTerminating(context.Background(), p.f.expt, p.f.run, p.f.space, p.event.Session)) + applied, err := p.manager.TerminateCurrentRun(context.Background(), p.f.expt, p.f.run, p.f.space, p.event.Session, nil, entity.WithStatus(entity.ExptStatus_Terminated), entity.NoAggrCalculate()) + require.NoError(t, err) + require.True(t, applied) + before := p.f.snapshot() + replay, err := p.manager.RetryInCurrentRun(context.Background(), &entity.RetryInCurrentRunRequest{SpaceID: p.f.space, ExptID: p.f.expt, ExpectedRunID: p.f.run, RequestID: receipt.RequestID, Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{p.f.item}, Session: p.event.Session}) + require.NoError(t, err) + require.Equal(t, receipt.RequestID, replay.RequestID) + require.Equal(t, receipt.AcceptedCount, replay.AcceptedCount) + require.Equal(t, receipt.PayloadHash, replay.PayloadHash) + require.Equal(t, receipt.SnapshotAtMS, replay.SnapshotAtMS) + require.Equal(t, before, p.f.snapshot()) + _, err = p.manager.TerminateCurrentRun(context.Background(), p.f.expt, p.f.run, p.f.space, p.event.Session, nil, entity.WithStatus(entity.ExptStatus_Failed)) + require.NoError(t, err) + require.Equal(t, before, p.f.snapshot(), "terminal replay preserves the original result and receipt") + var experiment model.Experiment + require.NoError(t, p.f.conn.Where("space_id=?", p.f.space).Take(&experiment).Error) + require.Equal(t, int32(entity.ExptStatus_Terminated), experiment.Status) + var item model.ExptItemResultRunLog + require.NoError(t, p.f.conn.Where("space_id=?", p.f.space).Take(&item).Error) + require.Equal(t, int32(entity.ItemRunState_Terminal), item.Status) +} + +func TestTerminateRowPublicationDoesNotCountTwice(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + require.NoError(t, p.manager.TerminateItems(context.Background(), p.f.expt, p.f.run, p.f.space, []int64{p.f.item}, p.event.Session)) + var stats model.ExptStats + require.NoError(t, p.f.conn.Where("space_id=?", p.f.space).Take(&stats).Error) + require.Zero(t, stats.PendingCnt) + require.Equal(t, int32(1), stats.TerminatedCnt) + _, err := p.result.RecordItemRunLogs(context.Background(), p.f.expt, p.f.run, p.f.item, p.f.space, p.expt) + require.NoError(t, err) + require.NoError(t, p.f.conn.Where("space_id=?", p.f.space).Take(&stats).Error) + require.Zero(t, stats.PendingCnt) + require.Equal(t, int32(1), stats.TerminatedCnt) + before := p.f.snapshot() + require.NoError(t, p.manager.TerminateItems(context.Background(), p.f.expt, p.f.run, p.f.space, []int64{p.f.item}, p.event.Session)) + require.Equal(t, before, p.f.snapshot()) + result, err := p.manager.FinishCurrentRun(context.Background(), p.event) + require.NoError(t, err) + require.Equal(t, entity.ExptStatus_Success, result.Experiment.Status) +} + +func TestRunCancelSchedulerOldErrorDoesNotEndNewRun(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + replacement := p.f.run + 100 + require.NoError(t, p.f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", p.f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + require.NoError(t, p.f.conn.Create(&model.ExptRunLog{ID: replacement, SpaceID: p.f.space, ExptID: p.f.expt, ExptRunID: replacement, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit))}).Error) + require.NoError(t, p.f.conn.Model(&model.Experiment{}).Where("space_id=?", p.f.space).Update("latest_run_id", replacement).Error) + before := p.f.snapshot() + scheduler := &ExptSchedulerImpl{Manager: p.manager} + err := scheduler.HandleEventErr(func(context.Context, *entity.ExptScheduleEvent) error { return errors.New("business failure") })(context.Background(), p.event) + require.NoError(t, err) + require.Equal(t, before, p.f.snapshot()) + require.Zero(t, p.resources.Load()) + require.Zero(t, p.notifications.Load()) +} + +func TestCleanupUnscheduledOldRunPreservesSuccess(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + require.NoError(t, p.f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", p.f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + require.NoError(t, p.f.conn.Model(&model.Experiment{}).Where("space_id=?", p.f.space).Updates(map[string]any{"status": int32(entity.ExptStatus_Success), "end_at": time.Now()}).Error) + before := p.f.snapshot() + p.manager.cleanupUnscheduledRun(context.Background(), p.f.expt, p.f.run, p.f.space, errors.New("late setup error")) + require.Equal(t, before, p.f.snapshot(), "late initialization cleanup must preserve terminal success") +} + +type cancelHookLocker struct { + lock.ILocker + hook func(string) +} + +func (l *cancelHookLocker) LockWithRenew(ctx context.Context, key string, ttl, hold time.Duration) (bool, context.Context, func(), error) { + if l.hook != nil { + l.hook(key) + } + return l.ILocker.LockWithRenew(ctx, key, ttl, hold) +} + +func TestDebtOriginalSourceRecheckedAfterItemLeaseReleased(t *testing.T) { + for _, replace := range []bool{false, true} { + t.Run(fmt.Sprint(replace), func(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + source := f.source() + event := &entity.ExptItemEvalEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, EvalSetItemID: f.item, ExptRunMode: source.Current.Mode} + config := confmocks.NewMockIConfiger(gomock.NewController(t)) + config.EXPECT().GetErrRetryConf(gomock.Any(), f.space, gomock.Any()).Return(&entity.RetryConf{IsInDebt: true}) + var once sync.Once + factory := p.factory + p.manager.manualRetryLocker = func() lock.ILocker { + return &cancelHookLocker{ILocker: factory(), hook: func(key string) { + if replace && strings.HasPrefix(key, "expt_run_exec_lock:") { + once.Do(func() { f.token("aa6fe7ea-7db6-4d16-8f5b-2d1344556677") }) + } + }} + } + svc := &ExptItemEventEvalServiceImpl{experimentRepo: p.manager.exptRepo, manager: p.manager, configer: config, itemLocker: func() lock.ILocker { return factory() }, exptItemResultRepo: f.itemRepo()} + before := f.snapshot() + err := svc.handleCurrentEventErr(context.Background(), event, errors.New("debt"), f.port, ¤tItemExecution{Source: source}) + require.NoError(t, err) + if replace { + after := f.snapshot() + for table, value := range before { + if table != "expt_item_result" { + require.Equal(t, value, after[table], table) + } + } + require.Zero(t, p.resources.Load()) + require.Zero(t, p.notifications.Load()) + } else { + var row model.Experiment + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&row).Error) + require.Equal(t, int32(entity.ExptStatus_Terminated), row.Status) + } + }) + } +} + +func TestRunCancelHeldWorkerItemLeaseDoesNotWaitForRun(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + worker := p.factory() + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", p.f.expt, p.f.item) + locked, _, cancel, err := worker.LockWithRenew(context.Background(), key, time.Second*5, time.Minute) + require.NoError(t, err) + require.True(t, locked) + before := p.f.snapshot() + started := time.Now() + err = p.manager.TerminateItems(context.Background(), p.f.expt, p.f.run, p.f.space, []int64{p.f.item}, p.event.Session) + require.True(t, errno.NeedMQRetry(err)) + require.Less(t, time.Since(started), 8*time.Second) + require.Equal(t, before, p.f.snapshot()) + cancel() + require.NoError(t, p.manager.TerminateItems(context.Background(), p.f.expt, p.f.run, p.f.space, []int64{p.f.item}, p.event.Session)) +} + +func TestRunCancelFirstRejectsFreshRequest(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + p.publishSuccess(t, false) + require.NoError(t, p.manager.SetExptTerminating(context.Background(), p.f.expt, p.f.run, p.f.space, p.event.Session)) + before := p.f.snapshot() + _, err := p.manager.RetryInCurrentRun(context.Background(), &entity.RetryInCurrentRunRequest{SpaceID: p.f.space, ExptID: p.f.expt, ExpectedRunID: p.f.run, RequestID: "aa6fe7ea-7db6-4d16-8f5b-2d1344556677", Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{p.f.item}, Session: p.event.Session}) + require.Error(t, err) + require.Equal(t, before, p.f.snapshot()) +} + +func TestTerminateRowOldSnapshotDoesNotTouchReadmittedRequest(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + original := p.factory + var snapshot map[string]string + var once sync.Once + p.manager.manualRetryLocker = func() lock.ILocker { + return &cancelHookLocker{ILocker: original(), hook: func(key string) { + if strings.HasPrefix(key, "expt_run_exec_lock:") { + once.Do(func() { f.token("aa6fe7ea-7db6-4d16-8f5b-2d1344556677"); snapshot = f.snapshot() }) + } + }} + } + require.NoError(t, p.manager.TerminateItems(context.Background(), f.expt, f.run, f.space, []int64{f.item}, p.event.Session)) + require.Equal(t, snapshot, f.snapshot()) + require.Zero(t, p.resources.Load()) +} + +type cancelResourceGuard struct { + *fakeGuard + check func(context.Context, string) +} + +func (g *cancelResourceGuard) Release(ctx context.Context, scope string, run, item int64, token, reason string) error { + g.check(ctx, token) + return g.fakeGuard.Release(ctx, scope, run, item, token, reason) +} + +func TestRunCancelResourcesSurviveParentCancelAfterCommit(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + token := "aa6fe7ea-7db6-4d16-8f5b-2d1344556677" + f.token(token) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + // Cancel after the SQL port returned, before lifecycle cleanup starts. + port := p.manager.runFinishRepo + p.manager.runFinishRepo = &cancelAfterCommitPort{IExptRunFinishRepo: port, termination: port.(repo.IExptRunTerminationRepo), cancel: cancel} + released := &fakeGuard{} + p.manager.centralGuard = &cancelResourceGuard{fakeGuard: released, check: func(ctx context.Context, observed string) { + require.NoError(t, ctx.Err()) + require.Equal(t, token, observed) + deadline, ok := ctx.Deadline() + require.True(t, ok) + require.LessOrEqual(t, time.Until(deadline), 8*time.Second) + }} + _, err := p.manager.TerminateCurrentRun(parent, f.expt, f.run, f.space, p.event.Session, nil, entity.NoAggrCalculate()) + require.True(t, errno.NeedMQRetry(err)) + require.Len(t, released.releases(), 1) + require.Equal(t, int32(1), p.resources.Load()) + require.Zero(t, p.notifications.Load()) + var row model.Experiment + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&row).Error) + require.Equal(t, int32(entity.ExptStatus_Terminated), row.Status) +} + +type cancelAfterCommitPort struct { + repo.IExptRunFinishRepo + termination repo.IExptRunTerminationRepo + cancel context.CancelFunc +} + +func (p *cancelAfterCommitPort) GetTerminationItemSource(ctx context.Context, space, expt, run, item int64) (*entity.ExptCurrentItemRunSource, error) { + return p.termination.GetTerminationItemSource(ctx, space, expt, run, item) +} + +func (p *cancelAfterCommitPort) TerminateCurrentRun(ctx context.Context, r *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + result, err := p.termination.TerminateCurrentRun(ctx, r) + if err == nil && result.Applied && r.Finalize { + p.cancel() + } + return result, err +} + +func TestNewRunRealInitializersAndResetClearOnlyManualExt(t *testing.T) { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems, entity.EvaluationModeFailRetry} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + f.token("aa6fe7ea-7db6-4d16-8f5b-2d1344556677") + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.ExptStatus_Success), "end_at": time.Now()}).Error) + oldLog := f.snapshot()["expt_run_log"] + _, err := p.manager.mutex.UnlockWithValue(context.Background(), p.manager.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run)) + require.NoError(t, err) + next := f.run + 100 + ctrl := gomock.NewController(t) + ids := idmocks.NewMockIIDGenerator(ctrl) + ids.EXPECT().GenID(gomock.Any()).Return(next, nil).AnyTimes() + var seq atomic.Int64 + ids.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, n int) ([]int64, error) { + out := make([]int64, n) + for i := range out { + out[i] = f.base + 12000 + seq.Add(1) + } + return out, nil + }).AnyTimes() + p.manager.idgenerator = ids + metric := metricsmocks.NewMockExptMetric(ctrl) + metric.EXPECT().EmitExptExecRun(f.space, int64(mode)) + p.manager.mtr = metric + if mode == entity.EvaluationModeRetryItems { + run, joined, err := p.manager.LogRetryItemsRun(context.Background(), f.expt, mode, f.space, []int64{f.item}, p.event.Session) + require.NoError(t, err) + require.False(t, joined) + require.Equal(t, next, run) + } else { + require.NoError(t, p.manager.LogRun(context.Background(), f.expt, next, mode, f.space, []int64{f.item}, p.event.Session)) + } + var item model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&item).Error) + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + require.NoError(t, err) + require.Contains(t, ext, entity.ManualRetryExtKey, "LogRun must not clear sources before migration") + event := *p.event + event.ExptRunID = next + event.ExptRunMode = mode + deps := retryItemResetDeps{exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), idgenerator: ids} + _, err = resetRetryRunLogsForItems(context.Background(), deps, &event, []*entity.EvaluationSetItem{{ItemID: f.item, ItemVersionID: &f.version, Turns: []*entity.Turn{{ID: 0}}}}, map[int64]int64{f.item: f.version}) + require.NoError(t, err) + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&item).Error) + ext, err = entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + require.NoError(t, err) + require.Equal(t, map[string]string{"other": "keep"}, ext) + original, err := p.manager.runLogRepo.Get(context.Background(), f.expt, f.run) + require.NoError(t, err) + require.Equal(t, int64(entity.ExptStatus_Success), original.Status) + var oldRows []map[string]any + require.NoError(t, f.conn.Table("expt_run_log").Where("space_id=? AND expt_run_id=?", f.space, f.run).Find(&oldRows).Error) + encoded, err := json.Marshal(oldRows) + require.NoError(t, err) + require.Equal(t, oldLog, string(encoded)) + run, err := p.manager.runLogRepo.Get(context.Background(), f.expt, next) + require.NoError(t, err) + for _, chunk := range run.ItemIds { + require.Nil(t, chunk.ManualRetryReceipt) + } + }) + } +} + +func TestRunCancelAscendingItemLockOrder(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + for _, offset := range []int64{1, 2} { + require.NoError(t, f.conn.Create(&model.ExptItemResult{ID: f.base + 13000 + offset, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + offset, ItemVersionID: 0, Status: int32(entity.ItemRunState_Queueing)}).Error) + require.NoError(t, f.conn.Create(&model.ExptItemResultRunLog{ID: f.base + 14000 + offset, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + offset, ItemVersionID: 0, Status: int32(entity.ItemRunState_Queueing), ResultState: gptr.Of(int32(entity.ExptItemResultStateDefault))}).Error) + } + factory := p.factory + keys := []string{} + p.manager.manualRetryLocker = func() lock.ILocker { + return &cancelHookLocker{ILocker: factory(), hook: func(key string) { keys = append(keys, key) }} + } + require.NoError(t, p.manager.TerminateItems(context.Background(), f.expt, f.run, f.space, []int64{f.item + 2, f.item, f.item + 1, f.item}, p.event.Session)) + require.Equal(t, []string{fmt.Sprintf("expt_run_exec_lock:%d:%d", f.expt, f.run), fmt.Sprintf("expt_item_eval_run_lock:%d:%d", f.expt, f.item), fmt.Sprintf("expt_item_eval_run_lock:%d:%d", f.expt, f.item+1), fmt.Sprintf("expt_item_eval_run_lock:%d:%d", f.expt, f.item+2)}, keys) +} + +type cancelLostLeaseLocker struct{ lock.ILocker } + +func (l *cancelLostLeaseLocker) LockWithRenew(ctx context.Context, key string, ttl, hold time.Duration) (bool, context.Context, func(), error) { + child, cancel := context.WithCancel(ctx) + cancel() + return true, child, func() {}, nil +} + +func TestRunCancelLostLeaseHasZeroStateWrites(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + before := p.f.snapshot() + p.manager.manualRetryLocker = func() lock.ILocker { return &cancelLostLeaseLocker{ILocker: p.factory()} } + require.True(t, errno.NeedMQRetry(p.manager.SetExptTerminating(context.Background(), p.f.expt, p.f.run, p.f.space, p.event.Session))) + require.Equal(t, before, p.f.snapshot()) + require.Zero(t, p.resources.Load()) +} + +func TestRunCancelR2LifecycleOwnerPreventsAnyCleanup(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + finished := *p.expt + finished.Status = entity.ExptStatus_Terminated + _, err := p.manager.mutex.UnlockWithValue(context.Background(), p.manager.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run)) + require.NoError(t, err) + replacement := p.factory() + locked, _, err := replacement.BackoffLockWithValue(context.Background(), p.manager.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run+100), time.Minute, time.Millisecond) + require.NoError(t, err) + require.True(t, locked) + source := f.source() + before := f.snapshot() + err = p.manager.cleanupTerminatedRun(context.Background(), p.event, &finished, []*entity.ExptCurrentItemRunSource{source}, &entity.CompleteExptOption{}) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, before, f.snapshot()) + require.Zero(t, p.resources.Load()) + require.Zero(t, p.notifications.Load()) +} + +func TestNewRunResetReleasesOriginalTerminalManualQuotaBeforeMigration(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + token := "aa6fe7ea-7db6-4d16-8f5b-2d1344556677" + f.token(token) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + guard := &fakeGuard{} + p.manager.centralGuard = guard + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Terminated)).Error) + _, err := p.manager.mutex.UnlockWithValue(context.Background(), p.manager.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run)) + require.NoError(t, err) + metric := metricsmocks.NewMockExptMetric(gomock.NewController(t)) + metric.EXPECT().EmitExptExecRun(f.space, int64(entity.EvaluationModeRetryAll)) + p.manager.mtr = metric + require.NoError(t, p.manager.LogRun(context.Background(), f.expt, f.run+100, entity.EvaluationModeRetryAll, f.space, []int64{f.item}, p.event.Session)) + calls := guard.releases() + require.Len(t, calls, 1) + require.Equal(t, f.run, calls[0].RunID) + require.Equal(t, token, calls[0].RequestID) +} + +func TestNewRunReleasesResidualTerminalQuota(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + f.token("aa6fe7ea-7db6-4d16-8f5b-2d1344556677") + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope, "status": int32(entity.ExptStatus_Terminated), "end_at": time.Now()}).Error) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Terminated)).Error) + for _, table := range []string{"expt_item_result", "expt_item_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=?", f.space).Update("status", int32(entity.ItemRunState_Terminal)).Error) + } + guard := &fakeGuard{} + p.manager.centralGuard = guard + _, err := p.manager.mutex.UnlockWithValue(context.Background(), p.manager.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run)) + require.NoError(t, err) + metric := metricsmocks.NewMockExptMetric(gomock.NewController(t)) + metric.EXPECT().EmitExptExecRun(f.space, int64(entity.EvaluationModeRetryAll)) + p.manager.mtr = metric + require.NoError(t, p.manager.LogRun(context.Background(), f.expt, f.run+100, entity.EvaluationModeRetryAll, f.space, []int64{f.item}, p.event.Session)) + require.Len(t, guard.releases(), 1) + require.Equal(t, "aa6fe7ea-7db6-4d16-8f5b-2d1344556677", guard.releases()[0].RequestID) +} + +func TestRunCancelSandboxUsesCapturedOriginalRecordIDs(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + original := f.base + 17000 + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("target_result_id", original).Error) + target := svcmocks.NewMockIEvalTargetService(gomock.NewController(t)) + target.EXPECT().TerminateAsyncRecordsAndDestroySandbox(gomock.Any(), f.space, []int64{original}, int32(errno.AsyncEvalTargetTerminatedCode), gomock.Any(), false).Do(func(ctx context.Context, _ int64, ids []int64, _ int32, _ string, _ bool, _ ...bool) { + require.NoError(t, ctx.Err()) + require.Equal(t, []int64{original}, ids) + }) + p.manager.evalTargetService = target + _, err := p.manager.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, p.event.Session, nil, entity.NoAggrCalculate()) + require.NoError(t, err) + var log model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&log).Error) + require.Equal(t, original, log.TargetResultID) +} + +func TestRunCancelFailedItemStillCleansOriginalAsyncRecord(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + original := f.base + 18000 + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("target_result_id", original).Error) + for _, table := range []string{"expt_item_result", "expt_item_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=?", f.space).Update("status", int32(entity.ItemRunState_Fail)).Error) + } + target := svcmocks.NewMockIEvalTargetService(gomock.NewController(t)) + target.EXPECT().TerminateAsyncRecordsAndDestroySandbox(gomock.Any(), f.space, []int64{original}, int32(errno.AsyncEvalTargetTerminatedCode), gomock.Any(), false) + p.manager.evalTargetService = target + _, err := p.manager.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, p.event.Session, nil, entity.WithStatus(entity.ExptStatus_Failed), entity.NoAggrCalculate()) + require.NoError(t, err) +} + +func TestNewRunActualRetryItemsResetReleasesHistoricalSourceToken(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + historical := f.run - 100 + next := f.run + 100 + token := "aa6fe7ea-7db6-4d16-8f5b-2d1344556677" + raw, err := json.Marshal(&entity.ManualRetryContext{RunID: historical, RequestID: token, Mode: entity.EvaluationModeRetryItems, Operator: "old", AcceptedAtMS: 1}) + require.NoError(t, err) + ext, err := json.Marshal(map[string]string{"other": "keep", entity.ManualRetryExtKey: string(raw)}) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=?", f.space).Updates(map[string]any{"expt_run_id": historical, "status": int32(entity.ItemRunState_Fail), "ext": ext}).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=?", f.space).Update("expt_run_id", historical).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=?", f.space).Updates(map[string]any{"expt_run_id": historical, "status": int32(entity.ItemRunState_Fail), "result_state": int32(entity.ExptItemResultStateResulted)}).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("expt_run_id", historical).Error) + require.NoError(t, f.conn.Create(&model.ExptRunLog{ID: historical, SpaceID: f.space, ExptID: f.expt, ExptRunID: historical, Status: gptr.Of(int64(entity.ExptStatus_Success)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit))}).Error) + require.NoError(t, f.conn.Create(&model.ExptRunLog{ID: next, SpaceID: f.space, ExptID: f.expt, ExptRunID: next, Status: gptr.Of(int64(entity.ExptStatus_Pending)), Mode: gptr.Of(int32(entity.EvaluationModeRetryItems))}).Error) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"latest_run_id": next, "scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + ctrl := gomock.NewController(t) + guard := confmocks.NewMockICentralReservationGuard(ctrl) + guard.EXPECT().ReleaseTerminalReservation(gomock.Any(), testScope, historical, f.item, token).Return(component.TerminalReservationReleased, nil) + p.manager.centralGuard = guard + items := svcmocks.NewMockEvaluationSetItemService(ctrl) + items.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, ItemVersionID: &f.version, Turns: []*entity.Turn{{ID: 0}}}}, nil) + event := *p.event + event.ExptRunID = next + event.ExptRunMode = entity.EvaluationModeRetryItems + expt := *p.expt + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 80, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 81}} + reset := &ExptRetryItemsExec{manager: p.manager, exptStatsRepo: p.manager.statsRepo, exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), evaluationSetItemService: items, idgenerator: p.manager.idgenerator} + require.NoError(t, reset.resetEvalItems(context.Background(), &event, &expt, []int64{f.item})) + var item model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&item).Error) + parsed, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + require.NoError(t, err) + require.Equal(t, map[string]string{"other": "keep"}, parsed) + require.Equal(t, next, item.ExptRunID) +} + +func TestNewRunResetQuotaMismatchAndErrorDoNotMigrate(t *testing.T) { + for _, kind := range []string{"mismatch", "error", "absent"} { + t.Run(kind, func(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + token := "aa6fe7ea-7db6-4d16-8f5b-2d1344556677" + f.token(token) + next := f.run + 100 + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"latest_run_id": next, "scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + guard := confmocks.NewMockICentralReservationGuard(gomock.NewController(t)) + result := component.TerminalReservationMismatch + var failure error + if kind == "error" { + failure = errors.New("quota unavailable") + } + if kind == "absent" { + result = component.TerminalReservationAbsent + } + guard.EXPECT().ReleaseTerminalReservation(gomock.Any(), testScope, f.run, f.item, token).Return(result, failure) + p.manager.centralGuard = guard + before := f.snapshot() + err := p.manager.ResetItemsForNewRun(context.Background(), f.space, f.expt, next, []int64{f.item}, map[string]any{"expt_run_id": next, "status": int32(entity.ItemRunState_Queueing)}) + if kind != "absent" { + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, before, f.snapshot()) + } else { + require.NoError(t, err) + var item model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&item).Error) + require.Equal(t, next, item.ExptRunID) + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + require.NoError(t, err) + require.Equal(t, map[string]string{"other": "keep"}, ext) + } + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_cancel_cleanup_lease_test.go b/backend/modules/evaluation/domain/service/expt_cancel_cleanup_lease_test.go new file mode 100644 index 0000000000..bae78f6bcf --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_cancel_cleanup_lease_test.go @@ -0,0 +1,342 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + infraredis "github.com/coze-dev/coze-loop/backend/infra/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + idemrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/idem" + idemredis "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/idem/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/gorm" +) + +func cleanupLeaseRealRedis(t *testing.T, f *filterSyncFixture, m *ExptMangerImpl) (infraredis.Cmdable, *redis.Client) { + t.Helper() + addr := os.Getenv("EXPT_RETRY_REAL_REDIS_ADDR") + require.NotEmpty(t, addr, "real Redis must be configured; this regression cannot skip") + client, err := infraredis.NewClient(&redis.Options{Addr: addr}) + require.NoError(t, err) + raw, ok := infraredis.Unwrap(client) + require.True(t, ok) + info, err := raw.Info(context.Background(), "server").Result() + require.NoError(t, err) + require.Contains(t, info, "redis_version:8.2.0") + t.Log("REAL_REDIS_EXECUTED version=8.2.0") + m.manualRetryLocker = func() lock.ILocker { return filterRealTestLocker(client) } + m.mutex = m.manualRetryLocker() + m.idem = idemrepo.NewIdempotentService(idemredis.NewIdemDAO(client)) + f.svc.resultPublishLocker = m.manualRetryLocker + keys := []string{m.makeExptMutexLockKey(f.expt), fmt.Sprintf("expt_run_exec_lock:%d:%d", f.expt, f.run), f.key(), fmt.Sprintf("[idempotent]idem_key:finish-cleanup:%d:%d", f.expt, f.run)} + require.NoError(t, raw.Set(context.Background(), keys[0], fmt.Sprint(f.run), 36*time.Hour).Err()) + t.Cleanup(func() { + _, err := raw.Del(context.Background(), keys...).Result() + require.NoError(t, err) + require.NoError(t, raw.Close()) + }) + return client, raw +} + +type cleanupLeaseFilterSink struct { + repo.IExptTurnResultFilterRepo + wait time.Duration + raw *redis.Client + lifecycleKey, itemKey string + rows []*entity.ExptTurnResultFilterEntity + observedCancel error + before func(context.Context) +} + +func (*cleanupLeaseFilterSink) GetExptTurnResultFilterKeyMappings(context.Context, int64, int64) ([]*entity.ExptTurnResultFilterKeyMapping, error) { + return nil, nil +} + +func (s *cleanupLeaseFilterSink) Save(ctx context.Context, rows []*entity.ExptTurnResultFilterEntity) error { + if s.before != nil { + s.before(ctx) + } + timer := time.NewTimer(s.wait) + defer timer.Stop() + select { + case <-ctx.Done(): + s.observedCancel = ctx.Err() + return ctx.Err() + case <-timer.C: + } + if err := ctx.Err(); err != nil { + return err + } + for _, key := range []string{s.lifecycleKey, s.itemKey} { + if ok, err := s.raw.Exists(ctx, key).Result(); err != nil || ok != 1 { + return fmt.Errorf("cleanup lease missing at sink: %s: %v", key, err) + } + } + s.rows = rows + return nil +} + +func TestCancelCleanupLeaseSlowFilterRealRedisMySQL(t *testing.T) { + f, m := newCancelFilterFixture(t) + _, raw := cleanupLeaseRealRedis(t, f, m) + sink := &cleanupLeaseFilterSink{wait: 9 * time.Second, raw: raw, lifecycleKey: m.makeExptMutexLockKey(f.expt), itemKey: f.key()} + f.svc.exptTurnResultFilterRepo = sink + start := time.Now() + _, err := m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + t.Logf("SLOW_FILTER_REAL_LEASE elapsed=%s sink_cancel=%v rows=%d", time.Since(start), sink.observedCancel, len(sink.rows)) + require.NoError(t, err, "a legitimate 9s filter must finish under the lifecycle lease") + require.GreaterOrEqual(t, time.Since(start), 9*time.Second) + require.Len(t, sink.rows, 1) + require.Equal(t, entity.ItemRunState_Terminal, sink.rows[0].Status) + marked, err := m.idem.Exist(context.Background(), fmt.Sprintf("finish-cleanup:%d:%d", f.expt, f.run)) + require.NoError(t, err) + require.True(t, marked) + before := f.snapshot() + sink.wait = 0 + _, err = m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.NoError(t, err) + require.Equal(t, before, f.snapshot(), "cleanup replay preserves terminal stats and end time") +} + +type cleanupLeaseSlowGuard struct { + component.ICentralReservationGuard + raw *redis.Client + wait time.Duration + keys map[int64]string + expected map[int64]string + completed []int64 + budgets []time.Duration + before func(context.Context, int64) + faultOnce error +} + +func (g *cleanupLeaseSlowGuard) Release(ctx context.Context, _ string, run, item int64, token, _ string) error { + if g.before != nil { + g.before(ctx, item) + } + deadline, ok := ctx.Deadline() + if !ok { + return fmt.Errorf("resource IO lacks bounded context") + } + g.budgets = append(g.budgets, time.Until(deadline)) + if token != g.expected[item] { + return fmt.Errorf("resource token changed") + } + if g.faultOnce != nil { + err := g.faultOnce + g.faultOnce = nil + return err + } + timer := time.NewTimer(g.wait) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + // The external sink retains the same original-token compare/delete contract. + const script = `if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]); end; return 0;` + if _, err := g.raw.Eval(ctx, script, []string{g.keys[item]}, token).Result(); err != nil { + return err + } + g.completed = append(g.completed, item) + return nil +} + +func TestCancelCleanupLeaseCancelDuringResourceIOPreservesBindings(t *testing.T) { + f, m := newCancelFilterFixture(t) + _, raw := cleanupLeaseRealRedis(t, f, m) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + guard := &cleanupLeaseSlowGuard{ICentralReservationGuard: component.NewNoopCentralReservationGuard(), raw: raw, keys: map[int64]string{f.item: fmt.Sprintf("cleanup-lease-active-cancel:%d", f.run)}, expected: map[int64]string{f.item: "rq1"}, faultOnce: context.Canceled} + first := true + guard.before = func(ioCtx context.Context, item int64) { + require.NoError(t, ioCtx.Err()) + if first { + first = false + cancel() + } + } + require.NoError(t, raw.Set(context.Background(), guard.keys[f.item], "rq1", time.Minute).Err()) + t.Cleanup(func() { _ = raw.Del(context.Background(), guard.keys[f.item]).Err() }) + m.centralGuard = guard + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + queries := 0 + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register("cleanup_cancel_during_io", func(tx *gorm.DB) { + if ctx.Err() != nil { + queries++ + } + })) + store := &filterStore{} + f.svc.exptTurnResultFilterRepo = store + target := svcmocks.NewMockIEvalTargetService(gomock.NewController(t)) + target.EXPECT().TerminateAsyncRecordsAndDestroySandbox(gomock.Any(), f.space, []int64{f.base + 41}, gomock.Any(), gomock.Any(), false).Do(func(ioCtx context.Context, _ int64, ids []int64, _ int32, _ string, _ bool, _ ...bool) { + require.NoError(t, ioCtx.Err()) + deadline, ok := ioCtx.Deadline() + require.True(t, ok) + require.LessOrEqual(t, time.Until(deadline), 8*time.Second) + require.Equal(t, []int64{f.base + 41}, ids) + }) + m.evalTargetService = target + _, err := m.TerminateCurrentRun(ctx, f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.True(t, errno.NeedMQRetry(err)) + require.Len(t, guard.completed, 1) + require.Zero(t, queries) + require.Empty(t, store.rows) + marked, err := m.idem.Exist(context.Background(), fmt.Sprintf("finish-cleanup:%d:%d", f.expt, f.run)) + require.NoError(t, err) + require.False(t, marked) +} + +func TestCancelCleanupLeaseCancelDuringFilterStopsNotification(t *testing.T) { + f, m := newCancelFilterFixture(t) + _, raw := cleanupLeaseRealRedis(t, f, m) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sink := &cleanupLeaseFilterSink{wait: 9 * time.Second, raw: raw, lifecycleKey: m.makeExptMutexLockKey(f.expt), itemKey: f.key(), before: func(context.Context) { cancel() }} + f.svc.exptTurnResultFilterRepo = sink + _, err := m.TerminateCurrentRun(ctx, f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.True(t, errno.NeedMQRetry(err)) + require.Empty(t, sink.rows) + require.ErrorIs(t, sink.observedCancel, context.Canceled) + marked, err := m.idem.Exist(context.Background(), fmt.Sprintf("finish-cleanup:%d:%d", f.expt, f.run)) + require.NoError(t, err) + require.False(t, marked) +} + +type cleanupLeaseLostLocker struct{ lock.ILocker } + +func (l *cleanupLeaseLostLocker) LockWithRenew(ctx context.Context, key string, ttl, hold time.Duration) (bool, context.Context, func(), error) { + if strings.HasPrefix(key, "expt_run_mutex_lock:") { + child, cancel := context.WithCancel(ctx) + cancel() + return true, child, func() {}, nil + } + return l.ILocker.LockWithRenew(ctx, key, ttl, hold) +} + +func TestCancelCleanupLeaseLossIsNotParentCancelResourcePermission(t *testing.T) { + f, m := newCancelFilterFixture(t) + client, _ := cleanupLeaseRealRedis(t, f, m) + m.manualRetryLocker = func() lock.ILocker { return &cleanupLeaseLostLocker{ILocker: filterRealTestLocker(client)} } + store := &filterStore{} + f.svc.exptTurnResultFilterRepo = store + _, err := m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.True(t, errno.NeedMQRetry(err)) + require.Empty(t, store.rows) + marked, err := m.idem.Exist(context.Background(), fmt.Sprintf("finish-cleanup:%d:%d", f.expt, f.run)) + require.NoError(t, err) + require.False(t, marked) +} + +func TestCancelCleanupLeaseCancelledCommitOnlyFrozenResources(t *testing.T) { + f, m := newCancelFilterFixture(t) + _, raw := cleanupLeaseRealRedis(t, f, m) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + queriesAfterCancel := 0 + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register("cleanup_no_detached_sql", func(tx *gorm.DB) { + if ctx.Err() != nil { + queriesAfterCancel++ + } + })) + guard := &cleanupLeaseSlowGuard{ICentralReservationGuard: component.NewNoopCentralReservationGuard(), raw: raw, keys: map[int64]string{f.item: fmt.Sprintf("cleanup-lease-cancel:%d", f.run)}, expected: map[int64]string{f.item: "rq1"}} + guard.before = func(ioCtx context.Context, item int64) { + require.NoError(t, ioCtx.Err()) + require.Equal(t, f.item, item) + } + m.centralGuard = guard + require.NoError(t, raw.Set(context.Background(), guard.keys[f.item], "rq1", time.Minute).Err()) + t.Cleanup(func() { _ = raw.Del(context.Background(), guard.keys[f.item]).Err() }) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + store := &filterStore{} + f.svc.exptTurnResultFilterRepo = store + m.runFinishRepo = &cancelFilterCommitHook{IExptRunFinishRepo: m.runFinishRepo, port: m.runFinishRepo.(repo.IExptRunTerminationRepo), after: cancel} + _, err := m.TerminateCurrentRun(ctx, f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.True(t, errno.NeedMQRetry(err)) + require.Len(t, guard.completed, 1) + require.Zero(t, queriesAfterCancel, "cancelled commit cleanup must use only its frozen source") + require.Empty(t, store.rows) + marked, err := m.idem.Exist(context.Background(), fmt.Sprintf("finish-cleanup:%d:%d", f.expt, f.run)) + require.NoError(t, err) + require.False(t, marked) +} + +func TestCancelCleanupLeaseReplacementOwnerHasZeroActions(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(fmt.Sprint(cancelled), func(t *testing.T) { + f, m := newCancelFilterFixture(t) + _, raw := cleanupLeaseRealRedis(t, f, m) + source := f.source() + finished, err := m.exptRepo.GetByID(context.Background(), f.expt, f.space) + require.NoError(t, err) + finished.Status = entity.ExptStatus_Terminated + require.NoError(t, raw.Set(context.Background(), m.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run+100), time.Minute).Err()) + guard := &fakeGuard{} + m.centralGuard = guard + store := &filterStore{} + f.svc.exptTurnResultFilterRepo = store + before := f.snapshot() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if cancelled { + cancel() + } + err = m.cleanupTerminatedRun(ctx, &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, Session: &entity.Session{}}, finished, []*entity.ExptCurrentItemRunSource{source}, &entity.CompleteExptOption{}) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, fmt.Sprint(f.run+100), raw.Get(context.Background(), m.makeExptMutexLockKey(f.expt)).Val()) + require.Equal(t, before, f.snapshot()) + require.Empty(t, guard.releases()) + require.Empty(t, store.rows) + marked, err := m.idem.Exist(context.Background(), fmt.Sprintf("finish-cleanup:%d:%d", f.expt, f.run)) + require.NoError(t, err) + require.False(t, marked) + }) + } +} + +func TestCancelCleanupLeaseWholeResourcesPastEightSeconds(t *testing.T) { + f, m := newCancelFilterFixture(t) + _, raw := cleanupLeaseRealRedis(t, f, m) + guard := &cleanupLeaseSlowGuard{ICentralReservationGuard: component.NewNoopCentralReservationGuard(), raw: raw, wait: 3 * time.Second, keys: map[int64]string{}, expected: map[int64]string{}} + sources := []*entity.ExptCurrentItemRunSource{} + for i := int64(0); i < 3; i++ { + item := f.item + i + key, token := fmt.Sprintf("cleanup-lease-test:%d:%d", f.run, item), fmt.Sprintf("rq%d", i) + guard.keys[item], guard.expected[item] = key, token + require.NoError(t, raw.Set(context.Background(), key, token, time.Minute).Err()) + sources = append(sources, &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, ItemID: item, RequestID: token, Status: entity.ItemRunState_Terminal}}) + } + t.Cleanup(func() { + for _, key := range guard.keys { + _ = raw.Del(context.Background(), key).Err() + } + }) + m.centralGuard = guard + start := time.Now() + err := m.releaseTerminationResources(context.Background(), &entity.Experiment{ID: f.expt, SpaceID: f.space, SchedulerScope: testScope, ExptDispatchMode: entity.ExptDispatchModeEnforce}, f.run, sources, true, &entity.Session{}) + t.Logf("WHOLE_RESOURCES elapsed=%s completed=%d bounded_io=%v", time.Since(start), len(guard.completed), guard.budgets) + require.NoError(t, err, "three valid bounded releases must reach the tail after 9s") + require.Len(t, guard.completed, 3) + for _, budget := range guard.budgets { + require.Positive(t, budget) + require.LessOrEqual(t, budget, 8*time.Second) + } + for _, key := range guard.keys { + require.Equal(t, int64(0), raw.Exists(context.Background(), key).Val()) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_cancel_primary_scan_test.go b/backend/modules/evaluation/domain/service/expt_cancel_primary_scan_test.go new file mode 100644 index 0000000000..a8701cce80 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_cancel_primary_scan_test.go @@ -0,0 +1,224 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + exptrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + exptmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + "gorm.io/plugin/dbresolver" +) + +// Both schemas live on the task-owned real MySQL instance. The replica keeps +// one committed log while the primary has two, without timing-dependent lag. +func newCancelPrimaryScanFixture(t *testing.T) (*pendingRunFixture, *gorm.DB) { + t.Helper() + primaryDSN, replicaDSN := os.Getenv("EXPT_CANCEL_SCAN_PRIMARY_DSN"), os.Getenv("EXPT_CANCEL_SCAN_REPLICA_DSN") + if primaryDSN == "" || replicaDSN == "" { + t.Skip("EXPT_CANCEL_SCAN_PRIMARY_DSN/EXPT_CANCEL_SCAN_REPLICA_DSN missing; REAL MYSQL ROUTING NOT EXECUTED") + } + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + provider, err := db.NewDB(mysql.Open(primaryDSN), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + primary := provider.NewSession(context.Background(), db.WithMaster()) + replica, err := gorm.Open(mysql.Open(replicaDSN), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + for conn, schema := range map[*gorm.DB]string{primary: "expt_cancel_scan_7384603622_primary", replica: "expt_cancel_scan_7384603622_replica"} { + var identity struct{ Version, Schema string } + require.NoError(t, conn.Raw("SELECT VERSION() AS version, DATABASE() AS `schema`").Scan(&identity).Error) + require.Equal(t, schema, identity.Schema) + t.Logf("REAL_MYSQL_ROUTING version=%s schema=%s", identity.Version, identity.Schema) + } + tables := []string{"experiment", "expt_run_log", "expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_turn_evaluator_result_ref", "expt_stats", "expt_item_ref", "eval_target_record", "evaluator_record", "expt_evaluator_ref"} + t.Cleanup(func() { + for _, table := range tables { + require.NoError(t, primary.Exec("DELETE FROM "+table+" WHERE space_id=?", f.space).Error) + } + require.NoError(t, replica.Exec("DELETE FROM expt_item_result_run_log WHERE space_id=?", f.space).Error) + for _, conn := range []*gorm.DB{primary, replica} { + pool, err := conn.DB() + require.NoError(t, err) + require.NoError(t, pool.Close()) + } + }) + for _, table := range tables { + require.NoError(t, primary.Exec("INSERT INTO "+table+" SELECT * FROM expt_running_retry_test."+table+" WHERE space_id=?", f.space).Error) + } + primaryPool, err := primary.DB() + require.NoError(t, err) + replicaPool, err := replica.DB() + require.NoError(t, err) + routed, err := db.NewDB(mysql.New(mysql.Config{Conn: primaryPool, SkipInitializeWithVersion: true}), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + require.NoError(t, routed.NewSession(context.Background()).Use(dbresolver.Register(dbresolver.Config{ + Sources: []gorm.Dialector{mysql.New(mysql.Config{Conn: primaryPool, SkipInitializeWithVersion: true})}, + Replicas: []gorm.Dialector{mysql.New(mysql.Config{Conn: replicaPool, SkipInitializeWithVersion: true})}, + }))) + f.provider, f.conn = routed, primary + items := exptrepo.NewExptItemResultRepo(exptmysql.NewExptItemResultDAO(routed)) + f.port = items.(repo.IExptCurrentResultRepo) + p.manager.itemResultRepo = items + p.manager.exptRepo = exptrepo.NewExptRepo(exptmysql.NewExptDAO(routed), exptmysql.NewExptEvaluatorRefDAO(routed), p.manager.idgenerator) + p.manager.runLogRepo = exptrepo.NewExptRunLogRepo(exptmysql.NewExptRunLogDAO(routed)) + p.manager.runFinishRepo = exptrepo.NewExptRunFinishRepo(routed) + p.manager.statsRepo = exptrepo.NewExptStatsRepo(exptmysql.NewExptStatsDAO(routed)) + p.manager.turnResultRepo = f.turnRepo() + p.manager.itemRefRepo = exptrepo.NewExptItemRefRepo(exptmysql.NewExptItemRefDAO(routed)) + f.token("original-token-one") + // Use the existing domain serializer so fixture sources match the real contract. + manual, err := json.Marshal(&entity.ManualRetryContext{RunID: f.run, RequestID: "original-token-two", Mode: entity.EvaluationModeFailRetry, Operator: "test", AcceptedAtMS: 1}) + require.NoError(t, err) + secondExt, err := json.Marshal(map[string]string{"other": "keep", entity.ManualRetryExtKey: string(manual)}) + require.NoError(t, err) + require.NoError(t, primary.Create(&model.ExptItemResult{ID: f.base + 20001, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, Status: int32(entity.ItemRunState_Processing), Ext: &secondExt}).Error) + require.NoError(t, primary.Create(&model.ExptItemResultRunLog{ID: f.base + 21001, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, Status: int32(entity.ItemRunState_Processing), ResultState: gptr.Of(int32(entity.ExptItemResultStateDefault))}).Error) + require.NoError(t, primary.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Processing).Error) + require.NoError(t, primary.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Processing).Error) + require.NoError(t, primary.Model(&model.ExptStats{}).Where("space_id=?", f.space).Updates(map[string]any{"pending_cnt": 0, "processing_cnt": 2}).Error) + require.NoError(t, primary.Create(&model.ExptTurnResult{ID: f.base + 22001, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, TurnID: 0, Status: int32(entity.TurnRunState_Processing)}).Error) + require.NoError(t, primary.Create(&model.ExptTurnResultRunLog{ID: f.base + 23001, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, TurnID: 0, Status: int32(entity.TurnRunState_Processing)}).Error) + require.NoError(t, replica.Exec("INSERT INTO expt_item_result_run_log SELECT * FROM expt_cancel_scan_7384603622_primary.expt_item_result_run_log WHERE space_id=? AND item_id=?", f.space, f.item).Error) + require.NoError(t, primary.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + return p, replica +} + +func TestCancelPrimaryScanRealMySQLRoutingAndPaging(t *testing.T) { + p, _ := newCancelPrimaryScanFixture(t) + f := p.f + for _, raw := range []bool{false, true} { + for _, retryOrder := range []bool{false, true} { + t.Run(fmt.Sprintf("raw_%t/retry_order_%t", raw, retryOrder), func(t *testing.T) { + filter := &entity.ExptItemRunLogFilter{RawFilter: raw, OrderByRetryTimesFirst: retryOrder} + if raw { + filter.RawCond.SQL = "status = ?" + filter.RawCond.Vars = []any{int32(entity.ItemRunState_Processing)} + } + legacy, _, err := f.itemRepo().ScanItemRunLogs(context.Background(), f.expt, f.run, filter, 0, 500, f.space) + require.NoError(t, err) + require.Len(t, legacy, 1, "ordinary reads retain replica routing") + require.Equal(t, f.item, legacy[0].ItemID) + ctx := contexts.WithCtxWriteDB(context.Background()) + all, _, err := f.itemRepo().ScanItemRunLogs(ctx, f.expt, f.run, filter, 0, 500, f.space) + require.NoError(t, err) + require.Len(t, all, 2, "primary enumeration must include the replica-missing committed log") + require.Equal(t, []int64{f.item, f.item + 1}, []int64{all[0].ItemID, all[1].ItemID}) + if !retryOrder { + var cursor int64 + var ids []int64 + for page := 0; page < 3; page++ { + rows, next, err := f.itemRepo().ScanItemRunLogs(ctx, f.expt, f.run, filter, cursor, 1, f.space) + require.NoError(t, err) + if page == 2 { + require.Empty(t, rows) + require.Zero(t, next) + break + } + require.Len(t, rows, 1) + require.Greater(t, next, cursor) + ids = append(ids, rows[0].ItemID) + cursor = next + } + require.Equal(t, []int64{f.item, f.item + 1}, ids) + } + }) + } + } +} + +func TestCancelPrimaryScanTerminatesReplicaMissingItem(t *testing.T) { + p, replica := newCancelPrimaryScanFixture(t) + f := p.f + guard := &fakeGuard{} + p.manager.centralGuard = guard + applied, err := p.manager.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, p.event.Session, nil, entity.NoAggrCalculate()) + require.NoError(t, err) + require.True(t, applied) + for _, table := range []string{"expt_item_result", "expt_item_result_run_log"} { + var rows []model.ExptItemResultRunLog + require.NoError(t, f.conn.Table(table).Where("space_id=?", f.space).Order("item_id").Find(&rows).Error) + require.Len(t, rows, 2) + for _, row := range rows { + require.Equal(t, f.run, row.ExptRunID) + require.Equal(t, int32(entity.ItemRunState_Terminal), row.Status, "every primary item must terminate") + } + } + var experiment model.Experiment + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&experiment).Error) + require.Equal(t, f.run, experiment.LatestRunID) + require.Equal(t, int32(entity.ExptStatus_Terminated), experiment.Status) + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&stats).Error) + require.Zero(t, stats.ProcessingCnt) + require.Equal(t, int32(2), stats.TerminatedCnt) + require.Len(t, guard.releases(), 2, "both original reservations must be released") + for i, release := range guard.releases() { + require.Equal(t, f.run, release.RunID) + require.Equal(t, f.item+int64(i), release.ItemID) + require.Equal(t, []string{"original-token-one", "original-token-two"}[i], release.RequestID) + } + var stale []model.ExptItemResultRunLog + require.NoError(t, replica.Where("space_id=?", f.space).Find(&stale).Error) + require.Len(t, stale, 1) + require.Equal(t, int32(entity.ItemRunState_Processing), stale[0].Status) + t.Log("REAL_MYSQL_CANCEL primary_items=2 replica_items=1 terminal_items=2 original_token_releases=2 latest_run_preserved=true") +} + +func TestCancelPrimaryScanNewRunReleasesBothOriginalTokens(t *testing.T) { + p, _ := newCancelPrimaryScanFixture(t) + f := p.f + next := f.run + 100 + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.ExptStatus_Terminated), "end_at": time.Now()}).Error) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Terminated)).Error) + guard := &fakeGuard{} + p.manager.centralGuard = guard + _, err := p.manager.mutex.UnlockWithValue(context.Background(), p.manager.makeExptMutexLockKey(f.expt), strconv.FormatInt(f.run, 10)) + require.NoError(t, err) + metric := metricsmocks.NewMockExptMetric(gomock.NewController(t)) + metric.EXPECT().EmitExptExecRun(f.space, int64(entity.EvaluationModeRetryAll)) + p.manager.mtr = metric + require.NoError(t, p.manager.LogRun(context.Background(), f.expt, next, entity.EvaluationModeRetryAll, f.space, []int64{f.item, f.item + 1}, p.event.Session)) + require.Len(t, guard.releases(), 2, "new run must release the replica-missing old original reservation") + for i, release := range guard.releases() { + require.Equal(t, f.run, release.RunID) + require.Equal(t, f.item+int64(i), release.ItemID) + require.Equal(t, []string{"original-token-one", "original-token-two"}[i], release.RequestID) + } + resetGuard := componentmocks.NewMockICentralReservationGuard(gomock.NewController(t)) + for i, token := range []string{"original-token-one", "original-token-two"} { + resetGuard.EXPECT().ReleaseTerminalReservation(gomock.Any(), testScope, f.run, f.item+int64(i), token).Return(component.TerminalReservationAbsent, nil) + } + p.manager.centralGuard = resetGuard + require.NoError(t, p.manager.ResetItemsForNewRun(context.Background(), f.space, f.expt, next, []int64{f.item, f.item + 1}, map[string]any{"status": int32(entity.ItemRunState_Queueing), "expt_run_id": next})) + var rows []model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=?", f.space).Order("item_id").Find(&rows).Error) + require.Len(t, rows, 2) + for _, row := range rows { + require.Equal(t, next, row.ExptRunID) + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(row.Ext)) + require.NoError(t, err) + require.Equal(t, map[string]string{"other": "keep"}, ext) + } + t.Log("REAL_MYSQL_NEW_RUN primary_items=2 replica_items=1 old_original_token_releases=2 manual_retry_only_cleared=true") +} diff --git a/backend/modules/evaluation/domain/service/expt_cancel_review_filter_test.go b/backend/modules/evaluation/domain/service/expt_cancel_review_filter_test.go new file mode 100644 index 0000000000..3eaa4d0c51 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_cancel_review_filter_test.go @@ -0,0 +1,230 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/coze-dev/coze-loop/backend/infra/lock" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + exptrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + exptck "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/ck" + exptmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + mysqlmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/mocks" + idemrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/idem" + idemredis "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/idem/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/driver/clickhouse" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func newCancelFilterFixture(t *testing.T) (*filterSyncFixture, *ExptMangerImpl) { + t.Helper() + f := newFilterSyncFixture(t) + // Keep a single real canonical item with a pending run and original request. + for _, table := range []string{"expt_item_result_run_log", "expt_item_result"} { + require.NoError(t, f.conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=? AND item_id<>?", f.space, f.expt, f.item).Error) + } + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=?", f.space).Update("status", entity.ItemRunState_Processing).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=?", f.space).Updates(map[string]any{"status": entity.ItemRunState_Processing, "result_state": entity.ExptItemResultStateDefault}).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=?", f.space).Update("status", entity.TurnRunState_Processing).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("status", entity.TurnRunState_Processing).Error) + ctrl := gomock.NewController(t) + m := newTestExptManager(ctrl) + m.manualRetryLocker = func() lock.ILocker { return filterRealTestLocker(f.client) } + m.mutex = m.manualRetryLocker() + m.runLogRepo = exptrepo.NewExptRunLogRepo(exptmysql.NewExptRunLogDAO(f.provider)) + m.exptRepo = exptrepo.NewExptRepo(exptmysql.NewExptDAO(f.provider), exptmysql.NewExptEvaluatorRefDAO(f.provider), m.idgenerator) + m.runFinishRepo = exptrepo.NewExptRunFinishRepo(f.provider) + m.itemResultRepo, m.turnResultRepo = f.itemRepo(), f.turnRepo() + m.exptResultService = f.svc + f.svc.ExperimentRepo = m.exptRepo + m.quotaRepo, m.publisher, m.exptAggrResultService = nil, nil, nil + m.evalTargetService = nil + m.mtr.(*metricsmocks.MockExptMetric).EXPECT().EmitExptExecResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + m.idem = idemrepo.NewIdempotentService(idemredis.NewIdemDAO(f.client)) + now := time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("start_at", now).Error) + return f, m +} + +func TestFilterRefreshTerminationRealPrimaryCKSQL(t *testing.T) { + f, m := newCancelFilterFixture(t) + ctrl := gomock.NewController(t) + sqlDB, sink, err := sqlmock.New(sqlmock.ValueConverterOption(filterSQLConverter{})) + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + conn, err := gorm.Open(clickhouse.New(clickhouse.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), &gorm.Config{DisableAutomaticPing: true, Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + mapping := mysqlmocks.NewMockIExptTurnResultFilterKeyMappingDAO(ctrl) + mapping.EXPECT().GetByExptID(gomock.Any(), f.space, f.expt).Return(nil, nil).AnyTimes() + f.svc.exptTurnResultFilterRepo = exptrepo.NewExptTurnResultFilterRepo(exptck.NewExptTurnResultFilterDAO(&filterCKProvider{conn: conn}, nil), mapping) + // The final insert must contain the primary canonical Terminal payload under + // the item lock; cached Processing turns must never reach the CK sink. + sink.ExpectBegin() + sink.ExpectPrepare("INSERT INTO `expt_turn_result_filter`").ExpectExec().WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), filterLockedSQLArg{f.server, f.key(), fmt.Sprint(f.item)}, fmt.Sprint(f.version), int64(0), "0", int64(entity.ItemRunState_Terminal), fmt.Sprintf("{\"actual_output\":\"%d\"}", f.base+41), "{}", "{}", "{}", "{}", "{}", int64(0), "0", float64(0), "0", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(0, 1)) + sink.ExpectCommit() + applied, err := m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.NoError(t, err) + require.True(t, applied) + require.NoError(t, sink.ExpectationsWereMet()) + source, err := m.terminationRepo() + require.NoError(t, err) + current, err := source.GetTerminationItemSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + require.Equal(t, entity.ItemRunState_Terminal, current.Current.Status) + require.Equal(t, entity.ExptItemResultStateLogged, current.Current.ResultState) + rows, _, err := f.turnRepo().ListTurnResultByItemIDs(context.Background(), f.space, f.expt, []int64{f.item}, entity.Page{}, false) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, int32(entity.TurnRunState_Terminal), rows[0].Status) + // Read through the production filtered query and then primary turn rows. + sink.ExpectQuery(`SELECT COUNT\(DISTINCT item_id\).*etrf.status = \?`).WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), int64(entity.ItemRunState_Terminal), "2026-10-20").WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + sink.ExpectQuery(`SELECT etrf.item_id, etrf.status.*etrf.status = \?`).WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), int64(entity.ItemRunState_Terminal), "2026-10-20", 20, 0).WillReturnRows(sqlmock.NewRows([]string{"item_id", "status"}).AddRow(fmt.Sprint(f.item), int32(entity.ItemRunState_Terminal))) + expt, err := m.exptRepo.GetByID(context.Background(), f.expt, f.space) + require.NoError(t, err) + f.svc.Metric = m.mtr + m.mtr.(*metricsmocks.MockExptMetric).EXPECT().EmitExptTurnResultFilterQueryLatency(f.space, gomock.Any(), false).AnyTimes() + turns, states, count, err := f.svc.ListTurnResult(context.Background(), &entity.MGetExperimentResultParam{SpaceID: f.space, BaseExptID: &f.expt, UseAccelerator: true, Page: entity.NewPage(1, 20), FilterAccelerators: map[int64]*entity.ExptTurnResultFilterAccelerator{f.expt: {ItemRunStatus: []*entity.FieldFilter{{Op: "=", Values: []any{int64(entity.ItemRunState_Terminal)}}}}}}, expt) + require.NoError(t, err) + require.Equal(t, int64(1), count) + require.Len(t, turns, 1) + require.Equal(t, entity.ItemRunState_Terminal, states[f.item]) + require.NoError(t, sink.ExpectationsWereMet()) + t.Log("REAL_PRIMARY_TERMINAL_LOGGED_SOURCE_AND_CANONICAL_READ_CK_SQL_ITEM_LOCK executed") +} + +func TestFilterRefreshFinishedReplayBeforeNotificationMarker(t *testing.T) { + for _, fault := range []string{"filter_write", "lifecycle_mq"} { + t.Run(fault, func(t *testing.T) { + f, m := newCancelFilterFixture(t) + store := &filterStore{fail: fault == "filter_write"} + f.svc.exptTurnResultFilterRepo = store + attempts := 0 + store.beforeSave = func(rows []*entity.ExptTurnResultFilterEntity) { + attempts++ + require.True(t, f.server.Exists(f.key())) + require.Equal(t, entity.ItemRunState_Terminal, rows[0].Status) + } + if fault == "lifecycle_mq" { + pub := eventmocks.NewMockExptEventPublisher(gomock.NewController(t)) + pub.EXPECT().PublishExptLifecycleEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("broker unavailable")) + pub.EXPECT().PublishExptLifecycleEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + m.publisher = pub + } + _, err := m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.Error(t, err, "a failed filter or notification must remain retryable after commit") + require.Positive(t, attempts) + before := f.snapshot() + store.fail = false + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ExptType: entity.ExptType_Offline, ExptRunMode: entity.EvaluationModeSubmit, Session: &entity.Session{}} + scheduler := &ExptSchedulerImpl{Manager: m} + require.NoError(t, scheduler.HandleEventCheck(func(context.Context, *entity.ExptScheduleEvent) error { + t.Fatal("finished retry entered execution") + return nil + })(context.Background(), event)) + require.Equal(t, before, f.snapshot(), "replay cannot rewrite stats, end time or logs") + previous := attempts + require.NoError(t, scheduler.HandleEventCheck(func(context.Context, *entity.ExptScheduleEvent) error { return nil })(context.Background(), event)) + require.Greater(t, attempts, previous, "notification marker must not skip filter calibration") + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestFilterRefreshMissingPayloadCannotMarkNotificationComplete(t *testing.T) { + f, m := newCancelFilterFixture(t) + store := &filterStore{} + f.svc.exptTurnResultFilterRepo = store + turns := f.svc.ExptTurnResultRepo + f.svc.ExptTurnResultRepo = &cancelMissingFilterTurns{IExptTurnResultRepo: turns} + _, err := m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.Error(t, err) + require.True(t, errno.NeedMQRetry(err)) + marked, err := m.idem.Exist(context.Background(), fmt.Sprintf("finish-cleanup:%d:%d", f.expt, f.run)) + require.NoError(t, err) + require.False(t, marked) + require.Empty(t, store.rows) + before := f.snapshot() + f.svc.ExptTurnResultRepo = turns + _, err = m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.NoError(t, err) + require.NotEmpty(t, store.rows) + require.Equal(t, entity.ItemRunState_Terminal, store.rows[0].Status) + require.Equal(t, before, f.snapshot()) +} + +type cancelMissingFilterTurns struct{ repo.IExptTurnResultRepo } + +func (*cancelMissingFilterTurns) ListTurnResultByItemIDs(context.Context, int64, int64, []int64, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + return nil, 0, nil +} + +func TestFilterRefreshParentCancelOnlyResourcesAndNoCompleteTurns(t *testing.T) { + for _, noComplete := range []bool{false, true} { + t.Run(fmt.Sprint(noComplete), func(t *testing.T) { + f, m := newCancelFilterFixture(t) + store := &filterStore{} + f.svc.exptTurnResultFilterRepo = store + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + store.beforeSave = func([]*entity.ExptTurnResultFilterEntity) { calls++ } + if !noComplete { + m.runFinishRepo = &cancelFilterCommitHook{IExptRunFinishRepo: m.runFinishRepo, port: m.runFinishRepo.(repo.IExptRunTerminationRepo), after: cancel} + } + opts := []entity.CompleteExptOptionFn{entity.NoAggrCalculate()} + if noComplete { + opts = append(opts, entity.NoCompleteItemTurn()) + } + _, err := m.TerminateCurrentRun(ctx, f.expt, f.run, f.space, &entity.Session{}, nil, opts...) + if noComplete { + require.NoError(t, err) + require.Zero(t, calls) + var turn model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&turn).Error) + require.Equal(t, int32(entity.TurnRunState_Processing), turn.Status) + } else { + require.Error(t, err) + require.Zero(t, calls, "parent cancellation must prevent CK writes after commit") + var expt model.Experiment + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&expt).Error) + require.Equal(t, int32(entity.ExptStatus_Terminated), expt.Status) + m.runFinishRepo = m.runFinishRepo.(*cancelFilterCommitHook).IExptRunFinishRepo + _, err = m.TerminateCurrentRun(context.Background(), f.expt, f.run, f.space, &entity.Session{}, nil, entity.NoAggrCalculate()) + require.NoError(t, err) + require.Positive(t, calls) + } + }) + } +} + +type cancelFilterCommitHook struct { + repo.IExptRunFinishRepo + port repo.IExptRunTerminationRepo + after func() +} + +func (p *cancelFilterCommitHook) GetTerminationItemSource(ctx context.Context, space, expt, run, item int64) (*entity.ExptCurrentItemRunSource, error) { + return p.port.GetTerminationItemSource(ctx, space, expt, run, item) +} + +func (p *cancelFilterCommitHook) TerminateCurrentRun(ctx context.Context, r *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + result, err := p.port.TerminateCurrentRun(ctx, r) + if err == nil && r.Finalize { + p.after() + } + return result, err +} diff --git a/backend/modules/evaluation/domain/service/expt_cancel_review_setup_test.go b/backend/modules/evaluation/domain/service/expt_cancel_review_setup_test.go new file mode 100644 index 0000000000..77e5d28af7 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_cancel_review_setup_test.go @@ -0,0 +1,188 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/platestwrite/mocks" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + exptrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + exptmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/gorm" +) + +// Keep the SQL repositories and lifecycle locker real; only detail dependencies +// and quota are faulted. LogRun creates Pending and establishes its run owner. +func newCancelSetupFixture(t *testing.T) (*lateWriteFixture, *ExptMangerImpl, *gomock.Controller, func() bool) { + t.Helper() + f := newLateWriteFixture(t, "") + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_run_log"} { + require.NoError(t, f.conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + } + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.ExptStatus_Pending), "latest_run_id": 0}).Error) + ctrl := gomock.NewController(t) + m := newTestExptManager(ctrl) + factory, server := finishRealLockerFactory(t) + m.mutex, m.manualRetryLocker = factory(), factory + m.runFinishRepo = exptrepo.NewExptRunFinishRepo(f.provider) + m.runLogRepo = exptrepo.NewExptRunLogRepo(exptmysql.NewExptRunLogDAO(f.provider)) + m.exptRepo = exptrepo.NewExptRepo(exptmysql.NewExptDAO(f.provider), exptmysql.NewExptEvaluatorRefDAO(f.provider), m.idgenerator) + m.itemResultRepo = f.itemRepo() + m.turnResultRepo = f.turnRepo() + m.lwt.(*mocks.MockILatestWriteTracker).EXPECT().CheckWriteFlagByID(gomock.Any(), gomock.Any(), gomock.Any()).Return(false).AnyTimes() + m.mtr.(*metricsmocks.MockExptMetric).EXPECT().EmitExptExecRun(f.space, int64(entity.EvaluationModeSubmit)).AnyTimes() + return f, m, ctrl, func() bool { return server.Exists(m.makeExptMutexLockKey(f.expt)) } +} + +func TestCancelSetupRealLogRunDetailAndQuotaFailure(t *testing.T) { + for _, failure := range []string{"GetDetail", "AllowExptRun"} { + t.Run(failure, func(t *testing.T) { + f, m, _, ownsLock := newCancelSetupFixture(t) + cause := errors.New("setup dependency unavailable") + sets := m.evaluationSetService.(*svcmocks.MockIEvaluationSetService) + if failure == "GetDetail" { + sets.EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), int64(0), gomock.Any(), gomock.Any()).Return(nil, cause) + } else { + sets.EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), int64(0), gomock.Any(), gomock.Any()).Return(&entity.EvaluationSet{}, nil) + m.exptResultService.(*svcmocks.MockExptResultService).EXPECT().MGetStats(gomock.Any(), []int64{f.expt}, f.space, gomock.Any()).Return(nil, nil) + m.exptAggrResultService.(*svcmocks.MockExptAggrResultService).EXPECT().BatchGetExptAggrResultByExperimentIDs(gomock.Any(), f.space, []int64{f.expt}).Return(nil, nil) + m.quotaRepo.(*repomocks.MockQuotaRepo).EXPECT().CreateOrUpdate(gomock.Any(), f.space, gomock.Any(), gomock.Any()).Return(cause) + } + session := &entity.Session{UserID: "setup"} + require.NoError(t, m.LogRun(context.Background(), f.expt, f.run, entity.EvaluationModeSubmit, f.space, nil, session)) + require.True(t, ownsLock()) + run, err := m.runLogRepo.Get(context.Background(), f.expt, f.run) + require.NoError(t, err) + require.Equal(t, int64(entity.ExptStatus_Pending), run.Status) + err = m.Run(context.Background(), f.expt, f.run, f.space, 0, session, entity.EvaluationModeSubmit, nil) + require.ErrorIs(t, err, cause, "the caller keeps the original setup failure") + run, err = m.runLogRepo.Get(context.Background(), f.expt, f.run) + require.NoError(t, err) + require.Equal(t, int64(entity.ExptStatus_Failed), run.Status) + require.Equal(t, cause.Error(), string(run.StatusMessage)) + require.False(t, ownsLock(), "a failed first setup must release its 36h owner") + expt, err := m.exptRepo.GetByID(context.Background(), f.expt, f.space) + require.NoError(t, err) + require.Equal(t, entity.ExptStatus_Pending, expt.Status, "setup abort leaves the experiment ready for its first execution") + require.Nil(t, expt.EndAt) + require.NoError(t, m.LogRun(context.Background(), f.expt, f.run+100, entity.EvaluationModeSubmit, f.space, nil, session)) + require.True(t, ownsLock()) + t.Log("REAL_LOGRUN_RUN_SETUP_ABORT_RETRY executed") + }) + } +} + +// These failures must never turn a read error into stale ACK or leave the +// original lifecycle value trapped. SQL is still bounded by the caller context. +func TestCancelSetupCleanupFaultsReleaseOriginalOwner(t *testing.T) { + for _, fault := range []string{"read_error", "write_error", "parent_cancel"} { + t.Run(fault, func(t *testing.T) { + f, m, _, ownsLock := newCancelSetupFixture(t) + require.NoError(t, m.LogRun(context.Background(), f.expt, f.run, entity.EvaluationModeSubmit, f.space, nil, &entity.Session{UserID: "setup"})) + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + switch fault { + case "parent_cancel": + cancel() + case "read_error": + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register("setup_read_fault", func(tx *gorm.DB) { _ = tx.AddError(errors.New("read unavailable")) })) + case "write_error": + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("setup_write_fault", func(tx *gorm.DB) { _ = tx.AddError(errors.New("write unavailable")) })) + } + m.cleanupUnscheduledRun(ctx, f.expt, f.run, f.space, errors.New("setup failure")) + require.False(t, ownsLock(), fmt.Sprintf("%s must release only the captured value", fault)) + }) + } +} + +func TestCancelSetupOldRunCannotChangeReplacement(t *testing.T) { + f, m, _, ownsLock := newCancelSetupFixture(t) + session := &entity.Session{UserID: "setup"} + require.NoError(t, m.LogRun(context.Background(), f.expt, f.run, entity.EvaluationModeSubmit, f.space, nil, session)) + released, err := m.mutex.UnlockWithValue(context.Background(), m.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run)) + require.NoError(t, err) + require.True(t, released) + require.NoError(t, m.LogRun(context.Background(), f.expt, f.run+100, entity.EvaluationModeSubmit, f.space, nil, session)) + before := f.snapshot() + m.cleanupUnscheduledRun(context.Background(), f.expt, f.run, f.space, errors.New("late R1 setup")) + require.True(t, ownsLock()) + require.Equal(t, before, f.snapshot(), "late R1 cleanup cannot write either run") + // Check the real lifecycle value by trying an R1 compare-and-delete. + released, err = m.mutex.UnlockWithValue(context.Background(), m.makeExptMutexLockKey(f.expt), fmt.Sprint(f.run)) + require.NoError(t, err) + require.False(t, released) + require.True(t, ownsLock()) +} + +func TestCancelSetupRepositoryRechecksLatestAndPending(t *testing.T) { + for _, kind := range []string{"normal_cancel", "replacement", "write_rollback"} { + t.Run(kind, func(t *testing.T) { + f, m, _, _ := newCancelSetupFixture(t) + require.NoError(t, m.LogRun(context.Background(), f.expt, f.run, entity.EvaluationModeSubmit, f.space, nil, &entity.Session{UserID: "setup"})) + request := &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, SetupAbort: kind != "normal_cancel", Finalize: kind == "normal_cancel", Status: entity.ExptStatus_Terminated} + if kind == "replacement" { + require.NoError(t, f.conn.Create(&model.ExptRunLog{ID: f.run + 100, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run + 100, Status: gptr.Of(int64(entity.ExptStatus_Pending))}).Error) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("latest_run_id", f.run+100).Error) + } + if kind == "write_rollback" { + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("setup_abort_rollback", func(tx *gorm.DB) { _ = tx.AddError(errors.New("setup abort SQL failure")) })) + } + before := f.snapshot() + port, err := m.terminationRepo() + require.NoError(t, err) + result, err := port.TerminateCurrentRun(context.Background(), request) + if kind == "replacement" { + require.NoError(t, err) + require.True(t, result.NonCurrent) + } else { + require.Error(t, err) + } + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestNewRunSupersededSourceFixtureScale(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + f := p.f + const count = 32 + for i := 1; i < count; i++ { + item := f.item + int64(i) + require.NoError(t, f.conn.Create(&model.ExptItemResult{ID: f.base + 20000 + int64(i), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: f.version, Status: int32(entity.ItemRunState_Terminal)}).Error) + require.NoError(t, f.conn.Create(&model.ExptItemResultRunLog{ID: f.base + 21000 + int64(i), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: f.version, Status: int32(entity.ItemRunState_Terminal), ResultState: gptr.Of(int32(entity.ExptItemResultStateLogged))}).Error) + } + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"scheduler_mode": entity.ExptDispatchModeEnforce, "scheduler_scope": testScope}).Error) + guard := &fakeGuard{} + p.manager.centralGuard = guard + var sourceReads atomic.Int32 + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register("superseded_source_scale", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_turn_evaluator_result_ref" { + sourceReads.Add(1) + } + })) + start := time.Now() + require.NoError(t, p.manager.releaseCurrentSupersededRunQuota(context.Background(), f.expt, f.space, f.run+100, entity.EvaluationModeRetryAll)) + releases := guard.releases() + require.Len(t, releases, count, "terminal residual debt remains included at fixture scale") + seen := map[int64]bool{} + for _, release := range releases { + require.Equal(t, f.run, release.RunID) + require.Empty(t, release.RequestID) + seen[release.ItemID] = true + } + require.Len(t, seen, count) + t.Logf("SUPERSEDED_SOURCE_FIXTURE items=%d full_primary_sources=%d elapsed=%s; candidate measurement only, no production performance bound claimed", count, sourceReads.Load(), time.Since(start)) +} diff --git a/backend/modules/evaluation/domain/service/expt_cancel_review_trial_test.go b/backend/modules/evaluation/domain/service/expt_cancel_review_trial_test.go new file mode 100644 index 0000000000..b002afc2b4 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_cancel_review_trial_test.go @@ -0,0 +1,80 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func newCancelTrialFixture(t *testing.T) *pendingRunFixture { + t.Helper() + p := newPendingRunFixture(t, entity.EvaluationModeTrialRun) + // Run the production TrialRun initializer rather than relabeling a Submit run. + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result"} { + require.NoError(t, p.f.conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=?", p.f.space, p.f.expt).Error) + } + require.NoError(t, p.manager.idem.Del(context.Background(), makeStartIdemKey(p.event))) + ctrl := gomock.NewController(t) + items := svcmocks.NewMockEvaluationSetItemService(ctrl) + items.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: p.f.item, ItemVersionID: &p.f.version, Turns: []*entity.Turn{{ID: 0}}}}, gptr.Of(int64(1)), nil, nil, nil) + p.start.evaluationSetItemService = items + expt := &entity.Experiment{ID: p.f.expt, SpaceID: p.f.space, LatestRunID: p.f.run, ExptType: entity.ExptType_Offline, TrialRunItemCount: 1, EvalSet: &entity.EvaluationSet{ID: p.f.base + 80, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: p.f.base + 81}}} + p.start.resultSvc = &cancelTrialStartResult{} + require.NoError(t, (&ExptTrialRunExec{ExptSubmitExec: p.start}).ExptStart(context.Background(), p.event, expt)) + require.NoError(t, p.f.turnRepo().BatchCreateNXRunLog(context.Background(), []*entity.ExptTurnResultRunLog{{ID: p.f.base + 9910, SpaceID: p.f.space, ExptID: p.f.expt, ExptRunID: p.f.run, ItemID: p.f.item, ItemVersionID: p.f.version, TurnID: 0, Status: entity.TurnRunState_Queueing}})) + return p +} + +type cancelTrialStartResult struct{ ExptResultService } + +func (*cancelTrialStartResult) UpsertExptTurnResultFilter(context.Context, int64, int64, []int64) error { + return nil +} + +func TestTrialFailureRealSchedulerExplicitFailed(t *testing.T) { + for _, state := range []string{"pending", "terminal_success"} { + t.Run(state, func(t *testing.T) { + p := newCancelTrialFixture(t) + if state == "terminal_success" { + p.publishSuccess(t, false) + } + scheduler := &ExptSchedulerImpl{Manager: p.manager} + err := scheduler.HandleEventErr(func(context.Context, *entity.ExptScheduleEvent) error { return errors.New("trial business failure") })(context.Background(), p.event) + require.NoError(t, err, "explicit failure must not wait for natural pending publication") + run, err := p.manager.runLogRepo.Get(context.Background(), p.f.expt, p.f.run) + require.NoError(t, err) + require.Equal(t, int64(entity.ExptStatus_Failed), run.Status) + expt, err := p.manager.exptRepo.GetByID(context.Background(), p.f.expt, p.f.space) + require.NoError(t, err) + require.Equal(t, entity.ExptStatus_Failed, expt.Status) + before := p.f.snapshot() + require.NoError(t, scheduler.HandleEventCheck(func(context.Context, *entity.ExptScheduleEvent) error { + t.Fatal("finished trial reentered execution") + return nil + })(context.Background(), p.event)) + require.Equal(t, before, p.f.snapshot()) + }) + } +} + +func TestTrialFailureOldRunHasZeroActions(t *testing.T) { + p := newCancelTrialFixture(t) + next := p.f.run + 100 + require.NoError(t, p.f.conn.Create(&model.ExptRunLog{ID: next, SpaceID: p.f.space, ExptID: p.f.expt, ExptRunID: next, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeTrialRun))}).Error) + require.NoError(t, p.f.conn.Model(&model.Experiment{}).Where("space_id=?", p.f.space).Update("latest_run_id", next).Error) + before := p.f.snapshot() + err := (&ExptSchedulerImpl{Manager: p.manager}).HandleEventErr(func(context.Context, *entity.ExptScheduleEvent) error { return errors.New("old trial failure") })(context.Background(), p.event) + require.NoError(t, err) + require.Equal(t, before, p.f.snapshot()) + require.Zero(t, p.resources.Load()) + require.Zero(t, p.notifications.Load()) +} diff --git a/backend/modules/evaluation/domain/service/expt_central_display_projection_test.go b/backend/modules/evaluation/domain/service/expt_central_display_projection_test.go index 0c04d53f7b..5496078475 100644 --- a/backend/modules/evaluation/domain/service/expt_central_display_projection_test.go +++ b/backend/modules/evaluation/domain/service/expt_central_display_projection_test.go @@ -6,139 +6,73 @@ package service import ( "context" "errors" + "fmt" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" repoMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" svcMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" ) -// 本文件守的不变量：**中心调度派发时，展示投影的四张表要一起前进** -// —— run log、主表 expt_item_result、turn 主表 expt_turn_result、CK 加速表 expt_turn_result_filter。 -// -// legacy 的 handleToSubmits 一直是五写（含 stats），中心调度这条平行路径逐项漏过： -// 先漏主表（item 跑了 14h 仍显示排队中），再漏 stats（计数走负），最后漏这两张。 -// -// 漏 CK 的表现最难察觉：`item_run_state` 筛选打的是 etrf.status，且开启加速器时结果里的 -// run_state 也从 CK 读，于是执行期间按「运行中」筛恒为空、按「排队中」筛反而捞出正在跑的 item。 -// 完成时那次 upsert 会纠回来，所以只有 in-flight 窗口是错的，对着终态数据复盘看不出来。 -// PPE 实测：主表已 status=1，CK 仍为 0。 - -// centralDisplayFixture 组一套「CAS 命中、主表从 Queueing 前进到 Processing」的最小依赖。 -func centralDisplayFixture(t *testing.T, prev entity.ItemRunState, mainTableErr error) ( - *ExptItemEventEvalServiceImpl, *repoMocks.MockIExptTurnResultRepo, *svcMocks.MockExptResultService, -) { - t.Helper() +func TestCentralReservationAtomicDAOHasNoSecondWrites(t *testing.T) { ctrl := gomock.NewController(t) - t.Cleanup(ctrl.Finish) - dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - dispatch.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) - dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil, nil).AnyTimes() - - items := repoMocks.NewMockIExptItemResultRepo(ctrl) - items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: prev}}, nil) - if prev != entity.ItemRunState_Processing { - items.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(mainTableErr) + dispatch.EXPECT().StartReservedItem(gomock.Any(), int64(3), int64(1), int64(2), int64(4), "rq1").Return(repo.StartReservedItemResult{CurrentRequest: true, Started: true}, nil) + dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + svc := &ExptItemEventEvalServiceImpl{ + centralGuard: &fakeGuard{confirmResult: true}, dispatchRepo: dispatch, + exptItemResultRepo: repoMocks.NewMockIExptItemResultRepo(ctrl), exptStatsRepo: repoMocks.NewMockIExptStatsRepo(ctrl), + exptTurnResultRepo: repoMocks.NewMockIExptTurnResultRepo(ctrl), } - - stats := repoMocks.NewMockIExptStatsRepo(ctrl) - stats.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - - turns := repoMocks.NewMockIExptTurnResultRepo(ctrl) - result := svcMocks.NewMockExptResultService(ctrl) - - return &ExptItemEventEvalServiceImpl{ - centralGuard: &fakeGuard{confirmResult: true}, - dispatchRepo: dispatch, - exptItemResultRepo: items, - exptStatsRepo: stats, - exptTurnResultRepo: turns, - resultSvc: result, - }, turns, result -} - -// TestCentralReservation_AdvancesTurnTableAndFilter 核心回归：主表真的前进时， -// turn 主表与 CK 加速表都必须跟着写。 -func TestCentralReservation_AdvancesTurnTableAndFilter(t *testing.T) { - svc, turns, result := centralDisplayFixture(t, entity.ItemRunState_Queueing, nil) - - var gotTurnFields map[string]any - turns.EXPECT().UpdateTurnResultsWithItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, _ int64, itemIDs []int64, _ int64, ufields map[string]any) error { - assert.Equal(t, []int64{4}, itemIDs) - gotTurnFields = ufields - return nil - }) - - var gotFilterItemIDs []int64 - result.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, spaceID, exptID int64, itemIDs []int64) error { - assert.Equal(t, int64(3), spaceID) - assert.Equal(t, int64(1), exptID) - gotFilterItemIDs = itemIDs - return nil - }) - - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) - - require.NotNil(t, gotTurnFields, "turn 主表没写 —— turn 层状态会一直停在排队中") - assert.Equal(t, int32(entity.TurnRunState_Processing), gotTurnFields["status"]) - require.NotNil(t, gotFilterItemIDs, "CK 加速表没写 —— 按「运行中」筛会恒为空") - assert.Equal(t, []int64{4}, gotFilterItemIDs) -} - -// TestCentralReservation_SkipsDisplayProjectionWhenAlreadyProcessing 主表已是 Processing 时 -// 不重复写：与 stats 记账同一个判据，避免重复投递把 CK 反复刷一遍。 -func TestCentralReservation_SkipsDisplayProjectionWhenAlreadyProcessing(t *testing.T) { - svc, turns, result := centralDisplayFixture(t, entity.ItemRunState_Processing, nil) - - turns.EXPECT().UpdateTurnResultsWithItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - result.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) -} - -// TestCentralReservation_FilterWrittenEvenIfTurnTableFails 两个写彼此独立： -// turn 主表写失败不能把 CK 那一写连带跳过，否则一次 DB 抖动就让筛选长期失真。 -func TestCentralReservation_FilterWrittenEvenIfTurnTableFails(t *testing.T) { - svc, turns, result := centralDisplayFixture(t, entity.ItemRunState_Queueing, nil) - - turns.EXPECT().UpdateTurnResultsWithItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(errors.New("db down")) - result.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) + svc.exptItemResultRepo.(*repoMocks.MockIExptItemResultRepo).EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(4)).Return(&entity.ExptItemCurrentRequest{RunID: 2, ItemID: 4, RequestID: "rq1", Status: entity.ItemRunState_Processing}, nil) + called := false + err := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"}) + require.NoError(t, err) + require.True(t, called) } -// TestCentralReservation_NoDisplayProjectionWhenMainTableWriteFails 主表没写成功就不是 -// 「真的发生了迁移」，此时写 turn / CK 会让它们领先于主表 —— 反而制造新的不一致。 -func TestCentralReservation_NoDisplayProjectionWhenMainTableWriteFails(t *testing.T) { - svc, turns, result := centralDisplayFixture(t, entity.ItemRunState_Queueing, errors.New("db down")) - - turns.EXPECT().UpdateTurnResultsWithItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - result.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) +func TestCentralReservationNonCurrentStops(t *testing.T) { + ctrl := gomock.NewController(t) + dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) + dispatch.EXPECT().StartReservedItem(gomock.Any(), int64(3), int64(1), int64(2), int64(4), "rq1").Return(repo.StartReservedItemResult{}, nil) + dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + svc := &ExptItemEventEvalServiceImpl{centralGuard: &fakeGuard{confirmResult: true}, dispatchRepo: dispatch} + called := false + require.NoError(t, svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"})) + require.False(t, called, "noncurrent/terminal DAO response must stop invocation") } -// TestCentralReservation_DisplayProjectionToleratesNilDeps 老调用方（未注入这两个依赖的单测/ -// 部署形态）不能因此 panic —— 与相邻两项一致，缺依赖就退化为不写。 -func TestCentralReservation_DisplayProjectionToleratesNilDeps(t *testing.T) { - svc, _, _ := centralDisplayFixture(t, entity.ItemRunState_Queueing, nil) - svc.exptTurnResultRepo = nil - svc.resultSvc = nil - - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) +// The held-lock filter refresh follows a committed transition; duplicates +// recover execution without repeating the refresh or canonical/turn writes. +func TestCentralReservationCommittedFilter(t *testing.T) { + for _, started := range []bool{false, true} { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("started_%t/filter_error_%t", started, fail), func(t *testing.T) { + ctrl := gomock.NewController(t) + dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) + dispatch.EXPECT().StartReservedItem(gomock.Any(), int64(3), int64(1), int64(2), int64(4), "rq1").Return(repo.StartReservedItemResult{CurrentRequest: true, Started: started}, nil) + dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + result := svcMocks.NewMockExptResultService(ctrl) + if started { + var err error + if fail { + err = errors.New("filter unavailable") + } + result.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), []int64{4}).Return(err) + } + svc := &ExptItemEventEvalServiceImpl{ + centralGuard: &fakeGuard{confirmResult: true}, dispatchRepo: dispatch, resultSvc: result, + exptItemResultRepo: repoMocks.NewMockIExptItemResultRepo(ctrl), exptStatsRepo: repoMocks.NewMockIExptStatsRepo(ctrl), exptTurnResultRepo: repoMocks.NewMockIExptTurnResultRepo(ctrl), + } + svc.exptItemResultRepo.(*repoMocks.MockIExptItemResultRepo).EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(4)).Return(&entity.ExptItemCurrentRequest{RunID: 2, ItemID: 4, RequestID: "rq1", Status: entity.ItemRunState_Processing}, nil) + called := false + require.NoError(t, svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"})) + require.True(t, called) + }) + } + } } diff --git a/backend/modules/evaluation/domain/service/expt_central_quota_release_test.go b/backend/modules/evaluation/domain/service/expt_central_quota_release_test.go index 2e4fe844ef..8487e50f25 100644 --- a/backend/modules/evaluation/domain/service/expt_central_quota_release_test.go +++ b/backend/modules/evaluation/domain/service/expt_central_quota_release_test.go @@ -12,6 +12,7 @@ import ( "go.uber.org/mock/gomock" lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" componentMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" @@ -24,10 +25,11 @@ import ( // releaseCall 记录一次 Release 调用，供断言"释放了几次、释放的是谁"。 type releaseCall struct { - Scope string - RunID int64 - ItemID int64 - Reason string + Scope string + RunID int64 + ItemID int64 + RequestID string + Reason string } // fakeGuard 手写的额度闸替身。 @@ -41,6 +43,7 @@ type fakeGuard struct { confirmResult bool confirmErr error confirmCalls int + confirmTokens []string releaseErr error releaseCalls []releaseCall @@ -57,17 +60,18 @@ type repriceCall struct { Consumption *entity.ExpectedQuotaConsumption } -func (f *fakeGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64) (bool, error) { +func (f *fakeGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID string) (bool, error) { f.mu.Lock() defer f.mu.Unlock() f.confirmCalls++ + f.confirmTokens = append(f.confirmTokens, expectedRequestID) return f.confirmResult, f.confirmErr } -func (f *fakeGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, reason string) error { +func (f *fakeGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID, reason string) error { f.mu.Lock() defer f.mu.Unlock() - f.releaseCalls = append(f.releaseCalls, releaseCall{Scope: schedulerScope, RunID: exptRunID, ItemID: itemID, Reason: reason}) + f.releaseCalls = append(f.releaseCalls, releaseCall{Scope: schedulerScope, RunID: exptRunID, ItemID: itemID, RequestID: expectedRequestID, Reason: reason}) return f.releaseErr } @@ -468,10 +472,10 @@ func TestHandleCentralReservation_RunsInsideLock(t *testing.T) { guard := &recordingGuard{seq: &seq, confirmResult: true} dispatchRepo := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - dispatchRepo.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, _, _, _, _ int64) (bool, error) { + dispatchRepo.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), ""). + DoAndReturn(func(_ context.Context, _, _, _, _ int64, _ string) (repo.StartReservedItemResult, error) { seq = append(seq, "start_reserved") - return true, nil + return repo.StartReservedItemResult{CurrentRequest: true, Started: true}, nil }) dispatchRepo.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return([]*repo.ExptDispatchObservation{ @@ -511,12 +515,12 @@ type recordingGuard struct { confirmResult bool } -func (r *recordingGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64) (bool, error) { +func (r *recordingGuard) ConfirmRunning(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID string) (bool, error) { *r.seq = append(*r.seq, "confirm") return r.confirmResult, nil } -func (r *recordingGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, reason string) error { +func (r *recordingGuard) Release(ctx context.Context, schedulerScope string, exptRunID, itemID int64, expectedRequestID, reason string) error { *r.seq = append(*r.seq, "release") return nil } @@ -526,183 +530,10 @@ func (r *recordingGuard) RepriceRunConsumption(ctx context.Context, schedulerSco return nil } -// TestHandleCentralReservation_AdvancesMainTableForDisplay 中心调度取得执行权后， -// **主表 expt_item_result 也必须推进到 Processing**，否则用户看到的状态是错的。 -// -// ★ 这条守的是一个真实缺陷（2026-08-23 PPE 实测）： -// run log 是执行真值，但用户看到的是主表 —— MGetExperimentResult 走 expt_item_result -// 构造 run_state。中心调度这条新派发路径只写了 run log，于是 5 个已 Processing 14 小时的 -// item 在 results 接口里**全部显示 queueing**，现象是"实验看着没动"，而它跑得很正常。 -// -// legacy 的 handleToSubmits 一直成对写这两张表（UpdateItemRunLog + UpdateItemsResult）， -// 新路径漏了后者 —— 平行实现漏字段的又一例，与团队记忆里 -// 「新增派发路径漏设 ExptRunMode」同族：**老路径对、新路径漏，且单测不断言就发现不了**。 -// -// 断言的是"主表被推进到 Processing"这件事本身，而不只是"某个方法被调过"， -// 所以把 ufields 的内容也钉住：只写别的字段不算修好。 -func TestHandleCentralReservation_AdvancesMainTableForDisplay(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - guard := &fakeGuard{confirmResult: true} - dispatchRepo := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - dispatchRepo.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(true, nil) - dispatchRepo.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*repo.ExptDispatchObservation{ - {ItemID: 4, Status: int32(entity.ItemRunState_Success)}, - }, nil) - - var gotFields map[string]any - var gotItemIDs []int64 - itemResultRepo := repoMocks.NewMockIExptItemResultRepo(ctrl) - // 推进前会先读主表当前状态：判据是"真的发生了状态迁移"，见 expt_stats_consistency_test.go。 - itemResultRepo.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: entity.ItemRunState_Queueing}}, nil) - itemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, _, _ int64, itemIDs []int64, ufields map[string]any) error { - gotItemIDs = itemIDs - gotFields = ufields - return nil - }) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: guard, - dispatchRepo: dispatchRepo, - exptItemResultRepo: itemResultRepo, - } - - event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4} - ctx := ctxcache.Init(context.Background()) - event.WithCtxCentralAdmittedExpt(ctx, &entity.Experiment{ - ID: 1, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: testScope, - }) - - handler := svc.HandleCentralReservation(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { - return nil - }) - assert.NoError(t, handler(ctx, event)) - - assert.NotNil(t, gotFields, "主表必须被推进 —— 只写 run log 会让用户一直看到 queueing") - assert.Equal(t, []int64{4}, gotItemIDs, "推进的必须是本次取得执行权的那个 item") - assert.Equal(t, int32(entity.ItemRunState_Processing), gotFields["status"], - "主表状态必须是 Processing，与 run log 一致") -} - -// TestHandleCentralReservation_AdvancesStatsToProcessing 中心调度取得执行权后， -// **expt_stats 也必须记 Queueing→Processing**，否则计数会单向下溢。 -// -// ★ 这条守的是一个真实缺陷（2026-08-24 PPE 长跑压测实测）： -// 完成侧（expt_result_impl.go 的 statsCntOp）做的是「从 item 原状态减 1、往新状态加 1」， -// 减的是 Processing 桶。legacy 在 handleToSubmits 派发时就把 item 计入 Processing，两边配对； -// 中心调度的派发只把 run log CAS 成 Queueing/reserved（**status 仍是 Queueing**）， -// 从未有人往 Processing 桶加过 —— 完成时减一个从未加过的计数。 -// 实测一个 14 题 enforce 实验：fail 累到 4 时 processing = -4，pending 恒 14 从不下降。 -// -// 断言 op 的内容而非"调过就行"：只加 Processing 不减 Queueing，pending 依旧不降。 -func TestHandleCentralReservation_AdvancesStatsToProcessing(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - guard := &fakeGuard{confirmResult: true} - dispatchRepo := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - dispatchRepo.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(true, nil) - dispatchRepo.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*repo.ExptDispatchObservation{ - {ItemID: 4, Status: int32(entity.ItemRunState_Success)}, - }, nil) - - itemResultRepo := repoMocks.NewMockIExptItemResultRepo(ctrl) - itemResultRepo.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: entity.ItemRunState_Queueing}}, nil) - itemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil).AnyTimes() - - var gotOp *entity.StatsCntArithOp - statsRepo := repoMocks.NewMockIExptStatsRepo(ctrl) - statsRepo.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, _, _ int64, op *entity.StatsCntArithOp) error { - gotOp = op - return nil - }) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: guard, - dispatchRepo: dispatchRepo, - exptItemResultRepo: itemResultRepo, - exptStatsRepo: statsRepo, - } - - event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4} - ctx := ctxcache.Init(context.Background()) - event.WithCtxCentralAdmittedExpt(ctx, &entity.Experiment{ - ID: 1, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: testScope, - }) - - handler := svc.HandleCentralReservation(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { - return nil - }) - assert.NoError(t, handler(ctx, event)) - - assert.NotNil(t, gotOp, "必须记 stats —— 不记会让完成侧减一个从未加过的计数") - assert.Equal(t, 1, gotOp.OpStatusCnt[entity.ItemRunState_Processing], - "Processing 必须 +1，与完成侧的 -1 配对") - assert.Equal(t, -1, gotOp.OpStatusCnt[entity.ItemRunState_Queueing], - "Queueing 必须 -1，否则 pending_turn_count 永不下降") +func (f *fakeGuard) ReleaseTerminalReservation(context.Context, string, int64, int64, string) (component.TerminalReservationReleaseResult, error) { + return 0, errors.New("unexpected terminal reservation release") } -// TestHandleCentralReservation_SkipsStatsOnDuplicateDelivery 主表已是 Processing 时**不得**再记 stats。 -// -// ⚠️ **本用例的判据换过一次**：原先绑的是 run log 的 CAS（started=false 就跳过），理由是 -// "CAS 未命中说明 item 早已计入 Processing"。那个假设在**首投就未命中**时不成立 —— -// 于是那条 item 从来没被计过，Queueing 永不减。PPE 实测一个 900 题 enforce 实验 -// pending 虚高 281、success 少 249，同泳道 legacy 分毫不差。 -// -// 现在判据是**主表实际发生了状态迁移**：主表既是完成侧做「-1」的锚点，也就该是记账的判据， -// 两边同源才不可能分叉。"重复投递不得多记"这个原意保留，只是换了更可靠的信号。 -func TestHandleCentralReservation_SkipsStatsOnDuplicateDelivery(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - guard := &fakeGuard{confirmResult: true} - dispatchRepo := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - // CAS 未命中：重复投递或已被 repair 修正 - dispatchRepo.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(false, nil) - dispatchRepo.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*repo.ExptDispatchObservation{ - {ItemID: 4, Status: int32(entity.ItemRunState_Success)}, - }, nil) - - itemResultRepo := repoMocks.NewMockIExptItemResultRepo(ctrl) - // 主表已是 Processing —— 这才是"已经计过账"的可靠证据。 - itemResultRepo.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: entity.ItemRunState_Processing}}, nil) - // 已是 Processing 就连主表都不该重写。 - itemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil).Times(0) - - statsRepo := repoMocks.NewMockIExptStatsRepo(ctrl) - // Times(0)：一次都不能调 —— 这是本用例的全部意义 - statsRepo.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil).Times(0) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: guard, - dispatchRepo: dispatchRepo, - exptItemResultRepo: itemResultRepo, - exptStatsRepo: statsRepo, - } - - event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4} - ctx := ctxcache.Init(context.Background()) - event.WithCtxCentralAdmittedExpt(ctx, &entity.Experiment{ - ID: 1, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: testScope, - }) - - handler := svc.HandleCentralReservation(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { - return nil - }) - assert.NoError(t, handler(ctx, event)) +func (f *recordingGuard) ReleaseTerminalReservation(context.Context, string, int64, int64, string) (component.TerminalReservationReleaseResult, error) { + return 0, errors.New("unexpected terminal reservation release") } diff --git a/backend/modules/evaluation/domain/service/expt_complete_quota_release_test.go b/backend/modules/evaluation/domain/service/expt_complete_quota_release_test.go index fe960b06a0..01587bf1cf 100644 --- a/backend/modules/evaluation/domain/service/expt_complete_quota_release_test.go +++ b/backend/modules/evaluation/domain/service/expt_complete_quota_release_test.go @@ -17,6 +17,7 @@ import ( metricsMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" eventsMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" repoMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" svcMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" ) @@ -49,9 +50,29 @@ func stubIncompleteItems(t *testing.T, ctrl *gomock.Controller, itemIDs ...int64 repo.EXPECT(). ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(logs, int64(len(logs)), nil).AnyTimes() + stubOriginalQuotaObservations(repo) return repo } +// Original non-manual fixtures observe empty request IDs explicitly. +func stubOriginalQuotaObservations(items *repoMocks.MockIExptItemResultRepo) { + items.EXPECT().ScanIncompleteDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, space, expt, run int64) ([]*repo.ExptDispatchObservation, error) { + logs, _, err := items.ScanItemRunLogs(ctx, expt, run, &entity.ExptItemRunLogFilter{Status: []entity.ItemRunState{entity.ItemRunState_Queueing, entity.ItemRunState_Processing}}, 0, 0, space) + if err != nil { + return nil, err + } + out := make([]*repo.ExptDispatchObservation, 0, len(logs)) + seen := map[int64]bool{} + for _, log := range logs { + if log != nil && !seen[log.ItemID] { + seen[log.ItemID] = true + out = append(out, &repo.ExptDispatchObservation{ItemID: log.ItemID, Status: log.Status}) + } + } + return out, nil + }).AnyTimes() +} + func enforceExpt() *entity.Experiment { return &entity.Experiment{ ID: 100, @@ -139,6 +160,7 @@ func TestReleaseCentralQuota_ScanFailDoesNotGuessItems(t *testing.T) { defer ctrl.Finish() repo := repoMocks.NewMockIExptItemResultRepo(ctrl) + stubOriginalQuotaObservations(repo) repo.EXPECT(). ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, int64(0), assert.AnError).AnyTimes() @@ -283,6 +305,7 @@ func TestReleaseCentralQuotaForIncompleteItems_NoopOnEmptyInputs(t *testing.T) { repo.EXPECT(). ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return([]*entity.ExptItemResultRunLog{nil, {ItemID: 11}, nil}, int64(3), nil).AnyTimes() + stubOriginalQuotaObservations(repo) guard := &fakeGuard{} svc := &ExptMangerImpl{centralGuard: guard, itemResultRepo: repo} diff --git a/backend/modules/evaluation/domain/service/expt_current_result_source.go b/backend/modules/evaluation/domain/service/expt_current_result_source.go new file mode 100644 index 0000000000..5c1477fd3e --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_current_result_source.go @@ -0,0 +1,593 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "fmt" + "reflect" + "strconv" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/idgen" + "github.com/coze-dev/coze-loop/backend/pkg/logs" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +var errExptNonCurrentSource = errors.New("execution source is no longer current") + +type currentItemSourceKey struct{} + +type currentItemExecution struct { + Source *entity.ExptCurrentItemRunSource + LockLost bool + Completed bool + QuotaReleased bool + SourceValidated bool + LogCtx context.Context +} + +func executionSource(ctx context.Context) *currentItemExecution { + source, _ := ctx.Value(currentItemSourceKey{}).(*currentItemExecution) + return source +} + +func (e *ExptItemEventEvalServiceImpl) validateResumeSource(ctx context.Context, event *entity.ExptItemEvalEvent, current *entity.ExptItemCurrentRequest) (bool, error) { + source := event.ResumeSource + if source == nil { + if event.AsyncReportTrigger || event.AsyncEvaluatorReportTrigger { + // A legacy event has no invoke ID. Only an original AsyncCtx keyed by an + // explicit identity can recover it; the latest turn record is not identity. + return false, errno.NewMQRetryErr("legacy async event lacks original recoverable invocation identity") + } + return true, nil + } + if source.RecordID <= 0 || source.TurnID < 0 || source.InvokeID != strconv.FormatInt(source.RecordID, 10) || source.Kind != "target" && source.Kind != "evaluator" { + return false, errno.NewMQRetryErr("invalid original resume source") + } + if e.exptTurnResultRepo == nil { + return false, errno.NewMQRetryErr("turn source reader missing") + } + var logs []*entity.ExptTurnResultRunLog + if execution := executionSource(ctx); execution != nil && execution.Source != nil { + for _, log := range execution.Source.Turns { + logs = append(logs, log) + } + } else { + var err error + logs, err = e.exptTurnResultRepo.GetItemTurnRunLogs(contexts.WithCtxWriteDB(ctx), event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + } + var turn *entity.ExptTurnResultRunLog + for _, log := range logs { + if log == nil { + return false, errno.NewMQRetryErr("incomplete current turn source") + } + if log.TurnID == source.TurnID { + if turn != nil { + return false, errno.NewMQRetryErr("duplicate current source turn") + } + turn = log + } + } + if turn == nil { + return false, errno.NewMQRetryErr("original source turn missing") + } + if turn.SpaceID != event.SpaceID || turn.ExptID != event.ExptID || turn.ExptRunID != event.ExptRunID || turn.ItemID != event.EvalSetItemID || turn.ItemVersionID != current.ItemVersionID { + return false, errno.NewMQRetryErr("foreign or incomplete source turn") + } + if turn.Status < entity.TurnRunState_Queueing || turn.Status > entity.TurnRunState_Terminal { + return false, errno.NewMQRetryErr("invalid source turn state") + } + if source.Kind == "target" { + if turn.TargetResultID != source.RecordID { + return false, nil + } + if e.evaTargetService == nil { + return false, errno.NewMQRetryErr("target source reader missing") + } + targetSpace := event.SpaceID + if e.manager != nil { + detail, err := e.manager.GetDetail(contexts.WithCtxWriteDB(ctx), event.ExptID, event.SpaceID, event.Session) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if detail == nil { + return false, errno.NewMQRetryErr("original target resource source missing") + } + targetSpace = resolveLoadSpaceID(event.SpaceID, detail.TargetSpaceID) + } + record, err := e.evaTargetService.GetRecordByID(contexts.WithCtxWriteDB(ctx), targetSpace, source.RecordID) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if record == nil { + return false, errno.NewMQRetryErr("original target record missing") + } + if record.ID <= 0 || record.SpaceID <= 0 || record.ExperimentRunID <= 0 || record.ItemID <= 0 || record.TurnID < 0 || record.Status == nil || *record.Status < entity.EvalTargetRunStatusSuccess || *record.Status > entity.EvalTargetRunStatusAsyncInvoking { + return false, errno.NewMQRetryErr("incomplete original target record") + } + return record.ID == source.RecordID && record.ExperimentRunID == event.ExptRunID && record.ItemID == event.EvalSetItemID && (record.ItemVersionID == 0 || record.ItemVersionID == current.ItemVersionID) && record.TurnID == source.TurnID, nil + } + if e.evaluatorRecordService == nil { + return false, errno.NewMQRetryErr("evaluator source reader missing") + } + record, err := e.evaluatorRecordService.GetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), source.RecordID, false) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if record == nil { + return false, errno.NewMQRetryErr("original evaluator record missing") + } + if record.ID <= 0 || record.SpaceID <= 0 || record.ExperimentID <= 0 || record.ExperimentRunID <= 0 || record.ItemID <= 0 || record.TurnID < 0 || record.Status < entity.EvaluatorRunStatusSuccess || record.Status > entity.EvaluatorRunStatusSkipped || record.SourceType < 0 || record.SourceType > entity.EvaluatorRecordSourceTypeInline { + return false, errno.NewMQRetryErr("incomplete original evaluator record") + } + if record.ID != source.RecordID || record.ExperimentID != event.ExptID || record.ExperimentRunID != event.ExptRunID || record.ItemID != event.EvalSetItemID || record.ItemVersionID != 0 && record.ItemVersionID != current.ItemVersionID || record.TurnID != source.TurnID { + return false, nil + } + if record.TargetRecordID != 0 && record.TargetRecordID != turn.TargetResultID { + return false, nil + } + refs := turn.EvaluatorResultIds + if refs == nil { + return false, errno.NewMQRetryErr("current evaluator source references missing") + } + if record.SourceType == entity.EvaluatorRecordSourceTypeInline { + if record.InlineKey == "" { + return false, errno.NewMQRetryErr("incomplete original inline evaluator source") + } + for _, ref := range refs.Inline { + if ref == nil { + return false, errno.NewMQRetryErr("incomplete inline source reference") + } + if ref.InlineKey == record.InlineKey { + return ref.RecordID == record.ID, nil + } + } + return false, nil + } + if record.EvaluatorVersionID <= 0 { + return false, errno.NewMQRetryErr("original evaluator version missing") + } + for _, ref := range refs.Registered { + if ref == nil { + return false, errno.NewMQRetryErr("incomplete registered source reference") + } + if ref.VersionID == record.EvaluatorVersionID && ref.Alias == record.Alias { + return ref.RecordID == record.ID, nil + } + } + return record.Alias == "" && refs.EvalVerIDToResID[record.EvaluatorVersionID] == record.ID, nil +} + +func sourceForExecution(ctx context.Context, port repo.IExptCurrentResultRepo, event *entity.ExptItemEvalEvent) (*entity.ExptCurrentItemRunSource, error) { + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if execution := executionSource(ctx); execution != nil && execution.Source != nil { + if execution.LockLost { + return nil, errno.NewMQRetryErr("item execution ownership lost") + } + return execution.Source, nil + } + source, err := port.GetCurrentItemRunSource(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if source == nil || source.Current.RunID != event.ExptRunID || source.Current.RequestID != event.RequestID { + return nil, errExptNonCurrentSource + } + if source.Current.Mode != event.ExptRunMode && event.ExptRunMode != entity.EvaluationModeUnknown { + return nil, errno.NewMQRetryErr("event source mode mismatch") + } + return source, nil +} + +func checkedCurrentWrite(ctx context.Context, applied bool, err error) error { + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !applied { + return errExptNonCurrentSource + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + return nil +} + +// handleCurrentEventErr runs after the execution lock exits. It acquires a fresh +// holder and rechecks the original source before budget, Fail, publication or release. +func (e *ExptItemEventEvalServiceImpl) handleCurrentEventErr(ctx context.Context, event *entity.ExptItemEvalEvent, evalErr error, port repo.IExptCurrentResultRepo, execution *currentItemExecution) error { + if execution.Source == nil { + return errno.WrapMQRetryErr(evalErr) + } + if execution.LockLost { + return errno.NewMQRetryErr("item execution ownership lost") + } + if e.itemLocker == nil { + return errno.NewMQRetryErr("independent error item locker unavailable") + } + if execution.LogCtx != nil { + ctx = logs.SetLogID(ctx, logs.GetLogID(execution.LogCtx)) + } + mutex := e.itemLocker() + if mutex == nil { + return errno.NewMQRetryErr("error item locker unavailable") + } + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", event.ExptID, event.EvalSetItemID) + locked, lockCtx, cancel, err := mutex.LockWithRenew(ctx, key, 5*time.Second, time.Minute) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !locked { + return errno.NewMQRetryErr("error item execution lock busy") + } + var debt bool + err = func() error { + defer cancel() + defer func() { + if _, err := mutex.Unlock(key); err != nil { + logs.CtxWarn(ctx, "unlock error item: %v", err) + } + }() + if err := lockCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + current, err := port.GetCurrentItemRunSource(lockCtx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !sameExecutionIdentity(execution.Source, current) { + return nil + } + finished := entity.IsItemRunFinished(current.Current.Status) + if finished && (!execution.Completed || current.Current.Status != entity.ItemRunState_Fail || current.Current.ResultState != entity.ExptItemResultStateLogged) { + return nil + } + valid, err := e.validateResumeSource(lockCtx, event, ¤t.Current) + if err != nil { + return err + } + if !valid { + return nil + } + if e.experimentRepo == nil { + return errno.NewMQRetryErr("frozen experiment reader unavailable") + } + expt, err := e.experimentRepo.GetByID(contexts.WithCtxWriteDB(lockCtx), event.ExptID, event.SpaceID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + event = event.Clone() + event.RetryTimes = max(event.RetryTimes, int(current.Current.RetryTimes)) + conf := e.configer.GetErrRetryConf(lockCtx, event.SpaceID, evalErr) + maxRetry := conf.GetRetryTimes() + if event.MaxRetryTimes > 0 { + maxRetry = event.MaxRetryTimes + } + // The frozen pointer distinguishes an explicit zero from an absent budget. + if expt != nil && expt.EvalConf != nil && expt.EvalConf.ItemRetryNum != nil { + maxRetry = *expt.EvalConf.ItemRetryNum + } + retry := event.RetryTimes < maxRetry && !event.CtxForceNoRetry(ctx) + if e.metric != nil { + code, stable, _ := errno.ParseStatusError(evalErr) + e.metric.EmitItemExecResult(event.SpaceID, int64(event.ExptRunMode), true, retry, stable, int64(code), event.CreateAt) + } + debt = conf.IsInDebt + if debt { + return nil + } + if finished { + return nil + } + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, ErrMsg: []byte(errno.SerializeErr(evalErr)), RetryTimes: int32(event.RetryTimes), FailPendingTurns: true} + if retry { + write.RetryTimes++ + write.RetryOnly = true + write.FailPendingTurns = false + if event.Ext[entity.RetryYieldExtKey] == "true" { + write.RetryOnly = false + write.Yield = true + } + } + if err := e.prepareMissingRawTurnLogs(lockCtx, execution.Source, write); err != nil { + return err + } + applied, err := port.CompleteCurrentItemRun(lockCtx, execution.Source, write) + if err := checkedCurrentWrite(lockCtx, applied, err); err != nil { + if errors.Is(err, errExptNonCurrentSource) { + return nil + } + return err + } + if retry { + if write.Yield { + return nil + } + event.RetryTimes = int(write.RetryTimes) + event.AsyncReportTrigger = false + event.AsyncEvaluatorReportTrigger = false + event.ResumeSource = nil + return e.publisher.PublishExptRecordEvalEvent(lockCtx, event, gptr.Of(conf.GetRetryInterval()), nil) + } + e.releaseCentralQuotaOutsideGate(lockCtx, event, evalErr, "item unretriable pre-exec failure") + return nil + }() + if err != nil { + return err + } + if debt { + if terminate, ok := e.manager.(IExptCurrentRunTerminator); ok { + _, err := terminate.TerminateCurrentRun(ctx, event.ExptID, event.ExptRunID, event.SpaceID, event.Session, execution.Source, entity.WithStatus(entity.ExptStatus_Terminated), entity.WithStatusMessage(evalErr.Error())) + return err + } + return errno.NewMQRetryErr("source-aware debt termination provider missing") + } + return nil +} + +func sameExecutionIdentity(expected, actual *entity.ExptCurrentItemRunSource) bool { + if expected == nil || actual == nil { + return false + } + a, b := expected.Current, actual.Current + if a.SpaceID != b.SpaceID || a.ExptID != b.ExptID || a.RunID != b.RunID || a.ItemID != b.ItemID || a.ItemVersionID != b.ItemVersionID || a.RequestID != b.RequestID || a.Mode != b.Mode || len(expected.Turns) != len(actual.Turns) { + return false + } + if len(expected.CanonicalTurns) != len(actual.CanonicalTurns) { + return false + } + for index, turn := range expected.CanonicalTurns { + other := actual.CanonicalTurns[index] + if turn == nil || other == nil || turn.ID != other.ID || turn.TurnID != other.TurnID || turn.ItemVersionID != other.ItemVersionID || turn.TargetResultID != other.TargetResultID { + return false + } + } + for id, turn := range expected.Turns { + other := actual.Turns[id] + if turn == nil || other == nil || turn.ID != other.ID || turn.TargetResultID != other.TargetResultID || !entity.SameCurrentEvaluatorRefs(turn.EvaluatorResultIds, other.EvaluatorResultIds) { + return false + } + } + return reflect.DeepEqual(expected.CurrentRefs, actual.CurrentRefs) +} + +func (e *ExptSchedulerImpl) withCurrentScannedItem(ctx context.Context, event *entity.ExptScheduleEvent, item *entity.ExptEvalItem, callback func(context.Context, repo.IExptCurrentResultRepo) (bool, error)) (bool, error) { + if err := ctx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + if item == nil || item.Source == nil { + return false, errno.NewMQRetryErr("same-scan original item source missing") + } + port, ok := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo) + if !ok { + return false, errno.NewMQRetryErr("source-conditional scan writer unavailable") + } + if e.itemLocker == nil { + return false, errno.NewMQRetryErr("scan item locker unavailable") + } + mutex := e.itemLocker() + if mutex == nil { + return false, errno.NewMQRetryErr("scan item locker unavailable") + } + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", event.ExptID, item.ItemID) + locked, lockCtx, cancel, err := mutex.LockWithRenew(ctx, key, 5*time.Second, time.Minute) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if !locked { + return false, nil + } + defer cancel() + defer func() { + if _, err := mutex.Unlock(key); err != nil { + logs.CtxWarn(ctx, "unlock scanned item: %v", err) + } + }() + current, err := port.GetCurrentItemRunSource(lockCtx, event.SpaceID, event.ExptID, event.ExptRunID, item.ItemID) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if !sameExecutionIdentity(item.Source, current) || current.Current.Status != entity.ItemRunState_Processing || item.Source.Current.RetryTimes != current.Current.RetryTimes || item.Source.Current.UpdatedAt == nil || current.Current.UpdatedAt == nil || !item.Source.Current.UpdatedAt.Equal(*current.Current.UpdatedAt) { + return false, nil + } + if err := lockCtx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + lockCtx = context.WithValue(lockCtx, currentItemSourceKey{}, ¤tItemExecution{Source: item.Source}) + applied, err := callback(lockCtx, port) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + return applied, nil +} + +func (e *ExptSchedulerImpl) handleCurrentZombies(ctx context.Context, event *entity.ExptScheduleEvent, items []*entity.ExptEvalItem, expt *entity.Experiment, zombieSecond int, asyncExec bool) (alives, zombies []*entity.ExptEvalItem, err error) { + for _, item := range items { + if item == nil || item.State != entity.ItemRunState_Processing { + continue + } + if item.UpdatedAt == nil || item.UpdatedAt.IsZero() || time.Since(*item.UpdatedAt).Seconds() <= float64(zombieSecond) { + alives = append(alives, item) + continue + } + applied, err := e.withCurrentScannedItem(ctx, event, item, func(lockCtx context.Context, port repo.IExptCurrentResultRepo) (bool, error) { + zombieErr := errno.NewItemZombieTimeoutErr(zombieSecond, asyncExec) + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, ErrMsg: []byte(errno.SerializeErr(zombieErr)), RetryTimes: item.RetryTimes, FailPendingTurns: true} + if err := prepareMissingRawTurnLogs(lockCtx, e.IDGen, item.Source, write); err != nil { + return false, err + } + if err := e.terminateZombieEvaluatorRecords(lockCtx, event, []int64{item.ItemID}); err != nil { + return false, err + } + if err := lockCtx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + if err := e.terminateZombieEvalTargetRecords(lockCtx, event, expt, []int64{item.ItemID}); err != nil { + return false, err + } + applied, err := port.CompleteCurrentItemRun(lockCtx, item.Source, write) + if err != nil || !applied { + return false, err + } + e.releaseCentralQuotaForItems(lockCtx, expt, event.ExptRunID, []int64{item.ItemID}, map[int64]string{item.ItemID: item.Source.Current.RequestID}, "item zombie timeout") + if e.sandboxAgentNotifier != nil { + if err := e.sandboxAgentNotifier.NotifyItemFail(lockCtx, expt, item.ItemID, zombieErr); err != nil { + logs.CtxWarn(lockCtx, "notify zombie item: %v", err) + } + } + return true, nil + }) + if err != nil { + return nil, nil, err + } + if applied { + copy := *item + copy.State = entity.ItemRunState_Fail + zombies = append(zombies, ©) + } else { + alives = append(alives, item) + } + } + return alives, zombies, nil +} + +func (e *ExptSchedulerImpl) sweepCurrentSandboxItems(ctx context.Context, event *entity.ExptScheduleEvent, items []*entity.ExptEvalItem, expt *entity.Experiment) (alives, terminated []*entity.ExptEvalItem, err error) { + for _, item := range items { + if item == nil || item.State != entity.ItemRunState_Processing { + alives = append(alives, item) + continue + } + applied, err := e.withCurrentScannedItem(ctx, event, item, func(lockCtx context.Context, port repo.IExptCurrentResultRepo) (bool, error) { + ids := frozenTargetRecordIDs(item.Source) + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, RetryTimes: item.RetryTimes, FailPendingTurns: true} + // Prepare before any sandbox RPC or cleanup. + if err := prepareMissingRawTurnLogs(lockCtx, e.IDGen, item.Source, write); err != nil { + return false, err + } + + if len(ids) == 0 { + return false, nil + } + space := resolveLoadSpaceID(event.SpaceID, expt.TargetSpaceID) + stopped, status := e.evalTargetService.CheckSandboxTerminated(lockCtx, space, ids) + if len(stopped) == 0 { + return false, nil + } + state := status[stopped[0]] + if state == "" { + state = "Terminated" + } + write.ErrMsg = []byte(errno.SerializeErr(errno.NewSandboxTerminatedBeforeReportErr(state))) + for _, log := range write.MissingTurnLogs { + log.ErrMsg = string(write.TurnError(log.Status)) + } + if err := lockCtx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + _ = e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox(lockCtx, space, stopped, int32(errno.SandboxTerminatedBeforeReportCode), fmt.Sprintf("sandbox execute reached terminal state (%s) before result was reported", state), false) + applied, err := port.CompleteCurrentItemRun(lockCtx, item.Source, write) + if err != nil || !applied { + return false, err + } + e.releaseCentralQuotaForItems(lockCtx, expt, event.ExptRunID, []int64{item.ItemID}, map[int64]string{item.ItemID: item.Source.Current.RequestID}, "sandbox terminated before report") + byRecord := map[int64][]int64{} + for _, id := range stopped { + byRecord[id] = []int64{item.ItemID} + } + e.emitSandboxSweptInvokeFinished(lockCtx, event, expt, stopped, byRecord) + return true, nil + }) + if err != nil { + return nil, nil, err + } + if applied { + copy := *item + copy.State = entity.ItemRunState_Fail + terminated = append(terminated, ©) + } else { + alives = append(alives, item) + } + } + return alives, terminated, nil +} + +// IDs are prepared before SQL and under the item lock; the transaction never calls IDGen. +func (e *ExptItemEventEvalServiceImpl) prepareMissingRawTurnLogs(ctx context.Context, source *entity.ExptCurrentItemRunSource, write *entity.ExptCurrentItemRunWrite) error { + return prepareMissingRawTurnLogs(ctx, e.idgen, source, write) +} + +func prepareMissingRawTurnLogs(ctx context.Context, gen idgen.IIDGenerator, source *entity.ExptCurrentItemRunSource, write *entity.ExptCurrentItemRunWrite) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if source.Current.RequestID != "" || write.RetryOnly { + return nil + } + missing := []*entity.ExptTurnResult{} + for _, turn := range source.CanonicalTurns { + if source.Turns[turn.TurnID] == nil { + missing = append(missing, turn) + } + } + if len(missing) == 0 { + return nil + } + if gen == nil { + return errno.NewMQRetryErr("raw turn log ID preparation unavailable") + } + ids, err := gen.GenMultiIDs(ctx, len(missing)) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if len(ids) != len(missing) { + return errno.NewMQRetryErr("raw turn log ID preparation incomplete") + } + for index, turn := range missing { + log := turn.ToRunLogDO() + log.ID = ids[index] + log.ExptRunID = source.Current.RunID + log.Status = entity.TurnRunState_Fail + if write.Yield { + log.Status = entity.TurnRunState_Queueing + } + if write.Status == entity.ItemRunState_Terminal { + log.Status = entity.TurnRunState_Terminal + } + log.ErrMsg = string(write.TurnError(log.Status)) + write.MissingTurnLogs = append(write.MissingTurnLogs, log) + } + return nil +} + +func frozenTargetRecordIDs(source *entity.ExptCurrentItemRunSource) []int64 { + seen := map[int64]bool{} + ids := []int64{} + for _, log := range source.Turns { + if log != nil && log.TargetResultID > 0 && !seen[log.TargetResultID] { + seen[log.TargetResultID] = true + ids = append(ids, log.TargetResultID) + } + } + for _, turn := range source.CanonicalTurns { + if turn != nil && source.Turns[turn.TurnID] == nil && turn.TargetResultID > 0 && !seen[turn.TargetResultID] { + seen[turn.TargetResultID] = true + ids = append(ids, turn.TargetResultID) + } + } + return ids +} diff --git a/backend/modules/evaluation/domain/service/expt_current_result_source_cc_coverage_test.go b/backend/modules/evaluation/domain/service/expt_current_result_source_cc_coverage_test.go new file mode 100644 index 0000000000..6f001c445e --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_current_result_source_cc_coverage_test.go @@ -0,0 +1,1398 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +func TestCCCurrentSourceCoverageExecutionSource(t *testing.T) { + require.Nil(t, executionSource(context.Background())) + require.Nil(t, executionSource(context.WithValue(context.Background(), currentItemSourceKey{}, "not-an-execution"))) + execution := ¤tItemExecution{LockLost: true} + require.Same(t, execution, executionSource(context.WithValue(context.Background(), currentItemSourceKey{}, execution))) +} + +func ccCurrentSourceCoverageIdentitySource() *entity.ExptCurrentItemRunSource { + return &entity.ExptCurrentItemRunSource{ + Current: entity.ExptItemCurrentRequest{ + SpaceID: 3, ExptID: 5, RunID: 7, ItemID: 11, ItemVersionID: 13, + RequestID: "rq-current", Mode: entity.EvaluationModeSubmit, + }, + CanonicalTurns: []*entity.ExptTurnResult{{ID: 21, TurnID: 0, ItemVersionID: 13, TargetResultID: 41}}, + Turns: map[int64]*entity.ExptTurnResultRunLog{ + 0: {ID: 21, TargetResultID: 41, EvaluatorResultIds: &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 61}}}, + }, + CurrentRefs: []*entity.ExptTurnEvaluatorResultRef{{ID: 71}}, + } +} + +func TestCCCurrentSourceCoverageSameExecutionIdentity(t *testing.T) { + require.False(t, sameExecutionIdentity(nil, nil)) + require.False(t, sameExecutionIdentity(ccCurrentSourceCoverageIdentitySource(), nil)) + require.False(t, sameExecutionIdentity(nil, ccCurrentSourceCoverageIdentitySource())) + + for _, tc := range []struct { + name string + mutate func(expected, actual *entity.ExptCurrentItemRunSource) + want bool + }{ + {name: "identical", mutate: func(_, _ *entity.ExptCurrentItemRunSource) {}, want: true}, + {name: "stale_space", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Current.SpaceID = 4 }}, + {name: "stale_expt", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Current.ExptID = 6 }}, + {name: "stale_run", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Current.RunID = 8 }}, + {name: "stale_item", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Current.ItemID = 12 }}, + {name: "stale_item_version", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Current.ItemVersionID = 14 }}, + {name: "stale_request", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Current.RequestID = "rq-replacement" }}, + {name: "stale_mode", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { + expected.Current.Mode = entity.EvaluationModeFailRetry + }}, + {name: "turn_count", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { + expected.Turns[1] = &entity.ExptTurnResultRunLog{ID: 22} + }}, + {name: "canonical_turn_count", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { + expected.CanonicalTurns = append(expected.CanonicalTurns, &entity.ExptTurnResult{ID: 22, TurnID: 1}) + }}, + {name: "canonical_turn_nil", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { + expected.CanonicalTurns = append(expected.CanonicalTurns, nil) + }}, + {name: "canonical_turn_id", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.CanonicalTurns[0].ID = 99 }}, + {name: "canonical_turn_turn_id", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.CanonicalTurns[0].TurnID = 1 }}, + {name: "canonical_turn_item_version", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.CanonicalTurns[0].ItemVersionID = 14 }}, + {name: "canonical_turn_target", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.CanonicalTurns[0].TargetResultID = 42 }}, + {name: "turn_log_nil", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Turns[0] = nil }}, + {name: "turn_log_missing_key", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { + expected.Turns = map[int64]*entity.ExptTurnResultRunLog{1: {ID: 21, TargetResultID: 41}} + }}, + {name: "turn_log_id", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Turns[0].ID = 99 }}, + {name: "turn_log_target", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { expected.Turns[0].TargetResultID = 42 }}, + {name: "turn_log_evaluator_refs", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { + expected.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 62}} + }}, + {name: "nil_and_empty_evaluator_refs_equal", mutate: func(expected, actual *entity.ExptCurrentItemRunSource) { + expected.Turns[0].EvaluatorResultIds = nil + actual.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{} + }, want: true}, + {name: "current_refs", mutate: func(expected, _ *entity.ExptCurrentItemRunSource) { + expected.CurrentRefs = []*entity.ExptTurnEvaluatorResultRef{{ID: 72}} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + expected := ccCurrentSourceCoverageIdentitySource() + actual := ccCurrentSourceCoverageIdentitySource() + tc.mutate(expected, actual) + require.Equal(t, tc.want, sameExecutionIdentity(expected, actual)) + }) + } +} + +func ccCurrentSourceCoverageEvalEvent(mode entity.ExptRunMode) *entity.ExptItemEvalEvent { + return &entity.ExptItemEvalEvent{ + SpaceID: 3, ExptID: 5, ExptRunID: 7, EvalSetItemID: 11, + RequestID: "rq-current", ExptRunMode: mode, + } +} + +func ccCurrentSourceCoverageRunSource() *entity.ExptCurrentItemRunSource { + return &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{ + SpaceID: 3, ExptID: 5, RunID: 7, ItemID: 11, ItemVersionID: 13, + RequestID: "rq-current", Mode: entity.EvaluationModeSubmit, + }} +} + +func TestCCCurrentSourceCoverageSourceForExecution(t *testing.T) { + t.Run("context_cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + source, err := sourceForExecution(ctx, nil, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeSubmit)) + require.Nil(t, source) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + }) + + t.Run("frozen_execution_source_reused", func(t *testing.T) { + frozen := ccCurrentSourceCoverageRunSource() + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: frozen}) + source, err := sourceForExecution(ctx, nil, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeSubmit)) + require.NoError(t, err) + require.Same(t, frozen, source) + }) + + t.Run("execution_ownership_lost", func(t *testing.T) { + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{ + Source: ccCurrentSourceCoverageRunSource(), LockLost: true, + }) + source, err := sourceForExecution(ctx, nil, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeSubmit)) + require.Nil(t, source) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "item execution ownership lost") + }) + + t.Run("port_read_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + port := repomocks.NewMockIExptCurrentResultRepo(ctrl) + fault := errors.New("current source read failed") + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(nil, fault) + source, err := sourceForExecution(context.Background(), port, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeSubmit)) + require.Nil(t, source) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + for _, stale := range []string{"nil", "run", "request"} { + t.Run("stale_"+stale, func(t *testing.T) { + ctrl := gomock.NewController(t) + port := repomocks.NewMockIExptCurrentResultRepo(ctrl) + observed := ccCurrentSourceCoverageRunSource() + switch stale { + case "nil": + observed = nil + case "run": + observed.Current.RunID = 8 + case "request": + observed.Current.RequestID = "rq-replacement" + } + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(observed, nil) + source, err := sourceForExecution(context.Background(), port, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeSubmit)) + require.Nil(t, source) + require.ErrorIs(t, err, errExptNonCurrentSource) + }) + } + + t.Run("mode_mismatch", func(t *testing.T) { + ctrl := gomock.NewController(t) + port := repomocks.NewMockIExptCurrentResultRepo(ctrl) + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(ccCurrentSourceCoverageRunSource(), nil) + source, err := sourceForExecution(context.Background(), port, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeFailRetry)) + require.Nil(t, source) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "event source mode mismatch") + }) + + t.Run("unknown_event_mode_accepted", func(t *testing.T) { + ctrl := gomock.NewController(t) + port := repomocks.NewMockIExptCurrentResultRepo(ctrl) + observed := ccCurrentSourceCoverageRunSource() + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(observed, nil) + source, err := sourceForExecution(context.Background(), port, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeUnknown)) + require.NoError(t, err) + require.Same(t, observed, source) + }) + + t.Run("current_source", func(t *testing.T) { + ctrl := gomock.NewController(t) + port := repomocks.NewMockIExptCurrentResultRepo(ctrl) + observed := ccCurrentSourceCoverageRunSource() + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(observed, nil) + source, err := sourceForExecution(context.Background(), port, ccCurrentSourceCoverageEvalEvent(entity.EvaluationModeSubmit)) + require.NoError(t, err) + require.Same(t, observed, source) + }) +} + +func TestCCCurrentSourceCoverageCheckedCurrentWrite(t *testing.T) { + fault := errors.New("write failed") + err := checkedCurrentWrite(context.Background(), false, fault) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + + err = checkedCurrentWrite(context.Background(), false, nil) + require.ErrorIs(t, err, errExptNonCurrentSource) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err = checkedCurrentWrite(ctx, true, nil) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + + // A stale write is rejected before cancellation is considered. + require.ErrorIs(t, checkedCurrentWrite(ctx, false, nil), errExptNonCurrentSource) + require.NoError(t, checkedCurrentWrite(context.Background(), true, nil)) +} + +func TestCCCurrentSourceCoverageFrozenTargetRecordIDs(t *testing.T) { + require.Empty(t, frozenTargetRecordIDs(&entity.ExptCurrentItemRunSource{})) + + t.Run("single", func(t *testing.T) { + source := &entity.ExptCurrentItemRunSource{Turns: map[int64]*entity.ExptTurnResultRunLog{ + 0: {TurnID: 0, TargetResultID: 41}, + }} + require.Equal(t, []int64{41}, frozenTargetRecordIDs(source)) + }) + + t.Run("dedupe_and_nil_skip", func(t *testing.T) { + source := &entity.ExptCurrentItemRunSource{Turns: map[int64]*entity.ExptTurnResultRunLog{ + 0: {TurnID: 0, TargetResultID: 41}, + 1: {TurnID: 1, TargetResultID: 41}, + 2: {TurnID: 2, TargetResultID: 0}, + 3: nil, + 4: {TurnID: 4, TargetResultID: -7}, + }} + require.Equal(t, []int64{41}, frozenTargetRecordIDs(source)) + }) + + t.Run("canonical_only_when_turn_log_missing", func(t *testing.T) { + source := &entity.ExptCurrentItemRunSource{ + Turns: map[int64]*entity.ExptTurnResultRunLog{ + 0: {TurnID: 0, TargetResultID: 41}, + }, + CanonicalTurns: []*entity.ExptTurnResult{ + {TurnID: 0, TargetResultID: 41}, // covered by run log + {TurnID: 1, TargetResultID: 42}, // missing run log + {TurnID: 2, TargetResultID: 41}, // duplicate of turn 0 ID + nil, + {TurnID: 3, TargetResultID: -1}, + }, + } + require.ElementsMatch(t, []int64{41, 42}, frozenTargetRecordIDs(source)) + require.Len(t, frozenTargetRecordIDs(source), 2) + }) +} + +func ccCurrentSourceCoverageRawTurnSource() *entity.ExptCurrentItemRunSource { + return &entity.ExptCurrentItemRunSource{ + Current: entity.ExptItemCurrentRequest{SpaceID: 3, ExptID: 5, RunID: 7, ItemID: 11}, + CanonicalTurns: []*entity.ExptTurnResult{ + {ID: 21, SpaceID: 3, ExptID: 5, ExptRunID: 7, ItemID: 11, ItemVersionID: 13, TurnID: 0, TargetResultID: 41}, + {ID: 22, SpaceID: 3, ExptID: 5, ExptRunID: 7, ItemID: 11, ItemVersionID: 13, TurnID: 1, TargetResultID: 42}, + }, + Turns: map[int64]*entity.ExptTurnResultRunLog{ + 0: {ID: 21, TurnID: 0}, + }, + } +} + +func TestCCCurrentSourceCoveragePrepareMissingRawTurnLogs(t *testing.T) { + t.Run("reuse_request_skips_preparation", func(t *testing.T) { + source := ccCurrentSourceCoverageRawTurnSource() + source.Current.RequestID = "rq-reused" + write := &entity.ExptCurrentItemRunWrite{} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), nil, source, write)) + require.Empty(t, write.MissingTurnLogs) + }) + + t.Run("retry_only_skips_preparation", func(t *testing.T) { + write := &entity.ExptCurrentItemRunWrite{RetryOnly: true} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), nil, ccCurrentSourceCoverageRawTurnSource(), write)) + require.Empty(t, write.MissingTurnLogs) + }) + + t.Run("no_missing_turns", func(t *testing.T) { + source := ccCurrentSourceCoverageRawTurnSource() + source.Turns[1] = &entity.ExptTurnResultRunLog{ID: 22, TurnID: 1} + write := &entity.ExptCurrentItemRunWrite{} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), nil, source, write)) + require.Empty(t, write.MissingTurnLogs) + }) + + t.Run("context_cancelled_before", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := prepareMissingRawTurnLogs(ctx, nil, ccCurrentSourceCoverageRawTurnSource(), &entity.ExptCurrentItemRunWrite{}) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + }) + + t.Run("generator_missing", func(t *testing.T) { + err := prepareMissingRawTurnLogs(context.Background(), nil, ccCurrentSourceCoverageRawTurnSource(), &entity.ExptCurrentItemRunWrite{}) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "raw turn log ID preparation unavailable") + }) + + t.Run("generator_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + fault := errors.New("id generation failed") + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return(nil, fault) + err := prepareMissingRawTurnLogs(context.Background(), gen, ccCurrentSourceCoverageRawTurnSource(), &entity.ExptCurrentItemRunWrite{}) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + t.Run("generator_id_count_mismatch", func(t *testing.T) { + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{}, nil) + err := prepareMissingRawTurnLogs(context.Background(), gen, ccCurrentSourceCoverageRawTurnSource(), &entity.ExptCurrentItemRunWrite{}) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "raw turn log ID preparation incomplete") + }) + + t.Run("cancelled_during_generation", func(t *testing.T) { + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + ctx, cancel := context.WithCancel(context.Background()) + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).DoAndReturn(func(context.Context, int) ([]int64, error) { + cancel() + return []int64{901}, nil + }) + write := &entity.ExptCurrentItemRunWrite{} + err := prepareMissingRawTurnLogs(ctx, gen, ccCurrentSourceCoverageRawTurnSource(), write) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, write.MissingTurnLogs) + }) + + t.Run("fail_status_is_prepared", func(t *testing.T) { + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{901}, nil) + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), gen, ccCurrentSourceCoverageRawTurnSource(), write)) + require.Len(t, write.MissingTurnLogs, 1) + log := write.MissingTurnLogs[0] + require.Equal(t, int64(901), log.ID) + require.Equal(t, int64(7), log.ExptRunID) + require.Equal(t, int64(11), log.ItemID) + require.Equal(t, int64(13), log.ItemVersionID) + require.Equal(t, int64(1), log.TurnID) + require.Equal(t, entity.TurnRunState_Fail, log.Status) + require.NotEmpty(t, log.ErrMsg) + }) + + t.Run("yield_status_is_queueing", func(t *testing.T) { + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{901}, nil) + write := &entity.ExptCurrentItemRunWrite{Yield: true} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), gen, ccCurrentSourceCoverageRawTurnSource(), write)) + require.Len(t, write.MissingTurnLogs, 1) + require.Equal(t, entity.TurnRunState_Queueing, write.MissingTurnLogs[0].Status) + require.Empty(t, write.MissingTurnLogs[0].ErrMsg) + }) + + t.Run("terminal_status_overrides_yield", func(t *testing.T) { + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{901}, nil) + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Terminal, Yield: true} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), gen, ccCurrentSourceCoverageRawTurnSource(), write)) + require.Len(t, write.MissingTurnLogs, 1) + require.Equal(t, entity.TurnRunState_Terminal, write.MissingTurnLogs[0].Status) + require.NotEmpty(t, write.MissingTurnLogs[0].ErrMsg) + }) + + t.Run("multiple_missing_turns_keep_canonical_order", func(t *testing.T) { + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + gen.EXPECT().GenMultiIDs(gomock.Any(), 2).Return([]int64{901, 902}, nil) + source := ccCurrentSourceCoverageRawTurnSource() + delete(source.Turns, 0) + write := &entity.ExptCurrentItemRunWrite{} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), gen, source, write)) + require.Len(t, write.MissingTurnLogs, 2) + require.Equal(t, int64(901), write.MissingTurnLogs[0].ID) + require.Equal(t, int64(0), write.MissingTurnLogs[0].TurnID) + require.Equal(t, int64(902), write.MissingTurnLogs[1].ID) + require.Equal(t, int64(1), write.MissingTurnLogs[1].TurnID) + }) +} + +func TestCCCurrentSourceCoveragePrepareMissingRawTurnLogsWrapper(t *testing.T) { + svc := &ExptItemEventEvalServiceImpl{} + source := &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{RequestID: "rq-reused"}} + write := &entity.ExptCurrentItemRunWrite{} + require.NoError(t, svc.prepareMissingRawTurnLogs(context.Background(), source, write)) + require.Empty(t, write.MissingTurnLogs) +} + +type ccCurrentSourceCoverageResumeFixture struct { + svc *ExptItemEventEvalServiceImpl + turns *repomocks.MockIExptTurnResultRepo + target *svcmocks.MockIEvalTargetService + records *svcmocks.MockEvaluatorRecordService + manager *svcmocks.MockIExptManager + event *entity.ExptItemEvalEvent + current *entity.ExptItemCurrentRequest +} + +func ccCurrentSourceCoverageNewResumeFixture(t *testing.T, kind string) *ccCurrentSourceCoverageResumeFixture { + t.Helper() + ctrl := gomock.NewController(t) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + manager := svcmocks.NewMockIExptManager(ctrl) + return &ccCurrentSourceCoverageResumeFixture{ + svc: &ExptItemEventEvalServiceImpl{ + exptTurnResultRepo: turns, evaTargetService: target, + evaluatorRecordService: records, manager: manager, + }, + turns: turns, target: target, records: records, manager: manager, + event: &entity.ExptItemEvalEvent{ + SpaceID: 3, ExptID: 5, ExptRunID: 7, EvalSetItemID: 11, + ResumeSource: &entity.ExptResumeSource{Kind: kind, TurnID: 0, RecordID: 61, InvokeID: "61"}, + }, + current: &entity.ExptItemCurrentRequest{SpaceID: 3, ExptID: 5, RunID: 7, ItemID: 11, ItemVersionID: 13, Mode: entity.EvaluationModeSubmit}, + } +} + +func ccCurrentSourceCoverageResumeTurn() *entity.ExptTurnResultRunLog { + return &entity.ExptTurnResultRunLog{ + SpaceID: 3, ExptID: 5, ExptRunID: 7, ItemID: 11, ItemVersionID: 13, + TurnID: 0, TargetResultID: 61, Status: entity.TurnRunState_Processing, + } +} + +func ccCurrentSourceCoverageExpectTurn(f *ccCurrentSourceCoverageResumeFixture, logs []*entity.ExptTurnResultRunLog, err error) { + f.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(5), int64(7), int64(11), int64(3)).Return(logs, err) +} + +func ccCurrentSourceCoverageExpectEvaluator(f *ccCurrentSourceCoverageResumeFixture, turn *entity.ExptTurnResultRunLog, record *entity.EvaluatorRecord, err error) { + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn}, nil) + f.records.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(61), false).Return(record, err) +} + +func ccCurrentSourceCoverageExpectTarget(f *ccCurrentSourceCoverageResumeFixture, turn *entity.ExptTurnResultRunLog, space int64, record *entity.EvalTargetRecord, err error) { + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn}, nil) + f.target.EXPECT().GetRecordByID(gomock.Any(), space, int64(61)).Return(record, err) +} + +func ccCurrentSourceCoverageTargetRecord(status entity.EvalTargetRunStatus) *entity.EvalTargetRecord { + return &entity.EvalTargetRecord{ + ID: 61, SpaceID: 3, ExperimentRunID: 7, ItemID: 11, + ItemVersionID: 13, TurnID: 0, Status: &status, + } +} + +func ccCurrentSourceCoverageEvaluatorRecord() *entity.EvaluatorRecord { + return &entity.EvaluatorRecord{ + ID: 61, SpaceID: 3, ExperimentID: 5, ExperimentRunID: 7, + ItemID: 11, ItemVersionID: 13, TurnID: 0, + EvaluatorVersionID: 51, Alias: "a", + SourceType: entity.EvaluatorRecordSourceTypeBuiltin, + Status: entity.EvaluatorRunStatusSuccess, + } +} + +func ccCurrentSourceCoverageEvaluatorRefs() *entity.EvaluatorResults { + return &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} +} + +func TestCCCurrentSourceCoverageValidateResumeWithoutSource(t *testing.T) { + for _, tc := range []struct { + name string + async bool + asyncE bool + want bool + retry bool + }{ + {name: "plain_legacy_event", want: true}, + {name: "async_report_without_identity", async: true, retry: true}, + {name: "async_evaluator_report_without_identity", asyncE: true, retry: true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.event.ResumeSource = nil + f.event.AsyncReportTrigger = tc.async + f.event.AsyncEvaluatorReportTrigger = tc.asyncE + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.Equal(t, tc.want, valid) + if tc.retry { + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "legacy async event lacks original recoverable invocation identity") + } else { + require.NoError(t, err) + } + }) + } +} + +func TestCCCurrentSourceCoverageValidateResumeSourceShape(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*entity.ExptResumeSource) + }{ + {name: "record_id_zero", mutate: func(s *entity.ExptResumeSource) { s.RecordID = 0 }}, + {name: "record_id_negative", mutate: func(s *entity.ExptResumeSource) { s.RecordID = -1 }}, + {name: "turn_id_negative", mutate: func(s *entity.ExptResumeSource) { s.TurnID = -1 }}, + {name: "invoke_id_mismatch", mutate: func(s *entity.ExptResumeSource) { s.InvokeID = "other" }}, + {name: "kind_unknown", mutate: func(s *entity.ExptResumeSource) { s.Kind = "other" }}, + {name: "kind_empty", mutate: func(s *entity.ExptResumeSource) { s.Kind = "" }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + tc.mutate(f.event.ResumeSource) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "invalid original resume source") + }) + } +} + +func TestCCCurrentSourceCoverageValidateResumeTurnBranches(t *testing.T) { + t.Run("turn_reader_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.exptTurnResultRepo = nil + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "turn source reader missing") + }) + + t.Run("turn_read_error", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + fault := errors.New("turn source read failed") + ccCurrentSourceCoverageExpectTurn(f, nil, fault) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + t.Run("nil_turn_log", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{nil}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "incomplete current turn source") + }) + + t.Run("duplicate_turn", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + turn := ccCurrentSourceCoverageResumeTurn() + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn, turn}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "duplicate current source turn") + }) + + t.Run("missing_turn", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + turn := ccCurrentSourceCoverageResumeTurn() + turn.TurnID = 9 + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "original source turn missing") + }) + + for _, tc := range []struct { + name string + mutate func(*entity.ExptTurnResultRunLog) + }{ + {name: "foreign_space", mutate: func(l *entity.ExptTurnResultRunLog) { l.SpaceID = 4 }}, + {name: "foreign_expt", mutate: func(l *entity.ExptTurnResultRunLog) { l.ExptID = 6 }}, + {name: "foreign_run", mutate: func(l *entity.ExptTurnResultRunLog) { l.ExptRunID = 8 }}, + {name: "foreign_item", mutate: func(l *entity.ExptTurnResultRunLog) { l.ItemID = 12 }}, + {name: "foreign_item_version", mutate: func(l *entity.ExptTurnResultRunLog) { l.ItemVersionID = 14 }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + turn := ccCurrentSourceCoverageResumeTurn() + tc.mutate(turn) + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "foreign or incomplete source turn") + }) + } + + for _, tc := range []struct { + name string + status entity.TurnRunState + }{ + {name: "below_queueing", status: entity.TurnRunState(-1)}, + {name: "above_terminal", status: entity.TurnRunState(5)}, + } { + t.Run("state_"+tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + turn := ccCurrentSourceCoverageResumeTurn() + turn.Status = tc.status + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "invalid source turn state") + }) + } + + for _, tc := range []struct { + name string + status entity.TurnRunState + }{ + {name: "queueing", status: entity.TurnRunState_Queueing}, + {name: "terminal", status: entity.TurnRunState_Terminal}, + } { + t.Run("legal_boundary_state_"+tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + turn := ccCurrentSourceCoverageResumeTurn() + turn.Status = tc.status + ccCurrentSourceCoverageExpectTarget(f, turn, int64(3), ccCurrentSourceCoverageTargetRecord(entity.EvalTargetRunStatusSuccess), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + } + + t.Run("frozen_execution_turns_reused", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + // The reader is required as an interface even though the frozen source + // supplies the turns; it must not be queried. + f.svc.manager = nil + turn := ccCurrentSourceCoverageResumeTurn() + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{ + Source: &entity.ExptCurrentItemRunSource{Turns: map[int64]*entity.ExptTurnResultRunLog{0: turn}}, + }) + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(61)).Return(ccCurrentSourceCoverageTargetRecord(entity.EvalTargetRunStatusSuccess), nil) + valid, err := f.svc.validateResumeSource(ctx, f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) +} + +func TestCCCurrentSourceCoverageValidateResumeTarget(t *testing.T) { + t.Run("target_result_id_mismatch_short_circuits", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.evaTargetService = nil + f.svc.manager = nil + turn := ccCurrentSourceCoverageResumeTurn() + turn.TargetResultID = 42 + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + + t.Run("target_reader_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.evaTargetService = nil + f.svc.manager = nil + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{ccCurrentSourceCoverageResumeTurn()}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "target source reader missing") + }) + + t.Run("target_detail_error", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + fault := errors.New("experiment detail read failed") + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{ccCurrentSourceCoverageResumeTurn()}, nil) + f.manager.EXPECT().GetDetail(gomock.Any(), int64(5), int64(3), gomock.Any()).Return(nil, fault) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + t.Run("target_detail_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{ccCurrentSourceCoverageResumeTurn()}, nil) + f.manager.EXPECT().GetDetail(gomock.Any(), int64(5), int64(3), gomock.Any()).Return(nil, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "original target resource source missing") + }) + + t.Run("target_space_resolved_from_experiment", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{ccCurrentSourceCoverageResumeTurn()}, nil) + f.manager.EXPECT().GetDetail(gomock.Any(), int64(5), int64(3), gomock.Any()).Return(&entity.Experiment{ID: 5, SpaceID: 3, TargetSpaceID: 99}, nil) + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(99), int64(61)).Return(ccCurrentSourceCoverageTargetRecord(entity.EvalTargetRunStatusSuccess), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + + t.Run("target_space_falls_back_to_event", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + ccCurrentSourceCoverageExpectTarget(f, ccCurrentSourceCoverageResumeTurn(), int64(3), ccCurrentSourceCoverageTargetRecord(entity.EvalTargetRunStatusSuccess), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + + t.Run("target_record_read_error", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + fault := errors.New("target record read failed") + ccCurrentSourceCoverageExpectTarget(f, ccCurrentSourceCoverageResumeTurn(), int64(3), nil, fault) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + t.Run("target_record_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + ccCurrentSourceCoverageExpectTarget(f, ccCurrentSourceCoverageResumeTurn(), int64(3), nil, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "original target record missing") + }) + + for _, tc := range []struct { + name string + mutate func(*entity.EvalTargetRecord) + }{ + {name: "id_zero", mutate: func(r *entity.EvalTargetRecord) { r.ID = 0 }}, + {name: "space_zero", mutate: func(r *entity.EvalTargetRecord) { r.SpaceID = 0 }}, + {name: "run_zero", mutate: func(r *entity.EvalTargetRecord) { r.ExperimentRunID = 0 }}, + {name: "item_zero", mutate: func(r *entity.EvalTargetRecord) { r.ItemID = 0 }}, + {name: "turn_negative", mutate: func(r *entity.EvalTargetRecord) { r.TurnID = -1 }}, + {name: "status_nil", mutate: func(r *entity.EvalTargetRecord) { r.Status = nil }}, + {name: "status_below_success", mutate: func(r *entity.EvalTargetRecord) { + s := entity.EvalTargetRunStatusUnknown + r.Status = &s + }}, + {name: "status_above_async", mutate: func(r *entity.EvalTargetRecord) { + s := entity.EvalTargetRunStatus(4) + r.Status = &s + }}, + } { + t.Run("incomplete_"+tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + record := ccCurrentSourceCoverageTargetRecord(entity.EvalTargetRunStatusSuccess) + tc.mutate(record) + ccCurrentSourceCoverageExpectTarget(f, ccCurrentSourceCoverageResumeTurn(), int64(3), record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "incomplete original target record") + }) + } + + for _, tc := range []struct { + name string + mutate func(*entity.EvalTargetRecord) + }{ + {name: "record_id", mutate: func(r *entity.EvalTargetRecord) { r.ID = 62 }}, + {name: "run_id", mutate: func(r *entity.EvalTargetRecord) { r.ExperimentRunID = 8 }}, + {name: "item_id", mutate: func(r *entity.EvalTargetRecord) { r.ItemID = 12 }}, + {name: "item_version", mutate: func(r *entity.EvalTargetRecord) { r.ItemVersionID = 14 }}, + {name: "turn_id", mutate: func(r *entity.EvalTargetRecord) { r.TurnID = 1 }}, + } { + t.Run("identity_"+tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + record := ccCurrentSourceCoverageTargetRecord(entity.EvalTargetRunStatusSuccess) + tc.mutate(record) + ccCurrentSourceCoverageExpectTarget(f, ccCurrentSourceCoverageResumeTurn(), int64(3), record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + } + + for _, tc := range []struct { + name string + status entity.EvalTargetRunStatus + }{ + {name: "success", status: entity.EvalTargetRunStatusSuccess}, + {name: "async_invoking", status: entity.EvalTargetRunStatusAsyncInvoking}, + } { + t.Run("legal_status_"+tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + ccCurrentSourceCoverageExpectTarget(f, ccCurrentSourceCoverageResumeTurn(), int64(3), ccCurrentSourceCoverageTargetRecord(tc.status), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + } + + t.Run("legacy_item_version_zero_allowed", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "target") + f.svc.manager = nil + record := ccCurrentSourceCoverageTargetRecord(entity.EvalTargetRunStatusSuccess) + record.ItemVersionID = 0 + ccCurrentSourceCoverageExpectTarget(f, ccCurrentSourceCoverageResumeTurn(), int64(3), record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) +} + +func TestCCCurrentSourceCoverageValidateResumeEvaluator(t *testing.T) { + t.Run("evaluator_reader_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + f.svc.evaluatorRecordService = nil + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + ccCurrentSourceCoverageExpectTurn(f, []*entity.ExptTurnResultRunLog{turn}, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "evaluator source reader missing") + }) + + t.Run("evaluator_read_error", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + fault := errors.New("evaluator record read failed") + turn := ccCurrentSourceCoverageResumeTurn() + ccCurrentSourceCoverageExpectEvaluator(f, turn, nil, fault) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + t.Run("evaluator_record_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + ccCurrentSourceCoverageExpectEvaluator(f, turn, nil, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "original evaluator record missing") + }) + + for _, tc := range []struct { + name string + mutate func(*entity.EvaluatorRecord) + }{ + {name: "id_zero", mutate: func(r *entity.EvaluatorRecord) { r.ID = 0 }}, + {name: "space_zero", mutate: func(r *entity.EvaluatorRecord) { r.SpaceID = 0 }}, + {name: "experiment_zero", mutate: func(r *entity.EvaluatorRecord) { r.ExperimentID = 0 }}, + {name: "run_zero", mutate: func(r *entity.EvaluatorRecord) { r.ExperimentRunID = 0 }}, + {name: "item_zero", mutate: func(r *entity.EvaluatorRecord) { r.ItemID = 0 }}, + {name: "turn_negative", mutate: func(r *entity.EvaluatorRecord) { r.TurnID = -1 }}, + {name: "status_below_success", mutate: func(r *entity.EvaluatorRecord) { r.Status = entity.EvaluatorRunStatusUnknown }}, + {name: "status_above_skipped", mutate: func(r *entity.EvaluatorRecord) { r.Status = entity.EvaluatorRunStatus(5) }}, + {name: "source_type_negative", mutate: func(r *entity.EvaluatorRecord) { r.SourceType = entity.EvaluatorRecordSourceType(-1) }}, + {name: "source_type_above_inline", mutate: func(r *entity.EvaluatorRecord) { r.SourceType = entity.EvaluatorRecordSourceType(3) }}, + } { + t.Run("incomplete_"+tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + record := ccCurrentSourceCoverageEvaluatorRecord() + tc.mutate(record) + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "incomplete original evaluator record") + }) + } + + for _, tc := range []struct { + name string + mutate func(*entity.EvaluatorRecord) + }{ + {name: "record_id", mutate: func(r *entity.EvaluatorRecord) { r.ID = 62 }}, + {name: "experiment_id", mutate: func(r *entity.EvaluatorRecord) { r.ExperimentID = 6 }}, + {name: "run_id", mutate: func(r *entity.EvaluatorRecord) { r.ExperimentRunID = 8 }}, + {name: "item_id", mutate: func(r *entity.EvaluatorRecord) { r.ItemID = 12 }}, + {name: "item_version", mutate: func(r *entity.EvaluatorRecord) { r.ItemVersionID = 14 }}, + {name: "turn_id", mutate: func(r *entity.EvaluatorRecord) { r.TurnID = 1 }}, + } { + t.Run("identity_"+tc.name, func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + record := ccCurrentSourceCoverageEvaluatorRecord() + tc.mutate(record) + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + } + + t.Run("target_record_reference_mismatch", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + record := ccCurrentSourceCoverageEvaluatorRecord() + record.TargetRecordID = 42 + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + + t.Run("evaluator_refs_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + ccCurrentSourceCoverageExpectEvaluator(f, turn, ccCurrentSourceCoverageEvaluatorRecord(), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "current evaluator source references missing") + }) + + t.Run("inline_key_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "k", RecordID: 61}}} + record := ccCurrentSourceCoverageEvaluatorRecord() + record.SourceType = entity.EvaluatorRecordSourceTypeInline + record.EvaluatorVersionID = 0 + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "incomplete original inline evaluator source") + }) + + t.Run("inline_nil_reference", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{nil}} + record := ccCurrentSourceCoverageEvaluatorRecord() + record.SourceType = entity.EvaluatorRecordSourceTypeInline + record.EvaluatorVersionID = 0 + record.InlineKey = "k" + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "incomplete inline source reference") + }) + + t.Run("inline_match", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "k", RecordID: 61}}} + record := ccCurrentSourceCoverageEvaluatorRecord() + record.SourceType = entity.EvaluatorRecordSourceTypeInline + record.EvaluatorVersionID = 0 + record.InlineKey = "k" + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + + t.Run("inline_record_mismatch", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "k", RecordID: 62}}} + record := ccCurrentSourceCoverageEvaluatorRecord() + record.SourceType = entity.EvaluatorRecordSourceTypeInline + record.EvaluatorVersionID = 0 + record.InlineKey = "k" + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + + t.Run("inline_key_not_found", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "other", RecordID: 61}}} + record := ccCurrentSourceCoverageEvaluatorRecord() + record.SourceType = entity.EvaluatorRecordSourceTypeInline + record.EvaluatorVersionID = 0 + record.InlineKey = "k" + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + + t.Run("registered_version_missing", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + record := ccCurrentSourceCoverageEvaluatorRecord() + record.EvaluatorVersionID = 0 + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "original evaluator version missing") + }) + + t.Run("registered_nil_reference", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{nil}} + ccCurrentSourceCoverageExpectEvaluator(f, turn, ccCurrentSourceCoverageEvaluatorRecord(), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.False(t, valid) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "incomplete registered source reference") + }) + + t.Run("registered_match", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + ccCurrentSourceCoverageExpectEvaluator(f, turn, ccCurrentSourceCoverageEvaluatorRecord(), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + + t.Run("registered_record_mismatch", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 62}}} + ccCurrentSourceCoverageExpectEvaluator(f, turn, ccCurrentSourceCoverageEvaluatorRecord(), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + + t.Run("legacy_fallback_match", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 61}} + record := ccCurrentSourceCoverageEvaluatorRecord() + record.Alias = "" + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + + t.Run("legacy_fallback_requires_empty_alias", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 61}} + ccCurrentSourceCoverageExpectEvaluator(f, turn, ccCurrentSourceCoverageEvaluatorRecord(), nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + + t.Run("legacy_fallback_mismatch", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 62}} + record := ccCurrentSourceCoverageEvaluatorRecord() + record.Alias = "" + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.False(t, valid) + }) + + t.Run("legacy_item_version_zero_allowed", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + record := ccCurrentSourceCoverageEvaluatorRecord() + record.ItemVersionID = 0 + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) + + t.Run("matching_target_reference_allowed", func(t *testing.T) { + f := ccCurrentSourceCoverageNewResumeFixture(t, "evaluator") + turn := ccCurrentSourceCoverageResumeTurn() + turn.EvaluatorResultIds = ccCurrentSourceCoverageEvaluatorRefs() + record := ccCurrentSourceCoverageEvaluatorRecord() + record.TargetRecordID = turn.TargetResultID + ccCurrentSourceCoverageExpectEvaluator(f, turn, record, nil) + valid, err := f.svc.validateResumeSource(context.Background(), f.event, f.current) + require.NoError(t, err) + require.True(t, valid) + }) +} + +type ccCurrentSourceCoverageScanRepo struct { + repo.IExptItemResultRepo + repo.IExptCurrentResultRepo +} + +func ccCurrentSourceCoverageScannedSource() *entity.ExptCurrentItemRunSource { + now := time.Now() + return &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{ + SpaceID: 3, ExptID: 5, RunID: 7, ItemID: 11, ItemVersionID: 13, + RequestID: "rq-current", Mode: entity.EvaluationModeSubmit, + Status: entity.ItemRunState_Processing, RetryTimes: 2, UpdatedAt: &now, + }} +} + +func ccCurrentSourceCoverageScannedFixture() (*entity.ExptCurrentItemRunSource, *entity.ExptEvalItem, *entity.ExptScheduleEvent) { + source := ccCurrentSourceCoverageScannedSource() + item := &entity.ExptEvalItem{ + ItemID: 11, ExptID: 5, State: entity.ItemRunState_Processing, + Source: source, RetryTimes: source.Current.RetryTimes, UpdatedAt: source.Current.UpdatedAt, + } + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 5, ExptRunID: 7} + return source, item, event +} + +func ccCurrentSourceCoverageScanLocker(ctrl *gomock.Controller, locked bool, lockErr error) *lockmocks.MockILocker { + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:5:11", 5*time.Second, 5*time.Minute). + DoAndReturn(func(ctx context.Context, _ string, _, _ time.Duration) (bool, context.Context, func(), error) { + if lockErr != nil { + return false, nil, nil, lockErr + } + if !locked { + return false, nil, nil, nil + } + return true, ctx, func() {}, nil + }) + if locked { + locker.EXPECT().Unlock("expt_item_eval_run_lock:5:11").Return(true, nil) + } + return locker +} + +func ccCurrentSourceCoverageScanPort(ctrl *gomock.Controller) (*ccCurrentSourceCoverageScanRepo, *repomocks.MockIExptCurrentResultRepo) { + port := repomocks.NewMockIExptCurrentResultRepo(ctrl) + return &ccCurrentSourceCoverageScanRepo{ + IExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl), + IExptCurrentResultRepo: port, + }, port +} + +func TestCCCurrentSourceCoverageWithCurrentScannedItem(t *testing.T) { + t.Run("context_cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + scheduler := &ExptSchedulerImpl{} + applied, err := scheduler.withCurrentScannedItem(ctx, &entity.ExptScheduleEvent{}, nil, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + }) + + t.Run("item_missing", func(t *testing.T) { + scheduler := &ExptSchedulerImpl{} + applied, err := scheduler.withCurrentScannedItem(context.Background(), &entity.ExptScheduleEvent{}, nil, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "same-scan original item source missing") + }) + + t.Run("source_missing", func(t *testing.T) { + scheduler := &ExptSchedulerImpl{} + applied, err := scheduler.withCurrentScannedItem(context.Background(), &entity.ExptScheduleEvent{}, &entity.ExptEvalItem{ItemID: 11}, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "same-scan original item source missing") + }) + + t.Run("repo_lacks_current_port", func(t *testing.T) { + ctrl := gomock.NewController(t) + _, _, event := ccCurrentSourceCoverageScannedFixture() + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl)} + item := &entity.ExptEvalItem{ItemID: 11, Source: ccCurrentSourceCoverageScannedSource()} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "source-conditional scan writer unavailable") + }) + + t.Run("locker_factory_missing", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, _ := ccCurrentSourceCoverageScanPort(ctrl) + _, item, event := ccCurrentSourceCoverageScannedFixture() + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "scan item locker unavailable") + }) + + t.Run("locker_factory_returns_nil", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, _ := ccCurrentSourceCoverageScanPort(ctrl) + _, item, event := ccCurrentSourceCoverageScannedFixture() + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return nil }} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorContains(t, err, "scan item locker unavailable") + }) + + t.Run("lock_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, _ := ccCurrentSourceCoverageScanPort(ctrl) + _, item, event := ccCurrentSourceCoverageScannedFixture() + fault := errors.New("lock backend unavailable") + locker := ccCurrentSourceCoverageScanLocker(ctrl, false, fault) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + t.Run("lock_not_acquired", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, _ := ccCurrentSourceCoverageScanPort(ctrl) + _, item, event := ccCurrentSourceCoverageScannedFixture() + locker := ccCurrentSourceCoverageScanLocker(ctrl, false, nil) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, nil) + require.NoError(t, err) + require.False(t, applied) + }) + + t.Run("current_read_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, port := ccCurrentSourceCoverageScanPort(ctrl) + _, item, event := ccCurrentSourceCoverageScannedFixture() + fault := errors.New("current source read failed") + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(nil, fault) + locker := ccCurrentSourceCoverageScanLocker(ctrl, true, nil) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + for _, tc := range []struct { + name string + mutate func(*entity.ExptCurrentItemRunSource, *entity.ExptEvalItem) + }{ + {name: "stale_run", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { current.Current.RunID = 8 }}, + {name: "stale_request", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { + current.Current.RequestID = "rq-replacement" + }}, + {name: "stale_turns", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { + current.Turns = map[int64]*entity.ExptTurnResultRunLog{0: {ID: 1}} + }}, + {name: "not_processing", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { + current.Current.Status = entity.ItemRunState_Queueing + }}, + {name: "terminal_item", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { + current.Current.Status = entity.ItemRunState_Terminal + }}, + {name: "retry_count_moved", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { + current.Current.RetryTimes = 3 + }}, + {name: "scanned_updated_at_missing", mutate: func(_ *entity.ExptCurrentItemRunSource, item *entity.ExptEvalItem) { + item.Source.Current.UpdatedAt = nil + }}, + {name: "current_updated_at_missing", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { + current.Current.UpdatedAt = nil + }}, + {name: "updated_at_moved", mutate: func(current *entity.ExptCurrentItemRunSource, _ *entity.ExptEvalItem) { + moved := current.Current.UpdatedAt.Add(time.Second) + current.Current.UpdatedAt = &moved + }}, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, port := ccCurrentSourceCoverageScanPort(ctrl) + _, item, event := ccCurrentSourceCoverageScannedFixture() + current := ccCurrentSourceCoverageScannedSource() + current.Current.UpdatedAt = item.Source.Current.UpdatedAt + tc.mutate(current, item) + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(current, nil) + locker := ccCurrentSourceCoverageScanLocker(ctrl, true, nil) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, func(context.Context, repo.IExptCurrentResultRepo) (bool, error) { + t.Fatal("stale scan must not invoke the callback") + return false, nil + }) + require.NoError(t, err) + require.False(t, applied) + }) + } + + t.Run("callback_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, port := ccCurrentSourceCoverageScanPort(ctrl) + source, item, event := ccCurrentSourceCoverageScannedFixture() + current := ccCurrentSourceCoverageScannedSource() + current.Current.UpdatedAt = source.Current.UpdatedAt + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(current, nil) + locker := ccCurrentSourceCoverageScanLocker(ctrl, true, nil) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + fault := errors.New("callback failed") + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, func(context.Context, repo.IExptCurrentResultRepo) (bool, error) { + return false, fault + }) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) + }) + + t.Run("lock_context_cancelled_before_callback", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, port := ccCurrentSourceCoverageScanPort(ctrl) + source, item, event := ccCurrentSourceCoverageScannedFixture() + current := ccCurrentSourceCoverageScannedSource() + current.Current.UpdatedAt = source.Current.UpdatedAt + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(current, nil) + lockCtx, cancel := context.WithCancel(context.Background()) + cancel() + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:5:11", 5*time.Second, 5*time.Minute). + Return(true, lockCtx, func() {}, nil) + locker.EXPECT().Unlock("expt_item_eval_run_lock:5:11").Return(true, nil) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, func(context.Context, repo.IExptCurrentResultRepo) (bool, error) { + t.Fatal("lost lock context must not invoke the callback") + return false, nil + }) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + }) + + t.Run("unlock_error_does_not_discard_applied_write", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, port := ccCurrentSourceCoverageScanPort(ctrl) + source, item, event := ccCurrentSourceCoverageScannedFixture() + current := ccCurrentSourceCoverageScannedSource() + current.Current.UpdatedAt = source.Current.UpdatedAt + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(current, nil) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:5:11", 5*time.Second, 5*time.Minute). + DoAndReturn(func(ctx context.Context, _ string, _, _ time.Duration) (bool, context.Context, func(), error) { + return true, ctx, func() {}, nil + }) + locker.EXPECT().Unlock("expt_item_eval_run_lock:5:11").Return(false, errors.New("unlock failed")) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, func(context.Context, repo.IExptCurrentResultRepo) (bool, error) { + return true, nil + }) + require.NoError(t, err) + require.True(t, applied) + }) + + t.Run("current_processing_scan_applied", func(t *testing.T) { + ctrl := gomock.NewController(t) + writer, port := ccCurrentSourceCoverageScanPort(ctrl) + source, item, event := ccCurrentSourceCoverageScannedFixture() + current := ccCurrentSourceCoverageScannedSource() + current.Current.UpdatedAt = source.Current.UpdatedAt + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(5), int64(7), int64(11)).Return(current, nil) + locker := ccCurrentSourceCoverageScanLocker(ctrl, true, nil) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: writer, itemLocker: func() lock.ILocker { return locker }} + var observed *entity.ExptCurrentItemRunSource + applied, err := scheduler.withCurrentScannedItem(context.Background(), event, item, func(lockCtx context.Context, got repo.IExptCurrentResultRepo) (bool, error) { + require.NotNil(t, got) + execution := executionSource(lockCtx) + require.NotNil(t, execution) + observed = execution.Source + return true, nil + }) + require.NoError(t, err) + require.True(t, applied) + require.Same(t, source, observed) + }) +} diff --git a/backend/modules/evaluation/domain/service/expt_current_result_source_unit_test.go b/backend/modules/evaluation/domain/service/expt_current_result_source_unit_test.go new file mode 100644 index 0000000000..309334c418 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_current_result_source_unit_test.go @@ -0,0 +1,928 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// currentUnitRepo serves the item-result repository and the narrower +// source-conditional writers in one value. +type currentUnitRepo struct { + *repomocks.MockIExptItemResultRepo + *repomocks.MockIExptCurrentResultRepo +} + +func currentUnitSource() *entity.ExptCurrentItemRunSource { + return &entity.ExptCurrentItemRunSource{ + Current: entity.ExptItemCurrentRequest{ + SpaceID: 3, ExptID: 1, RunID: 2, ItemID: 9, ItemVersionID: 7, + RequestID: "req", Mode: entity.EvaluationModeFailRetry, Status: entity.ItemRunState_Processing, + }, + Turns: map[int64]*entity.ExptTurnResultRunLog{}, + } +} + +func currentUnitEvent() *entity.ExptItemEvalEvent { + return &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 9, RequestID: "req", ExptRunMode: entity.EvaluationModeFailRetry} +} + +type currentUnitDeps struct { + ctrl *gomock.Controller + items *currentUnitRepo + turns *repomocks.MockIExptTurnResultRepo + expts *repomocks.MockIExperimentRepo + config *componentmocks.MockIConfiger + publisher *eventmocks.MockExptEventPublisher + records *svcmocks.MockEvaluatorRecordService + targets *svcmocks.MockIEvalTargetService + manager *svcmocks.MockIExptManager + locker *lockmocks.MockILocker +} + +func newCurrentUnitDeps(t *testing.T) *currentUnitDeps { + t.Helper() + ctrl := gomock.NewController(t) + return ¤tUnitDeps{ + ctrl: ctrl, + items: ¤tUnitRepo{MockIExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl), MockIExptCurrentResultRepo: repomocks.NewMockIExptCurrentResultRepo(ctrl)}, + turns: repomocks.NewMockIExptTurnResultRepo(ctrl), + expts: repomocks.NewMockIExperimentRepo(ctrl), + config: componentmocks.NewMockIConfiger(ctrl), + publisher: eventmocks.NewMockExptEventPublisher(ctrl), + records: svcmocks.NewMockEvaluatorRecordService(ctrl), + targets: svcmocks.NewMockIEvalTargetService(ctrl), + manager: svcmocks.NewMockIExptManager(ctrl), + locker: lockmocks.NewMockILocker(ctrl), + } +} + +func (d *currentUnitDeps) eventService() *ExptItemEventEvalServiceImpl { + return &ExptItemEventEvalServiceImpl{ + manager: d.manager, + publisher: d.publisher, + exptItemResultRepo: d.items.MockIExptItemResultRepo, + exptTurnResultRepo: d.turns, + experimentRepo: d.expts, + configer: d.config, + itemLocker: func() lock.ILocker { return d.locker }, + evaluatorRecordService: d.records, + evaTargetService: d.targets, + } +} + +func (d *currentUnitDeps) scheduler() *ExptSchedulerImpl { + return &ExptSchedulerImpl{ + ExptItemResultRepo: d.items, + ExptTurnResultRepo: d.turns, + itemLocker: func() lock.ILocker { return d.locker }, + evalTargetService: d.targets, + } +} + +func TestSameExecutionIdentityUnit(t *testing.T) { + require.False(t, sameExecutionIdentity(nil, nil)) + require.False(t, sameExecutionIdentity(currentUnitSource(), nil)) + + base := currentUnitSource() + require.True(t, sameExecutionIdentity(base, currentUnitSource())) + + changed := currentUnitSource() + changed.Current.RequestID = "other" + require.False(t, sameExecutionIdentity(base, changed)) + + changed = currentUnitSource() + changed.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0}} + require.False(t, sameExecutionIdentity(base, changed)) + + left := currentUnitSource() + left.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0, TargetResultID: 5}} + right := currentUnitSource() + right.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0, TargetResultID: 6}} + require.False(t, sameExecutionIdentity(left, right)) + + left.Turns = map[int64]*entity.ExptTurnResultRunLog{0: {ID: 1, TargetResultID: 5}} + right.Turns = map[int64]*entity.ExptTurnResultRunLog{0: {ID: 1, TargetResultID: 5}} + right.CanonicalTurns[0].TargetResultID = 5 + require.True(t, sameExecutionIdentity(left, right)) +} + +func TestSourceForExecutionUnit(t *testing.T) { + d := newCurrentUnitDeps(t) + event := currentUnitEvent() + + t.Run("cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := sourceForExecution(ctx, d.items.MockIExptCurrentResultRepo, event) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("execution_lock_lost", func(t *testing.T) { + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: currentUnitSource(), LockLost: true}) + _, err := sourceForExecution(ctx, d.items.MockIExptCurrentResultRepo, event) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("execution_source", func(t *testing.T) { + source := currentUnitSource() + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: source}) + got, err := sourceForExecution(ctx, d.items.MockIExptCurrentResultRepo, event) + require.NoError(t, err) + require.Same(t, source, got) + }) + t.Run("read_error", func(t *testing.T) { + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, errors.New("read")) + _, err := sourceForExecution(context.Background(), d.items.MockIExptCurrentResultRepo, event) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("non_current", func(t *testing.T) { + source := currentUnitSource() + source.Current.RequestID = "other" + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + _, err := sourceForExecution(context.Background(), d.items.MockIExptCurrentResultRepo, event) + require.ErrorIs(t, err, errExptNonCurrentSource) + }) + t.Run("mode_mismatch", func(t *testing.T) { + source := currentUnitSource() + source.Current.Mode = entity.EvaluationModeRetryAll + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + _, err := sourceForExecution(context.Background(), d.items.MockIExptCurrentResultRepo, event) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("unknown_mode_allowed", func(t *testing.T) { + source := currentUnitSource() + source.Current.Mode = entity.EvaluationModeRetryAll + event := currentUnitEvent() + event.ExptRunMode = entity.EvaluationModeUnknown + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + got, err := sourceForExecution(context.Background(), d.items.MockIExptCurrentResultRepo, event) + require.NoError(t, err) + require.Same(t, source, got) + }) +} + +func TestCheckedCurrentWriteUnit(t *testing.T) { + require.True(t, errno.NeedMQRetry(checkedCurrentWrite(context.Background(), true, errors.New("write")))) + require.ErrorIs(t, checkedCurrentWrite(context.Background(), false, nil), errExptNonCurrentSource) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, errno.NeedMQRetry(checkedCurrentWrite(ctx, true, nil))) + require.NoError(t, checkedCurrentWrite(context.Background(), true, nil)) +} + +func TestFrozenTargetRecordIDsUnit(t *testing.T) { + require.Empty(t, frozenTargetRecordIDs(&entity.ExptCurrentItemRunSource{})) + source := &entity.ExptCurrentItemRunSource{ + Turns: map[int64]*entity.ExptTurnResultRunLog{0: {TargetResultID: 5}, 1: {TargetResultID: 5}}, + CanonicalTurns: []*entity.ExptTurnResult{{TurnID: 2, TargetResultID: 6}}, + } + require.Equal(t, []int64{5, 6}, frozenTargetRecordIDs(source)) +} + +func TestPrepareMissingRawTurnLogsUnit(t *testing.T) { + t.Run("cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, errno.NeedMQRetry(prepareMissingRawTurnLogs(ctx, nil, currentUnitSource(), &entity.ExptCurrentItemRunWrite{}))) + }) + t.Run("request_id_present_skips", func(t *testing.T) { + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), nil, currentUnitSource(), &entity.ExptCurrentItemRunWrite{})) + }) + t.Run("retry_only_skips", func(t *testing.T) { + source := currentUnitSource() + source.Current.RequestID = "" + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0}} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), nil, source, &entity.ExptCurrentItemRunWrite{RetryOnly: true})) + }) + t.Run("no_missing_skips", func(t *testing.T) { + source := currentUnitSource() + source.Current.RequestID = "" + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0}} + source.Turns = map[int64]*entity.ExptTurnResultRunLog{0: {TurnID: 0}} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), nil, source, &entity.ExptCurrentItemRunWrite{})) + }) + t.Run("generator_missing", func(t *testing.T) { + source := currentUnitSource() + source.Current.RequestID = "" + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0}} + err := prepareMissingRawTurnLogs(context.Background(), nil, source, &entity.ExptCurrentItemRunWrite{}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("generator_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + gen := idmocks.NewMockIIDGenerator(d.ctrl) + source := currentUnitSource() + source.Current.RequestID = "" + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0}} + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return(nil, errors.New("idgen")) + err := prepareMissingRawTurnLogs(context.Background(), gen, source, &entity.ExptCurrentItemRunWrite{}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("yield_and_terminal_states", func(t *testing.T) { + d := newCurrentUnitDeps(t) + gen := idmocks.NewMockIIDGenerator(d.ctrl) + source := currentUnitSource() + source.Current.RequestID = "" + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0, ExptRunID: 2}} + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{700}, nil) + write := &entity.ExptCurrentItemRunWrite{Yield: true} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), gen, source, write)) + require.Len(t, write.MissingTurnLogs, 1) + require.Equal(t, entity.TurnRunState_Queueing, write.MissingTurnLogs[0].Status) + require.Equal(t, int64(700), write.MissingTurnLogs[0].ID) + + gen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{701}, nil) + write = &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Terminal} + require.NoError(t, prepareMissingRawTurnLogs(context.Background(), gen, source, write)) + require.Equal(t, entity.TurnRunState_Terminal, write.MissingTurnLogs[0].Status) + }) +} + +func TestValidateResumeSourceUnit(t *testing.T) { + eventWith := func(source *entity.ExptResumeSource) *entity.ExptItemEvalEvent { + event := currentUnitEvent() + event.ResumeSource = source + return event + } + current := &entity.ExptItemCurrentRequest{ItemVersionID: 7} + validSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "evaluator"} + + t.Run("nil_source_async_rejected", func(t *testing.T) { + d := newCurrentUnitDeps(t) + event := currentUnitEvent() + event.AsyncReportTrigger = true + _, err := d.eventService().validateResumeSource(context.Background(), event, current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("nil_source_plain_allowed", func(t *testing.T) { + d := newCurrentUnitDeps(t) + valid, err := d.eventService().validateResumeSource(context.Background(), currentUnitEvent(), current) + require.NoError(t, err) + require.True(t, valid) + }) + for _, bad := range []*entity.ExptResumeSource{ + {RecordID: 0, TurnID: 0, InvokeID: "0", Kind: "evaluator"}, + {RecordID: 61, TurnID: -1, InvokeID: "61", Kind: "evaluator"}, + {RecordID: 61, TurnID: 0, InvokeID: "62", Kind: "evaluator"}, + {RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "other"}, + } { + t.Run("invalid_source", func(t *testing.T) { + d := newCurrentUnitDeps(t) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(bad), current) + require.True(t, errno.NeedMQRetry(err)) + }) + } + t.Run("turn_reader_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + svc := d.eventService() + svc.exptTurnResultRepo = nil + _, err := svc.validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("execution_source_used", func(t *testing.T) { + d := newCurrentUnitDeps(t) + source := currentUnitSource() + source.Turns = map[int64]*entity.ExptTurnResultRunLog{0: {ID: 1, SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}} + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: source}) + svc := d.eventService() + svc.manager = nil + d.targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(61)).Return(&entity.EvalTargetRecord{ID: 61, SpaceID: 3, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + valid, err := svc.validateResumeSource(ctx, eventWith(targetSource), current) + require.NoError(t, err) + require.True(t, valid) + }) + t.Run("turn_read_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), int64(9), int64(3)).Return(nil, errors.New("read")) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("nil_turn_log", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{nil}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("duplicate_turn", func(t *testing.T) { + d := newCurrentUnitDeps(t) + log := &entity.ExptTurnResultRunLog{TurnID: 0} + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{log, log}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("turn_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{TurnID: 1}}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("foreign_turn", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 99, TurnID: 0}}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("invalid_turn_state", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState(99)}}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_record_mismatch", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 62}}, nil) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + valid, err := d.eventService().validateResumeSource(context.Background(), eventWith(targetSource), current) + require.NoError(t, err) + require.False(t, valid) + }) + t.Run("target_reader_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + svc := d.eventService() + svc.evaTargetService = nil + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + _, err := svc.validateResumeSource(context.Background(), eventWith(targetSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("detail_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + d.manager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), gomock.Any()).Return(nil, errors.New("detail")) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(targetSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("detail_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + d.manager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), gomock.Any()).Return(nil, nil) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(targetSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_record_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + d.manager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), gomock.Any()).Return(&entity.Experiment{ID: 1, SpaceID: 3, TargetSpaceID: 4}, nil) + d.targets.EXPECT().GetRecordByID(gomock.Any(), int64(4), int64(61)).Return(nil, errors.New("read")) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(targetSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_record_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + d.manager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), gomock.Any()).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(61)).Return(nil, nil) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(targetSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_record_incomplete", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + d.manager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), gomock.Any()).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.targets.EXPECT().GetRecordByID(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvalTargetRecord{ID: 61}, nil) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(targetSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_record_identity", func(t *testing.T) { + for _, expected := range []bool{true, false} { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + d.manager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), gomock.Any()).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + record := &entity.EvalTargetRecord{ID: 61, SpaceID: 3, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)} + if !expected { + record.ItemVersionID = 8 + } + d.targets.EXPECT().GetRecordByID(gomock.Any(), gomock.Any(), gomock.Any()).Return(record, nil) + targetSource := &entity.ExptResumeSource{RecordID: 61, TurnID: 0, InvokeID: "61", Kind: "target"} + valid, err := d.eventService().validateResumeSource(context.Background(), eventWith(targetSource), current) + require.NoError(t, err) + require.Equal(t, expected, valid) + } + }) + t.Run("evaluator_reader_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + svc := d.eventService() + svc.evaluatorRecordService = nil + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success}}, nil) + _, err := svc.validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_read_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success}}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(61), false).Return(nil, errors.New("read")) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_record_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success}}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_record_incomplete", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success}}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 61}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_record_identity", func(t *testing.T) { + d := newCurrentUnitDeps(t) + log := &entity.ExptTurnResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, EvaluatorResultIds: &entity.EvaluatorResults{}} + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{log}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 99, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess}, nil) + valid, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.NoError(t, err) + require.False(t, valid) + }) + t.Run("evaluator_target_record_mismatch", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, TargetResultID: 61}}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess, TargetRecordID: 62}, nil) + valid, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.NoError(t, err) + require.False(t, valid) + }) + t.Run("evaluator_refs_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success}}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("inline_valid_and_missing", func(t *testing.T) { + for _, expected := range []bool{true, false} { + d := newCurrentUnitDeps(t) + log := &entity.ExptTurnResultRunLog{ + SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, + EvaluatorResultIds: &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "k", RecordID: 61}}}, + } + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{log}, nil) + record := &entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess, SourceType: entity.EvaluatorRecordSourceTypeInline, InlineKey: "k"} + if !expected { + record.InlineKey = "other" + } + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(record, nil) + valid, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.NoError(t, err) + require.Equal(t, expected, valid) + } + }) + t.Run("inline_key_empty", func(t *testing.T) { + d := newCurrentUnitDeps(t) + log := &entity.ExptTurnResultRunLog{ + SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, + EvaluatorResultIds: &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "k", RecordID: 61}}}, + } + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{log}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess, SourceType: entity.EvaluatorRecordSourceTypeInline}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("registered_valid", func(t *testing.T) { + d := newCurrentUnitDeps(t) + log := &entity.ExptTurnResultRunLog{ + SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}}, + } + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{log}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess, EvaluatorVersionID: 51, Alias: "a"}, nil) + valid, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.NoError(t, err) + require.True(t, valid) + }) + t.Run("registered_version_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + log := &entity.ExptTurnResultRunLog{ + SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0, Status: entity.TurnRunState_Success, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}}, + } + d.turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{log}, nil) + d.records.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess}, nil) + _, err := d.eventService().validateResumeSource(context.Background(), eventWith(validSource), current) + require.True(t, errno.NeedMQRetry(err)) + }) +} + +func TestWithCurrentScannedItemUnit(t *testing.T) { + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + now := time.Now() + item := func() *entity.ExptEvalItem { + return &entity.ExptEvalItem{ItemID: 9, State: entity.ItemRunState_Processing, Source: currentUnitSource()} + } + + t.Run("cancelled", func(t *testing.T) { + d := newCurrentUnitDeps(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := d.scheduler().withCurrentScannedItem(ctx, event, item(), nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("missing_source", func(t *testing.T) { + d := newCurrentUnitDeps(t) + _, err := d.scheduler().withCurrentScannedItem(context.Background(), event, &entity.ExptEvalItem{}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("locker_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + scheduler := d.scheduler() + scheduler.itemLocker = nil + _, err := scheduler.withCurrentScannedItem(context.Background(), event, item(), nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("locker_nil", func(t *testing.T) { + d := newCurrentUnitDeps(t) + scheduler := d.scheduler() + scheduler.itemLocker = func() lock.ILocker { return nil } + _, err := scheduler.withCurrentScannedItem(context.Background(), event, item(), nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("lock_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + _, err := d.scheduler().withCurrentScannedItem(context.Background(), event, item(), nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("lock_busy", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + applied, err := d.scheduler().withCurrentScannedItem(context.Background(), event, item(), nil) + require.NoError(t, err) + require.False(t, applied) + }) + t.Run("source_read_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, errors.New("read")) + _, err := d.scheduler().withCurrentScannedItem(context.Background(), event, item(), nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("identity_changed", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + other := currentUnitSource() + other.Current.RequestID = "other" + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(other, nil) + applied, err := d.scheduler().withCurrentScannedItem(context.Background(), event, item(), nil) + require.NoError(t, err) + require.False(t, applied) + }) + t.Run("updated_at_changed", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + other := currentUnitSource() + other.Current.UpdatedAt = &now + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(other, nil) + scanned := item() + later := now.Add(time.Second) + scanned.Source.Current.UpdatedAt = &later + applied, err := d.scheduler().withCurrentScannedItem(context.Background(), event, scanned, func(context.Context, repo.IExptCurrentResultRepo) (bool, error) { + t.Fatal("stale scan reached callback") + return false, nil + }) + require.NoError(t, err) + require.False(t, applied) + }) + t.Run("callback_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + currentSource := currentUnitSource() + currentSource.Current.UpdatedAt = &now + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(currentSource, nil) + scanned := item() + scanned.Source.Current.UpdatedAt = &now + _, err := d.scheduler().withCurrentScannedItem(context.Background(), event, scanned, func(context.Context, repo.IExptCurrentResultRepo) (bool, error) { + return false, errors.New("callback") + }) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("success", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + currentSource := currentUnitSource() + currentSource.Current.UpdatedAt = &now + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(currentSource, nil) + scanned := item() + scanned.Source.Current.UpdatedAt = &now + applied, err := d.scheduler().withCurrentScannedItem(context.Background(), event, scanned, func(context.Context, repo.IExptCurrentResultRepo) (bool, error) { + return true, nil + }) + require.NoError(t, err) + require.True(t, applied) + }) +} + +type currentUnitTerminator struct { + *svcmocks.MockIExptManager + called bool + err error +} + +func (m *currentUnitTerminator) TerminateCurrentRun(context.Context, int64, int64, int64, *entity.Session, *entity.ExptCurrentItemRunSource, ...entity.CompleteExptOptionFn) (bool, error) { + m.called = true + return true, m.err +} + +func TestHandleCurrentEventErrUnit(t *testing.T) { + newExec := func() *currentItemExecution { + return ¤tItemExecution{Source: currentUnitSource()} + } + lockOK := func(d *currentUnitDeps) { + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + } + sameSource := func(d *currentUnitDeps) { + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(currentUnitSource(), nil) + } + + t.Run("source_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + err := d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, ¤tItemExecution{}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("lock_lost", func(t *testing.T) { + d := newCurrentUnitDeps(t) + err := d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, ¤tItemExecution{Source: currentUnitSource(), LockLost: true}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("item_locker_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + svc := d.eventService() + svc.itemLocker = nil + err := svc.handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("factory_nil", func(t *testing.T) { + d := newCurrentUnitDeps(t) + svc := d.eventService() + svc.itemLocker = func() lock.ILocker { return nil } + err := svc.handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("lock_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + err := d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("busy", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + err := d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("read_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("read")) + err := d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("identity_changed", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + other := currentUnitSource() + other.Current.RequestID = "other" + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(other, nil) + require.NoError(t, d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec())) + }) + t.Run("finished_terminal", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + current := currentUnitSource() + current.Current.Status = entity.ItemRunState_Success + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(current, nil) + require.NoError(t, d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec())) + }) + t.Run("resume_invalid", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + event := currentUnitEvent() + event.ResumeSource = &entity.ExptResumeSource{RecordID: 0, Kind: "evaluator"} + err := d.eventService().handleCurrentEventErr(context.Background(), event, errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("experiment_reader_missing", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + svc := d.eventService() + svc.experimentRepo = nil + err := svc.handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("experiment_read_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, errors.New("read")) + err := d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("debt_without_terminator", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.config.EXPECT().GetErrRetryConf(gomock.Any(), int64(3), gomock.Any()).Return(&entity.RetryConf{IsInDebt: true}) + err := d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("debt_with_terminator", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.config.EXPECT().GetErrRetryConf(gomock.Any(), int64(3), gomock.Any()).Return(&entity.RetryConf{IsInDebt: true}) + svc := d.eventService() + terminator := ¤tUnitTerminator{MockIExptManager: d.manager} + svc.manager = terminator + require.NoError(t, svc.handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec())) + require.True(t, terminator.called) + }) + t.Run("retry_publish", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.config.EXPECT().GetErrRetryConf(gomock.Any(), int64(3), gomock.Any()).Return(&entity.RetryConf{RetryTimes: 1, RetryIntervalSecond: 1}) + d.items.MockIExptCurrentResultRepo.EXPECT().CompleteCurrentItemRun(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ *entity.ExptCurrentItemRunSource, write *entity.ExptCurrentItemRunWrite) (bool, error) { + require.True(t, write.RetryOnly) + require.Equal(t, int32(1), write.RetryTimes) + return true, nil + }) + d.publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + require.NoError(t, d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec())) + }) + t.Run("retry_yield", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.config.EXPECT().GetErrRetryConf(gomock.Any(), int64(3), gomock.Any()).Return(&entity.RetryConf{RetryTimes: 1}) + d.items.MockIExptCurrentResultRepo.EXPECT().CompleteCurrentItemRun(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + event := currentUnitEvent() + event.Ext = map[string]string{entity.RetryYieldExtKey: "true"} + require.NoError(t, d.eventService().handleCurrentEventErr(context.Background(), event, errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec())) + }) + t.Run("unretriable_success", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.config.EXPECT().GetErrRetryConf(gomock.Any(), int64(3), gomock.Any()).Return(&entity.RetryConf{RetryTimes: 0}) + d.items.MockIExptCurrentResultRepo.EXPECT().CompleteCurrentItemRun(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + require.NoError(t, d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec())) + }) + t.Run("write_not_applied", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.config.EXPECT().GetErrRetryConf(gomock.Any(), int64(3), gomock.Any()).Return(&entity.RetryConf{RetryTimes: 0}) + d.items.MockIExptCurrentResultRepo.EXPECT().CompleteCurrentItemRun(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil) + require.NoError(t, d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec())) + }) + t.Run("write_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + lockOK(d) + sameSource(d) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3}, nil) + d.config.EXPECT().GetErrRetryConf(gomock.Any(), int64(3), gomock.Any()).Return(&entity.RetryConf{RetryTimes: 0}) + d.items.MockIExptCurrentResultRepo.EXPECT().CompleteCurrentItemRun(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, errors.New("write")) + require.True(t, errno.NeedMQRetry(d.eventService().handleCurrentEventErr(context.Background(), currentUnitEvent(), errors.New("eval"), d.items.MockIExptCurrentResultRepo, newExec()))) + }) +} + +func TestHandleCurrentZombiesUnit(t *testing.T) { + now := time.Now() + old := now.Add(-10 * time.Second) + source := func() *entity.ExptCurrentItemRunSource { + s := currentUnitSource() + s.Current.UpdatedAt = &old + return s + } + item := func(state entity.ItemRunState, updated *time.Time) *entity.ExptEvalItem { + return &entity.ExptEvalItem{ItemID: 9, State: state, UpdatedAt: updated, Source: source()} + } + + t.Run("skips_nil_and_unfinished_and_recent", func(t *testing.T) { + d := newCurrentUnitDeps(t) + alives, zombies, err := d.scheduler().handleCurrentZombies(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{nil, item(entity.ItemRunState_Success, &old), item(entity.ItemRunState_Processing, &now), item(entity.ItemRunState_Processing, nil)}, nil, 1, false) + require.NoError(t, err) + require.Len(t, zombies, 0) + require.Len(t, alives, 2) + }) + t.Run("callback_error", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + _, _, err := d.scheduler().handleCurrentZombies(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{item(entity.ItemRunState_Processing, &old)}, nil, 1, false) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("applied_and_not_applied", func(t *testing.T) { + for _, applied := range []bool{true, false} { + d := newCurrentUnitDeps(t) + scanned := item(entity.ItemRunState_Processing, &old) + current := source() + current.Current.UpdatedAt = &old + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(current, nil) + d.items.MockIExptCurrentResultRepo.EXPECT().CompleteCurrentItemRun(gomock.Any(), gomock.Any(), gomock.Any()).Return(applied, nil) + alives, zombies, err := d.scheduler().handleCurrentZombies(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{scanned}, nil, 1, false) + require.NoError(t, err) + if applied { + require.Len(t, zombies, 1) + } else { + require.Len(t, alives, 1) + } + } + }) +} + +func TestSweepCurrentSandboxItemsUnit(t *testing.T) { + now := time.Now() + old := now.Add(-10 * time.Second) + source := func(withTarget bool) *entity.ExptCurrentItemRunSource { + s := currentUnitSource() + s.Current.UpdatedAt = &old + if withTarget { + s.CanonicalTurns = []*entity.ExptTurnResult{{ID: 71, TurnID: 0, TargetResultID: 88}} + } + return s + } + item := func(state entity.ItemRunState, withTarget bool) *entity.ExptEvalItem { + return &entity.ExptEvalItem{ItemID: 9, State: state, UpdatedAt: &old, Source: source(withTarget)} + } + expt := &entity.Experiment{ID: 1, SpaceID: 3} + + t.Run("non_processing_alive", func(t *testing.T) { + d := newCurrentUnitDeps(t) + alives, terminated, err := d.scheduler().sweepCurrentSandboxItems(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{nil, item(entity.ItemRunState_Success, false)}, expt) + require.NoError(t, err) + require.Len(t, alives, 2) + require.Len(t, terminated, 0) + }) + t.Run("no_target_ids", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source(false), nil) + alives, terminated, err := d.scheduler().sweepCurrentSandboxItems(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{item(entity.ItemRunState_Processing, false)}, expt) + require.NoError(t, err) + require.Len(t, alives, 1) + require.Len(t, terminated, 0) + }) + t.Run("not_stopped", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source(true), nil) + d.targets.EXPECT().CheckSandboxTerminated(gomock.Any(), int64(3), []int64{88}).Return(nil, nil) + alives, _, err := d.scheduler().sweepCurrentSandboxItems(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{item(entity.ItemRunState_Processing, true)}, expt) + require.NoError(t, err) + require.Len(t, alives, 1) + }) + t.Run("stopped_terminates", func(t *testing.T) { + d := newCurrentUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.items.MockIExptCurrentResultRepo.EXPECT().GetCurrentItemRunSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source(true), nil) + d.targets.EXPECT().CheckSandboxTerminated(gomock.Any(), gomock.Any(), []int64{88}).Return([]int64{88}, map[int64]string{88: ""}) + d.targets.EXPECT().TerminateAsyncRecordsAndDestroySandbox(gomock.Any(), gomock.Any(), []int64{88}, gomock.Any(), gomock.Any(), false).Return(nil) + d.items.MockIExptCurrentResultRepo.EXPECT().CompleteCurrentItemRun(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + _, terminated, err := d.scheduler().sweepCurrentSandboxItems(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{item(entity.ItemRunState_Processing, true)}, expt) + require.NoError(t, err) + require.Len(t, terminated, 1) + }) +} diff --git a/backend/modules/evaluation/domain/service/expt_current_retry_budget_test.go b/backend/modules/evaluation/domain/service/expt_current_retry_budget_test.go new file mode 100644 index 0000000000..5206cc2f1a --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_current_retry_budget_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + confmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func retryBudgetHandler(t *testing.T) (*ExptItemEventEvalServiceImpl, *repomocks.MockIExptCurrentResultRepo, *repomocks.MockIExperimentRepo, *entity.ExptCurrentItemRunSource, *entity.ExptItemEvalEvent, *bool) { + t.Helper() + ctrl := gomock.NewController(t) + mutex := lockmocks.NewMockILocker(ctrl) + held := false + mutex.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:7:10", 5*time.Second, 5*time.Minute). + DoAndReturn(func(ctx context.Context, _ string, _, _ time.Duration) (bool, context.Context, func(), error) { + held = true + return true, ctx, func() {}, nil + }) + mutex.EXPECT().Unlock("expt_item_eval_run_lock:7:10").DoAndReturn(func(string) (bool, error) { + require.True(t, held) + held = false + return true, nil + }) + port := repomocks.NewMockIExptCurrentResultRepo(ctrl) + expts := repomocks.NewMockIExperimentRepo(ctrl) + svc := &ExptItemEventEvalServiceImpl{ + itemLocker: func() lock.ILocker { return mutex }, experimentRepo: expts, + configer: confmocks.NewMockIConfiger(ctrl), publisher: eventmocks.NewMockExptEventPublisher(ctrl), + } + source := &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{ + SpaceID: 9, ExptID: 7, RunID: 8, ItemID: 10, ItemVersionID: 11, + RequestID: "current-request", Mode: entity.EvaluationModeFailRetry, Status: entity.ItemRunState_Processing, + }} + event := &entity.ExptItemEvalEvent{SpaceID: 9, ExptID: 7, ExptRunID: 8, EvalSetItemID: 10, RequestID: source.Current.RequestID, ExptRunMode: source.Current.Mode} + return svc, port, expts, source, event, &held +} + +func TestHandleCurrentEventErrFrozenRetryBudget(t *testing.T) { + for _, tc := range []struct { + name string + frozen *entity.EvaluationConfiguration + space, eventLimit, db, message int + want int32 + retry bool + }{ + {name: "explicit_zero", frozen: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(0)}, space: 3, want: 0}, + {name: "remaining_db_max", frozen: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3)}, space: 3, db: 1, want: 2, retry: true}, + {name: "remaining_event_max", frozen: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3)}, space: 3, message: 1, want: 2, retry: true}, + {name: "exhausted", frozen: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3)}, space: 3, db: 3, message: 1, want: 3}, + {name: "frozen_overrides_positive_event", frozen: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3)}, eventLimit: 1, db: 1, want: 2, retry: true}, + {name: "absent_evalconf_event_fallback", eventLimit: 1, space: 3, db: 1, want: 1}, + {name: "absent_evalconf_space_fallback", space: 3, db: 1, want: 2, retry: true}, + {name: "absent_pointer_event_fallback", frozen: &entity.EvaluationConfiguration{}, eventLimit: 3, db: 1, want: 2, retry: true}, + {name: "absent_pointer_space_fallback", frozen: &entity.EvaluationConfiguration{}, space: 3, db: 1, want: 2, retry: true}, + } { + t.Run(tc.name, func(t *testing.T) { + svc, port, expts, source, event, held := retryBudgetHandler(t) + source.Current.RetryTimes = int32(tc.db) + event.RetryTimes, event.MaxRetryTimes = tc.message, tc.eventLimit + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(9), int64(7), int64(8), int64(10)).Return(source, nil) + reads := 0 + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).DoAndReturn(func(context.Context, int64, int64) (*entity.Experiment, error) { + require.True(t, *held, "frozen budget read must remain inside the item lock") + reads++ + return &entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, EvalConf: tc.frozen}, nil + }).AnyTimes() + svc.configer.(*confmocks.MockIConfiger).EXPECT().GetErrRetryConf(gomock.Any(), int64(9), gomock.Any()).Return(&entity.RetryConf{RetryTimes: tc.space}) + var written *entity.ExptCurrentItemRunWrite + port.EXPECT().CompleteCurrentItemRun(gomock.Any(), source, gomock.Any()).DoAndReturn(func(_ context.Context, _ *entity.ExptCurrentItemRunSource, w *entity.ExptCurrentItemRunWrite) (bool, error) { + require.True(t, *held) + written = w + return true, nil + }) + var published *entity.ExptItemEvalEvent + svc.publisher.(*eventmocks.MockExptEventPublisher).EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), nil). + DoAndReturn(func(_ context.Context, e *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + published = e + return nil + }).AnyTimes() + err := svc.handleCurrentEventErr(context.Background(), event, errors.New("execution failed"), port, ¤tItemExecution{Source: source}) + require.NoError(t, err) + require.NotNil(t, written) + require.Equal(t, tc.retry, written.RetryOnly, "explicit zero must terminate rather than schedule an automatic retry") + require.Equal(t, tc.want, written.RetryTimes) + require.Equal(t, entity.ItemRunState_Fail, written.Status) + require.Equal(t, !tc.retry, written.FailPendingTurns) + require.False(t, written.Yield) + require.Equal(t, 1, reads) + require.Equal(t, tc.message, event.RetryTimes, "input event must remain unchanged") + require.False(t, *held) + if tc.retry { + require.NotNil(t, published) + require.Equal(t, int(tc.want), published.RetryTimes) + require.Equal(t, event.RequestID, published.RequestID) + } else { + require.Nil(t, published) + } + }) + } +} + +func TestHandleCurrentEventErrFrozenRetryReadFailure(t *testing.T) { + svc, port, expts, source, event, held := retryBudgetHandler(t) + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(9), int64(7), int64(8), int64(10)).Return(source, nil) + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).DoAndReturn(func(context.Context, int64, int64) (*entity.Experiment, error) { + require.True(t, *held) + return nil, errors.New("frozen experiment read failed") + }) + err := svc.handleCurrentEventErr(context.Background(), event, errors.New("execution failed"), port, ¤tItemExecution{Source: source}) + require.ErrorContains(t, err, "frozen experiment read failed") + require.True(t, errno.NeedMQRetry(err)) + require.False(t, *held) + require.Zero(t, event.RetryTimes) + require.Zero(t, source.Current.RetryTimes) + // Unconfigured mock methods reject any result/source completion or publication. +} + +func TestHandleCurrentEventErrStaleBeforeFrozenBudgetRead(t *testing.T) { + for _, stale := range []string{"request", "run", "missing_source"} { + t.Run(stale, func(t *testing.T) { + svc, port, _, source, event, held := retryBudgetHandler(t) + current := *source + observed := ¤t + switch stale { + case "request": + current.Current.RequestID = "replacement-request" + case "run": + current.Current.RunID = 18 + case "missing_source": + observed = nil + } + port.EXPECT().GetCurrentItemRunSource(gomock.Any(), int64(9), int64(7), int64(8), int64(10)).Return(observed, nil) + require.NoError(t, svc.handleCurrentEventErr(context.Background(), event, errors.New("late failure"), port, ¤tItemExecution{Source: source})) + require.False(t, *held) + require.Zero(t, event.RetryTimes) + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_guard_source_test.go b/backend/modules/evaluation/domain/service/expt_guard_source_test.go new file mode 100644 index 0000000000..9f73d78771 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_guard_source_test.go @@ -0,0 +1,436 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/ctxcache" +) + +// Removing the source check before Confirm must make these real-DAO cases fail. +func TestGuardSourceMySQLDirectConsumerChecksBeforeConfirm(t *testing.T) { + for _, kind := range []string{"old_empty", "old_rq1", "terminal", "read_error"} { + t.Run(kind, func(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + event := f.event() + switch kind { + case "old_empty": + event.RequestID = "" + f.token("rq2") + case "old_rq1": + f.token("rq2") + case "terminal": + for _, table := range []string{"expt_item_result", "expt_item_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Success).Error) + } + case "read_error": + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("ext", []byte(`{"manual_retry":"invalid"}`)).Error) + } + guard := &fakeGuard{confirmResult: true} + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, exptItemResultRepo: f.itemRepo(), dispatchRepo: experimentrepo.NewExptItemDispatchRepo(experimentmysql.NewExptItemDispatchDAO(f.provider))} + ctx := ctxcache.Init(context.Background()) + event.WithCtxCentralAdmittedExpt(ctx, &entity.Experiment{SchedulerScope: testScope, ExptDispatchMode: entity.ExptDispatchModeEnforce}) + before := f.snapshot() + executed := 0 + err := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { executed++; return nil })(ctx, event) + if kind == "read_error" { + require.True(t, errno.NeedMQRetry(err), "source read error must remain broker retry") + } else { + require.NoError(t, err) + } + require.Zero(t, guard.confirmCalls, "stale/terminal/unreadable original source must stop before quota") + require.Zero(t, executed) + require.Empty(t, guard.releases()) + require.Equal(t, before, f.snapshot()) + t.Log("REAL_MYSQL_GUARD_SOURCE_PRECHECK zero quota/execution/state effects") + }) + } +} + +func TestGuardSourceReleaseRejectsNewTerminalObservation(t *testing.T) { + ctrl := gomock.NewController(t) + dispatch := repomocks.NewMockIExptItemDispatchRepo(ctrl) + dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), int64(3), int64(1), int64(2), []int64{4}).Return([]*repo.ExptDispatchObservation{{ItemID: 4, RequestID: "rq2", Status: int32(entity.ItemRunState_Success)}}, nil) + guard := &fakeGuard{} + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"} + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, dispatchRepo: dispatch} + svc.releaseQuotaIfItemTerminal(context.Background(), event, testScope, guard, nil) + require.Empty(t, guard.releases(), "rq2 terminal is not evidence that processing-start rq1 completed") +} + +type guardSourceContextKey struct{} + +type boundedSourceGuard struct { + fakeGuard + t *testing.T +} + +func (g *boundedSourceGuard) Release(ctx context.Context, scope string, run, item int64, expectedRequestID, reason string) error { + require.NoError(g.t, ctx.Err(), "resource release must survive parent cancellation") + require.Equal(g.t, "scope-value", ctx.Value(guardSourceContextKey{})) + deadline, ok := ctx.Deadline() + require.True(g.t, ok) + require.Positive(g.t, time.Until(deadline)) + require.LessOrEqual(g.t, time.Until(deadline), 8*time.Second) + return g.fakeGuard.Release(ctx, scope, run, item, expectedRequestID, reason) +} + +func TestGuardSourceOutsideReleaseHasIndependentBound(t *testing.T) { + guard := &boundedSourceGuard{t: t} + ctx, cancel := context.WithCancel(ctxcache.Init(context.WithValue(context.Background(), guardSourceContextKey{}, "scope-value"))) + event := &entity.ExptItemEvalEvent{ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"} + event.WithCtxCentralQuotaHeld(ctx, testScope) + ctx = context.WithValue(ctx, currentItemSourceKey{}, ¤tItemExecution{}) + cancel() + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard} + svc.releaseCentralQuotaOutsideGate(ctx, event, errors.New("debt"), "expt terminated on indebt") + svc.releaseCentralQuotaOutsideGate(ctx, event, errors.New("debt"), "expt terminated on indebt") + require.Len(t, guard.releases(), 1) + require.True(t, executionSource(ctx).QuotaReleased) +} + +func TestGuardSourceMySQLCompletedThenCancelReleases(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + source := f.source() + guard := &boundedSourceGuard{t: t} + guard.confirmResult = true + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, exptItemResultRepo: f.itemRepo(), dispatchRepo: experimentrepo.NewExptItemDispatchRepo(experimentmysql.NewExptItemDispatchDAO(f.provider))} + ctx, cancel := context.WithCancel(ctxcache.Init(context.WithValue(context.Background(), guardSourceContextKey{}, "scope-value"))) + defer cancel() + ctx = context.WithValue(ctx, currentItemSourceKey{}, ¤tItemExecution{Source: source}) + event := f.event() + event.WithCtxCentralAdmittedExpt(ctx, &entity.Experiment{SchedulerScope: testScope, ExptDispatchMode: entity.ExptDispatchModeEnforce}) + var completed map[string]string + err := svc.HandleCentralReservation(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + applied, err := f.port.CompleteCurrentItemRun(ctx, source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, FailPendingTurns: true}) + require.NoError(t, err) + require.True(t, applied) + executionSource(ctx).Completed = true + completed = f.snapshot() + cancel() + return errors.New("debt") + })(ctx, event) + require.True(t, errno.NeedMQRetry(err)) + require.Len(t, guard.releases(), 1, "committed terminal source still owes release after cancellation") + require.Equal(t, completed, f.snapshot(), "independent release must not authorize state writes") + t.Log("REAL_MYSQL_GUARD_SOURCE_COMPLETED_CANCEL resource-only release") +} + +func TestGuardSourceMySQLManagerUsesObservedTokens(t *testing.T) { + for _, kind := range []string{"original", "canonical_terminal", "terminal_run", "moved", "bad_ext"} { + t.Run(kind, func(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + if kind == "canonical_terminal" { + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Fail).Error) + } + if kind == "terminal_run" { + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Update("status", entity.ExptStatus_Terminated).Error) + } + if kind == "moved" { + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("expt_run_id", f.run+1).Error) + } + if kind == "bad_ext" { + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("ext", []byte(`{"manual_retry":"invalid"}`)).Error) + } + guard := &fakeGuard{} + manager := &ExptMangerImpl{centralGuard: guard, itemResultRepo: f.itemRepo()} + expt := &entity.Experiment{ID: f.expt, SpaceID: f.space, LatestRunID: f.run, SchedulerScope: testScope, ExptDispatchMode: entity.ExptDispatchModeEnforce} + before := f.snapshot() + manager.releaseCentralQuotaForRun(context.Background(), expt, &f.run, "run terminal", "test") + if kind == "original" || kind == "canonical_terminal" || kind == "terminal_run" { + require.Len(t, guard.releases(), 3) + for _, release := range guard.releases() { + if release.ItemID == f.item { + require.Equal(t, "rq1", release.RequestID) + } else { + require.Empty(t, release.RequestID) + } + } + } else { + require.Empty(t, guard.releases(), "incomplete source cannot manufacture an empty or latest token") + } + require.Equal(t, before, f.snapshot()) + t.Log("REAL_MYSQL_GUARD_MANAGER original primary observations retained") + }) + } +} + +func TestGuardSourceOutsideReleaseUsesFrozenExecution(t *testing.T) { + guard := &fakeGuard{} + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq2"} + ctx := ctxcache.Init(context.Background()) + event.WithCtxCentralQuotaHeld(ctx, testScope) + ctx = context.WithValue(ctx, currentItemSourceKey{}, ¤tItemExecution{Source: &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{SpaceID: 3, ExptID: 1, RunID: 2, ItemID: 4, RequestID: "rq1"}}}) + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard} + svc.releaseCentralQuotaOutsideGate(ctx, event, errors.New("debt"), "terminal") + require.Len(t, guard.releases(), 1) + require.Equal(t, "rq1", guard.releases()[0].RequestID) +} + +func TestGuardSourceMySQLConfirmReceivesOriginalToken(t *testing.T) { + for _, token := range []string{"", "rq1", "rq2"} { + t.Run("token="+token, func(t *testing.T) { + f := newLateWriteFixture(t, token) + event := f.event() + event.RequestID = token + if token == "" { + event.ExptRunMode = entity.EvaluationModeSubmit + } + guard := &fakeGuard{confirmResult: true} + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, exptItemResultRepo: f.itemRepo(), dispatchRepo: experimentrepo.NewExptItemDispatchRepo(experimentmysql.NewExptItemDispatchDAO(f.provider))} + ctx := admittedEnforceCtx(event) + executed := 0 + err := svc.HandleCentralReservation(func(ctx context.Context, ev *entity.ExptItemEvalEvent) error { + require.Equal(t, token, executionSource(ctx).Source.Current.RequestID) + require.Equal(t, token, ev.RequestID) + executed++ + // A downstream mutation cannot change the caller's original event. + ev.RequestID = "changed" + return nil + })(ctx, event) + require.NoError(t, err) + require.Equal(t, []string{token}, guard.confirmTokens) + require.Equal(t, 1, executed) + require.Equal(t, token, event.RequestID) + require.Empty(t, guard.releases()) + t.Log("REAL_MYSQL_GUARD_CONFIRM_ORIGINAL_TOKEN exact source checked before guard") + }) + } +} + +func TestGuardSourceMySQLCompletionThenRequeueDoesNotReleaseNew(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + source := f.source() + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, FailPendingTurns: true}) + require.NoError(t, err) + require.True(t, applied) + f.token("rq2") + for _, table := range []string{"expt_item_result", "expt_item_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Success).Error) + } + guard := &fakeGuard{} + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, dispatchRepo: experimentrepo.NewExptItemDispatchRepo(experimentmysql.NewExptItemDispatchDAO(f.provider))} + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: source, Completed: true}) + before := f.snapshot() + svc.releaseQuotaIfItemTerminal(ctx, f.event(), testScope, guard, nil) + require.Empty(t, guard.releases()) + require.False(t, executionSource(ctx).QuotaReleased) + require.Equal(t, before, f.snapshot()) + t.Log("REAL_MYSQL_GUARD_LATE_RELEASE rq2 terminal never releases rq1 execution") +} + +func TestGuardSourceOutsideReleaseFailureDoesNotSetFlag(t *testing.T) { + guard := &fakeGuard{releaseErr: errors.New("storage unavailable")} + event := &entity.ExptItemEvalEvent{ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"} + ctx := ctxcache.Init(context.Background()) + event.WithCtxCentralQuotaHeld(ctx, testScope) + ctx = context.WithValue(ctx, currentItemSourceKey{}, ¤tItemExecution{}) + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard} + svc.releaseCentralQuotaOutsideGate(ctx, event, errors.New("debt"), "terminal") + require.False(t, executionSource(ctx).QuotaReleased) + guard.releaseErr = nil + svc.releaseCentralQuotaOutsideGate(ctx, event, errors.New("debt"), "terminal") + svc.releaseCentralQuotaOutsideGate(ctx, event, errors.New("debt"), "terminal") + require.Len(t, guard.releases(), 2) + require.True(t, executionSource(ctx).QuotaReleased) +} + +func TestGuardSourceMySQLDebtFinishThenCancelStillReleases(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + source := f.source() + event := f.event() + event.RetryTimes = 3 + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetErrRetryConf(gomock.Any(), f.space, gomock.Any()).Return(&entity.RetryConf{RetryTimes: 3, IsInDebt: true}).AnyTimes() + manager := svcmocks.NewMockIExptManager(ctrl) + ctx, cancel := context.WithCancel(ctxcache.Init(context.WithValue(context.Background(), guardSourceContextKey{}, "scope-value"))) + defer cancel() + event.WithCtxCentralQuotaHeld(ctx, testScope) + guard := &boundedSourceGuard{t: t} + terminator := &guardSourceTestTerminator{IExptManager: manager, finish: func(ctx context.Context, original *entity.ExptCurrentItemRunSource) (bool, error) { + require.True(t, sameExecutionIdentity(source, original)) + cancel() + releaseCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) + defer stop() + if err := guard.Release(releaseCtx, testScope, f.run, f.item, original.Current.RequestID, "debt"); err != nil { + return true, err + } + return true, errno.WrapMQRetryErr(ctx.Err()) + }} + svc := &ExptItemEventEvalServiceImpl{experimentRepo: f.experimentRepo(), centralGuard: guard, exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), manager: terminator, configer: config, itemLocker: lateWriteLocker(t, ctrl)} + var completed map[string]string + err := svc.HandleEventErr(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + executionSource(ctx).Source = source + executor := &ExptItemEvalCtxExecutor{ItemResultRepo: f.itemRepo(), Configer: config} + err := executor.CompleteItemRun(ctx, &entity.ExptItemEvalCtx{Event: event}, errors.New("debt")) + completed = f.snapshot() + return err + })(ctx, event) + require.True(t, errno.NeedMQRetry(err)) + require.Len(t, guard.releases(), 1) + require.Equal(t, "rq1", guard.releases()[0].RequestID) + require.Equal(t, completed, f.snapshot()) + t.Log("REAL_MYSQL_GUARD_DEBT_CANCEL_AFTER_FINISH bounded resource release, no new writes") +} + +type guardSourceScannerRepo struct { + repo.IExptItemResultRepo + repo.IExptCurrentResultRepo + afterComplete func() +} + +func (r *guardSourceScannerRepo) CompleteCurrentItemRun(ctx context.Context, source *entity.ExptCurrentItemRunSource, write *entity.ExptCurrentItemRunWrite) (bool, error) { + applied, err := r.IExptCurrentResultRepo.CompleteCurrentItemRun(ctx, source, write) + if applied && err == nil { + r.afterComplete() + } + return applied, err +} + +func TestGuardSourceMySQLZombieRetainsScannedToken(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + old := time.Now().Add(-time.Hour) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"updated_at": old}).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("evaluator_result_ids", []byte(`{}`)).Error) + source := f.source() + guard := &fakeGuard{} + items := &guardSourceScannerRepo{IExptItemResultRepo: f.itemRepo(), IExptCurrentResultRepo: f.port, afterComplete: func() { f.token("rq2") }} + ctrl := gomock.NewController(t) + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: items, centralGuard: guard, itemLocker: lateWriteLocker(t, ctrl)} + scanned := &entity.ExptEvalItem{ItemID: f.item, Source: source, RequestID: "rq1", State: entity.ItemRunState_Processing, UpdatedAt: source.Current.UpdatedAt, RetryTimes: source.Current.RetryTimes} + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run} + expt := &entity.Experiment{ID: f.expt, SchedulerScope: testScope, ExptDispatchMode: entity.ExptDispatchModeEnforce} + _, zombies, err := scheduler.handleCurrentZombies(context.Background(), event, []*entity.ExptEvalItem{scanned}, expt, 1, false) + require.NoError(t, err) + require.Len(t, zombies, 1) + require.Len(t, guard.releases(), 1) + require.Equal(t, "rq1", guard.releases()[0].RequestID) + require.Equal(t, "rq2", f.source().Current.RequestID) + t.Log("REAL_MYSQL_GUARD_ZOMBIE captured rq1 retained after rq2 replacement") +} + +func TestGuardSourceMySQLProductionChainStopsBeforeQuota(t *testing.T) { + for _, kind := range []string{"old", "terminal", "error"} { + t.Run(kind, func(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + event := f.event() + switch kind { + case "old": + f.token("rq2") + case "terminal": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Success).Error) + case "error": + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("ext", []byte(`{"manual_retry":"invalid"}`)).Error) + } + ctrl := gomock.NewController(t) + guard := &fakeGuard{confirmResult: true} + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, exptItemResultRepo: f.itemRepo(), itemLocker: lateWriteLocker(t, ctrl)} + calls := 0 + before := f.snapshot() + handler := RecordEvalChain(svc.HandleEventErr, svc.HandleEventLock, svc.HandleCurrentSource, svc.HandleCentralReservation)(func(context.Context, *entity.ExptItemEvalEvent) error { calls++; return nil }) + err := handler(admittedEnforceCtx(event), event) + if kind == "error" { + require.True(t, errno.NeedMQRetry(err)) + } else { + require.NoError(t, err) + } + require.Zero(t, calls) + require.Zero(t, guard.confirmCalls) + require.Empty(t, guard.releases()) + require.Equal(t, before, f.snapshot()) + t.Log("REAL_MYSQL_GUARD_PRODUCTION_CHAIN source/lock before quota, error reaches broker") + }) + } +} + +func TestGuardSourceMySQLAbsentConfirmRequeuesOriginal(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + event := f.event() + guard := &fakeGuard{confirmResult: false} + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, exptItemResultRepo: f.itemRepo(), dispatchRepo: experimentrepo.NewExptItemDispatchRepo(experimentmysql.NewExptItemDispatchDAO(f.provider))} + err := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { + t.Fatal("absent must stop execution") + return nil + })(admittedEnforceCtx(event), event) + require.NoError(t, err) + require.Equal(t, []string{"rq1"}, guard.confirmTokens) + current, err := f.itemRepo().GetCurrentRequest(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + require.Equal(t, "rq1", current.RequestID) + require.Equal(t, entity.ItemRunState_Queueing, current.Status) + require.Empty(t, guard.releases()) + t.Log("REAL_MYSQL_GUARD_FALSE original rq1 requeue CAS, no wildcard/new token") +} + +func TestGuardSourceMySQLConcurrentOldAndCurrentConsumer(t *testing.T) { + f := newFilterSyncFixture(t) + // Use the real Redis item mutex with the real MySQL source repository. + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Processing).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Processing), "result_state": int32(entity.ExptItemResultStateDefault)}).Error) + f.token("rq2") + guard := &fakeGuard{confirmResult: true} + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, exptItemResultRepo: f.itemRepo(), itemLocker: func() lock.ILocker { return filterRealTestLocker(f.client) }} + var wg sync.WaitGroup + start := make(chan struct{}) + type delivery struct { + token string + err error + } + errs := make(chan delivery, 2) + for _, token := range []string{"rq1", "rq2"} { + wg.Add(1) + go func(token string) { + defer wg.Done() + <-start + event := f.event() + event.RequestID = token + handler := RecordEvalChain(svc.HandleEventLock, svc.HandleCurrentSource, svc.HandleCentralReservation)(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) + errs <- delivery{token: token, err: handler(admittedEnforceCtx(event), event)} + }(token) + } + close(start) + wg.Wait() + close(errs) + for result := range errs { + if errno.NeedMQRetry(result.err) { + event := f.event() + event.RequestID = result.token + handler := RecordEvalChain(svc.HandleEventLock, svc.HandleCurrentSource, svc.HandleCentralReservation)(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) + require.NoError(t, handler(admittedEnforceCtx(event), event)) + } else { + require.NoError(t, result.err) + } + } + require.Equal(t, []string{"rq2"}, guard.confirmTokens) + require.Empty(t, guard.releases()) + t.Log("REAL_MYSQL_GUARD_CONCURRENT real item lock serializes rq1/rq2; old quota zero") +} + +type guardSourceTestTerminator struct { + IExptManager + finish func(context.Context, *entity.ExptCurrentItemRunSource) (bool, error) +} + +func (t *guardSourceTestTerminator) TerminateCurrentRun(ctx context.Context, _, _, _ int64, _ *entity.Session, source *entity.ExptCurrentItemRunSource, _ ...entity.CompleteExptOptionFn) (bool, error) { + return t.finish(ctx, source) +} diff --git a/backend/modules/evaluation/domain/service/expt_item_complete_source_test.go b/backend/modules/evaluation/domain/service/expt_item_complete_source_test.go new file mode 100644 index 0000000000..32ca8d4d90 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_item_complete_source_test.go @@ -0,0 +1,289 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/gorm" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + entitymocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity/mocks" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + runtimejson "github.com/coze-dev/coze-loop/backend/pkg/json" +) + +type serializedItemCompletePublisher struct { + bodies [][]byte + events []*component.ItemCompleteEvent + err error +} + +func (p *serializedItemCompletePublisher) PublishItemComplete(ctx context.Context, event *component.ItemCompleteEvent) error { + if err := ctx.Err(); err != nil { + return err + } + body, err := runtimejson.Marshal(event) + if err != nil { + return err + } + p.bodies = append(p.bodies, body) + p.events = append(p.events, event) + return p.err +} + +func TestItemCompleteSchedulerRealSQLRetryPayload(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + request := "f059e038-4d6e-41c8-a20d-68f8801adeaf" + f.token(request) + ctrl := gomock.NewController(t) + expt.TargetID = f.base + 90 + expt.Target = &entity.EvalTarget{SourceTargetID: "business-object", EvalTargetVersion: &entity.EvalTargetVersion{SandboxAgent: &entity.SandboxAgent{EnableAnalysis: true}}} + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 70, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 71}} + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil).AnyTimes() + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, EvaluationSetID: f.base + 70, ItemKey: "item-key"}}, nil).AnyTimes() + notify := &serializedItemCompletePublisher{} + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + events := eventmocks.NewMockExptEventPublisher(ctrl) + events.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + scheduler := &ExptSchedulerImpl{ResultSvc: &publicationSchedulerResult{svc}, ExptItemResultRepo: f.itemRepo(), itemCompletePublisher: notify, evaluationSetItemService: dataset, Publisher: events} + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run} + item := &entity.ExptEvalItem{ItemID: f.item, State: entity.ItemRunState_Success, RequestID: request, Publication: source, PublicationObserved: true} + before := f.snapshot() + fault := errors.New("item-complete commit fault") + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("source_payload_commit_fault", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_result_run_log" { + _ = tx.AddError(fault) + } + })) + err = scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt) + require.NoError(t, f.conn.Callback().Update().Remove("source_payload_commit_fault")) + require.ErrorIs(t, err, fault) + require.Equal(t, before, f.snapshot(), "send succeeds but the entire publication transaction rolls back") + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt)) + require.Len(t, notify.bodies, 2) + require.Equal(t, notify.bodies[0], notify.bodies[1], "a rolled-back publication must resend the original notification unchanged") + var payload map[string]json.RawMessage + require.NoError(t, json.Unmarshal(notify.bodies[0], &payload)) + require.NotContains(t, payload, "request_id", "internal retry request must not change the original notification") + require.NotContains(t, payload, "source_turn_records", "item-complete must keep its original payload") + require.JSONEq(t, strconv.Quote(strconv.FormatInt(f.run, 10)), string(payload["expt_run_id"])) + require.JSONEq(t, `"business-object"`, string(payload["source_target_id"])) + require.JSONEq(t, `true`, string(payload["enable_analysis"])) + after := f.snapshot() + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt)) + require.Len(t, notify.bodies, 2, "Resulted replay must not republish the committed projection") + require.Equal(t, after, f.snapshot()) +} + +func TestItemCompleteSchedulerFrozenRecordsAndNewRequest(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + firstRequest := "f059e038-4d6e-41c8-a20d-68f8801adeaf" + secondRequest := "79bcdd8f-b312-477c-9803-5072d82e0661" + f.token(firstRequest) + refs := &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: f.base + 61}, {VersionID: f.base + 51, Alias: "second", RecordID: f.base + 62}}, Inline: []*entity.InlineEvalResult{{InlineKey: "inline", RecordID: f.base + 63}}} + raw, err := json.Marshal(refs) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("evaluator_result_ids", raw).Error) + ctrl := gomock.NewController(t) + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 70, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 71}} + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, EvaluationSetID: f.base + 70}}, nil).AnyTimes() + first := publicationRecord(f, gptr.Of(0.0)) + first.ExperimentRunID = f.run - 1 // successful prior-stage records are reused by explicit ID + second := *first + second.ID, second.Alias = f.base+62, "second" + inline := *first + inline.ID, inline.SourceType, inline.EvaluatorVersionID, inline.Alias, inline.InlineKey = f.base+63, entity.EvaluatorRecordSourceTypeInline, 0, "", "inline" + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{f.base + 61, f.base + 62, f.base + 63}, false, false).Return([]*entity.EvaluatorRecord{first, &second, &inline}, nil).AnyTimes() + notify := &serializedItemCompletePublisher{} + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) + events := eventmocks.NewMockExptEventPublisher(ctrl) + events.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) + scheduler := &ExptSchedulerImpl{ResultSvc: &publicationSchedulerResult{svc}, ExptItemResultRepo: f.itemRepo(), itemCompletePublisher: notify, evaluationSetItemService: dataset, Publisher: events} + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run} + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + item := &entity.ExptEvalItem{ItemID: f.item, State: entity.ItemRunState_Success, RequestID: firstRequest, Publication: source, PublicationObserved: true} + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt)) + require.Len(t, notify.bodies, 1) + var payload map[string]json.RawMessage + require.NoError(t, json.Unmarshal(notify.bodies[0], &payload)) + require.NotContains(t, payload, "request_id") + require.NotContains(t, payload, "source_turn_records") + // The next admitted identity may reuse the same successful records; its UUID still differs. + f.token(secondRequest) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("result_state", int32(entity.ExptItemResultStateLogged)).Error) + before := f.snapshot() + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt)) + require.Len(t, notify.bodies, 1, "old preparation cannot send for the next request") + require.Equal(t, before, f.snapshot()) + newSource, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + newItem := &entity.ExptEvalItem{ItemID: f.item, State: entity.ItemRunState_Success, RequestID: secondRequest, Publication: newSource, PublicationObserved: true} + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{newItem}, mode, expt)) + require.Len(t, notify.bodies, 2) + var next map[string]json.RawMessage + require.NoError(t, json.Unmarshal(notify.bodies[1], &next)) + require.NotContains(t, next, "request_id") + require.NotContains(t, next, "source_turn_records") + require.Equal(t, notify.bodies[0], notify.bodies[1], "new internal requests still send the same original consumer identity") + var committed []model.ExptTurnEvaluatorResultRef + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Order("evaluator_result_id").Find(&committed).Error) + require.Len(t, committed, 3) + require.Equal(t, []int64{f.base + 61, f.base + 62, f.base + 63}, []int64{committed[0].EvaluatorResultID, committed[1].EvaluatorResultID, committed[2].EvaluatorResultID}) +} + +func TestItemCompleteSchedulerCancelAndBadScopeNoEffects(t *testing.T) { + for _, kind := range []string{"cancelled", "lost_lease", "wrong_run", "wrong_space", "wrong_experiment", "wrong_item", "wrong_request", "source_changed", "send_failure"} { + t.Run(kind, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + ctrl := gomock.NewController(t) + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 70, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 71}} + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, EvaluationSetID: f.base + 70}}, nil).AnyTimes() + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, nil)}, nil).AnyTimes() + notify := &serializedItemCompletePublisher{} + scheduler := &ExptSchedulerImpl{ResultSvc: &publicationSchedulerResult{svc}, ExptItemResultRepo: f.itemRepo(), itemCompletePublisher: notify, evaluationSetItemService: dataset} + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + item := &entity.ExptEvalItem{ItemID: f.item, State: entity.ItemRunState_Success, RequestID: "rq1", Publication: source, PublicationObserved: true} + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run} + ctx := context.Background() + switch kind { + case "cancelled": + child, cancel := context.WithCancel(ctx) + cancel() + ctx = child + case "lost_lease": + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ string, _ time.Duration, _ time.Duration) (bool, context.Context, func(), error) { + child, cancel := context.WithCancel(ctx) + cancel() + return true, child, cancel, nil + }) + locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + svc.resultPublishLocker = func() lock.ILocker { return locker } + case "wrong_run": + event.ExptRunID++ + case "wrong_space": + event.SpaceID++ + case "wrong_experiment": + event.ExptID++ + case "wrong_item": + item.ItemID++ + case "wrong_request": + item.RequestID = "another-request" + case "source_changed": + f.token("79bcdd8f-b312-477c-9803-5072d82e0661") + default: + notify.err = errors.New("MQ unavailable") + } + before := f.snapshot() + err = scheduler.recordEvalItemRunLogs(ctx, event, []*entity.ExptEvalItem{item}, entitymocks.NewMockExptSchedulerMode(ctrl), expt) + if kind == "source_changed" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + if kind == "send_failure" { + require.Len(t, notify.bodies, 1) + } else { + require.Empty(t, notify.bodies) + } + require.Equal(t, before, f.snapshot(), "failed, cancelled or stale publication must leave all eleven tables unchanged") + }) + } +} + +func TestItemCompleteSchedulerMultipleItemsRawAndManual(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + request := "f059e038-4d6e-41c8-a20d-68f8801adeaf" + f.token(request) + require.NoError(t, f.conn.Create(&model.ExptTurnResult{ID: f.base + 410, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, TurnID: 0, Status: int32(entity.TurnRunState_Processing)}).Error) + require.NoError(t, f.conn.Create(&model.ExptTurnResultRunLog{ID: f.base + 510, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, TurnID: 0, Status: int32(entity.TurnRunState_Success)}).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item+1).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": int32(entity.ExptItemResultStateLogged)}).Error) + ctrl := gomock.NewController(t) + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 70, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 71}} + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, EvaluationSetID: f.base + 70}, {ItemID: f.item + 1, EvaluationSetID: f.base + 70}}, nil).Times(1) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{f.base + 61}, false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, nil)}, nil).Times(1) + notify := &serializedItemCompletePublisher{} + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) + events := eventmocks.NewMockExptEventPublisher(ctrl) + events.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + scheduler := &ExptSchedulerImpl{ResultSvc: &publicationSchedulerResult{svc}, ExptItemResultRepo: f.itemRepo(), itemCompletePublisher: notify, evaluationSetItemService: dataset, Publisher: events} + items := []*entity.ExptEvalItem{} + for _, id := range []int64{f.item, f.item + 1} { + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, id) + require.NoError(t, err) + items = append(items, &entity.ExptEvalItem{ItemID: id, State: entity.ItemRunState_Success, RequestID: source.Source.Current.RequestID, Publication: source, PublicationObserved: true}) + } + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run}, items, mode, expt)) + require.Len(t, notify.bodies, 2) + for _, body := range notify.bodies { + var payload map[string]json.RawMessage + require.NoError(t, json.Unmarshal(body, &payload)) + require.NotContains(t, payload, "request_id") + require.NotContains(t, payload, "source_turn_records") + } + require.Equal(t, strconv.FormatInt(f.item, 10), notify.events[0].ItemID) + require.Equal(t, strconv.FormatInt(f.item+1, 10), notify.events[1].ItemID) + notify.events[1].ItemKey = "changed-item" + first, err := runtimejson.Marshal(notify.events[0]) + require.NoError(t, err) + require.Equal(t, notify.bodies[0], first, "subsequent items cannot mutate an earlier notification") + stats, err := svc.ExptStatsRepo.Get(context.Background(), f.expt, f.space) + require.NoError(t, err) + require.Equal(t, int32(2), stats.SuccessItemCnt) + require.Zero(t, stats.ProcessingItemCnt) + require.Equal(t, int32(1), stats.PendingItemCnt) +} + +// The existing sender may deliver the same notification repeatedly. Its consumer +// identity remains independent of internal manual retry and result source data. +func TestItemCompleteSchedulerDuplicateSendOriginalPayload(t *testing.T) { + const expected = `{"eval_target_workspace_id":"","eval_target_id":"","source_target_id":"","expt_workspace_id":"7384603622000000001","expt_id":"7384603622000000002","expt_run_id":"7384603622000000003","dataset_workspace_id":"7384603622000000001","dataset_id":"7384603622000000005","dataset_version_id":"7384603622000000006","dataset_key":"","dataset_version_name":"","experiment_group_key":"","item_id":"7384603622000000004","item_key":"case","enable_analysis":false,"created_by":""}` + for _, request := range []string{"", "manual-retry-request"} { + t.Run("request="+request, func(t *testing.T) { + notify := &serializedItemCompletePublisher{} + scheduler := &ExptSchedulerImpl{itemCompletePublisher: notify} + event := &entity.ExptScheduleEvent{SpaceID: 7384603622000000001, ExptID: 7384603622000000002, ExptRunID: 7384603622000000003} + item := &entity.ExptEvalItem{ + ItemID: 7384603622000000004, State: entity.ItemRunState_Success, RequestID: request, + Publication: &entity.ExptItemResultPublishSource{}, PublicationObserved: true, + } + meta := &entity.EvaluationSetItem{SpaceID: event.SpaceID, EvaluationSetID: 7384603622000000005, ItemID: item.ItemID, ItemKey: "case"} + for delivery := 0; delivery < 2; delivery++ { + require.NoError(t, scheduler.sendItemComplete(context.Background(), event, nil, item, meta, 7384603622000000006)) + } + require.Len(t, notify.bodies, 2) + require.Equal(t, expected, string(notify.bodies[0])) + require.Equal(t, notify.bodies[0], notify.bodies[1]) + worker := buildItemCompleteEvent(&entity.ExptItemEvalCtx{Event: &entity.ExptItemEvalEvent{SpaceID: event.SpaceID, ExptID: event.ExptID, ExptRunID: event.ExptRunID, EvalSetItemID: item.ItemID, RequestID: request}, EvalSetItem: meta, EvalSetVersionID: 7384603622000000006}) + body, err := runtimejson.Marshal(worker) + require.NoError(t, err) + require.Equal(t, notify.bodies[0], body, "worker and scheduler retain the same original payload") + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_item_terminate_test.go b/backend/modules/evaluation/domain/service/expt_item_terminate_test.go index b335c93a52..8e3a2409f1 100644 --- a/backend/modules/evaluation/domain/service/expt_item_terminate_test.go +++ b/backend/modules/evaluation/domain/service/expt_item_terminate_test.go @@ -703,15 +703,15 @@ func TestCompleteItemRunOnUnretriableErr_TerminalNotOverwritten(t *testing.T) { // 该行已 Terminal → 返回 nil 且 **不进入** next（对应「排队中的行被终止后不再被调度」）。 // 闸门优先级：实验级终态判定 > item 级 Terminal 判定 > 正常执行。 func TestHandleEventCheck_ItemLevelTerminalGate(t *testing.T) { - event := &entity.ExptItemEvalEvent{SpaceID: 7, ExptID: 1001, ExptRunID: 2002, EvalSetItemID: 3003} + event := &entity.ExptItemEvalEvent{SpaceID: 7, ExptID: 1001, ExptRunID: 2002, EvalSetItemID: 3003, ExptRunMode: entity.EvaluationModeSubmit} newSvc := func(ctrl *gomock.Controller, itemState entity.ItemRunState) (*ExptItemEventEvalServiceImpl, *bool) { manager := svcMocks.NewMockIExptManager(ctrl) manager.EXPECT().GetRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(&entity.ExptRunLog{Status: int64(entity.ExptStatus_Processing)}, nil) itemRepo := repoMocks.NewMockIExptItemResultRepo(ctrl) - itemRepo.EXPECT().GetItemRunLog(gomock.Any(), int64(1001), int64(2002), int64(3003), int64(7)). - Return(&entity.ExptItemResultRunLog{Status: int32(itemState)}, nil) + itemRepo.EXPECT().GetCurrentRequest(gomock.Any(), int64(7), int64(1001), int64(2002), int64(3003)). + Return(&entity.ExptItemCurrentRequest{RunID: 2002, Status: itemState, Mode: entity.EvaluationModeSubmit}, nil) reached := false return &ExptItemEventEvalServiceImpl{manager: manager, exptItemResultRepo: itemRepo}, &reached } @@ -721,7 +721,7 @@ func TestHandleEventCheck_ItemLevelTerminalGate(t *testing.T) { defer ctrl.Finish() svc, reached := newSvc(ctrl, entity.ItemRunState_Terminal) - err := svc.HandleEventCheck(func(context.Context, *entity.ExptItemEvalEvent) error { + err := RecordEvalChain(svc.HandleEventCheck, svc.HandleCurrentSource)(func(context.Context, *entity.ExptItemEvalEvent) error { *reached = true return nil })(context.Background(), event) @@ -735,7 +735,7 @@ func TestHandleEventCheck_ItemLevelTerminalGate(t *testing.T) { defer ctrl.Finish() svc, reached := newSvc(ctrl, entity.ItemRunState_Processing) - err := svc.HandleEventCheck(func(context.Context, *entity.ExptItemEvalEvent) error { + err := RecordEvalChain(svc.HandleEventCheck, svc.HandleCurrentSource)(func(context.Context, *entity.ExptItemEvalEvent) error { *reached = true return nil })(context.Background(), event) diff --git a/backend/modules/evaluation/domain/service/expt_late_write_compatibility_test.go b/backend/modules/evaluation/domain/service/expt_late_write_compatibility_test.go new file mode 100644 index 0000000000..eecde2385b --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_late_write_compatibility_test.go @@ -0,0 +1,536 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/pkg/ctxcache" + "gorm.io/gorm" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + metricmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + evaluatorrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/evaluator" + evaluatormysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/evaluator/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + targetmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/target/mysql" + targetconvert "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/target/mysql/convertor" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +// Keep the real producer's record fields and the real SQL DAO, mocking only unrelated target configuration. +type compatibilityTargetRepo struct { + repo.IEvalTargetRepo + dao targetmysql.EvalTargetRecordDAO +} + +func (r *compatibilityTargetRepo) CreateEvalTargetRecord(ctx context.Context, record *entity.EvalTargetRecord, _ *bool) (int64, error) { + po, err := targetconvert.EvalTargetRecordDO2PO(record) + if err != nil { + return 0, err + } + return r.dao.Create(ctx, po) +} + +func (r *compatibilityTargetRepo) GetEvalTargetRecordByIDAndSpaceID(ctx context.Context, space, id int64) (*entity.EvalTargetRecord, error) { + po, err := r.dao.GetByIDAndSpaceID(ctx, id, space) + if err != nil { + return nil, err + } + return targetconvert.EvalTargetRecordPO2DO(po) +} + +func TestLateCompatProductionRecordVersionZero(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + ctrl := gomock.NewController(t) + ctx := context.Background() + // Use the real async producer; its record has no ItemVersionID assignment. + dao := targetmysql.NewEvalTargetRecordDAO(f.provider) + operator := svcmocks.NewMockISourceEvalTargetOperateService(ctrl) + metric := metricmocks.NewMockEvalTargetMetrics(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + operator.EXPECT().ValidateInput(gomock.Any(), f.space, gomock.Any(), gomock.Any()).Return(nil) + operator.EXPECT().AsyncExecute(gomock.Any(), f.space, gomock.Any()).Return(f.base+901, "callee", map[string]string{}, nil) + metric.EXPECT().EmitRun(f.space, gomock.Any(), gomock.Any()) + config.EXPECT().BuildEvalExt(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + baseRepo := repomocks.NewMockIEvalTargetRepo(ctrl) + baseRepo.EXPECT().GetEvalTargetVersion(gomock.Any(), f.space, f.base+43).Return(&entity.EvalTarget{ID: f.base + 42, SpaceID: f.space, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, EvalTargetVersion: &entity.EvalTargetVersion{ID: f.base + 43}}, nil).AnyTimes() + targetSvc := &EvalTargetServiceImpl{evalTargetRepo: &compatibilityTargetRepo{IEvalTargetRepo: baseRepo, dao: dao}, typedOperators: map[entity.EvalTargetType]ISourceEvalTargetOperateService{entity.EvalTargetTypeCustomRPCServer: operator}, metric: metric, configer: config} + target := &entity.EvalTarget{ID: f.base + 42, SpaceID: f.space, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, EvalTargetVersion: &entity.EvalTargetVersion{ID: f.base + 43}} + produced, _, err := targetSvc.asyncExecuteTarget(ctx, f.space, target, &entity.ExecuteTargetCtx{ExperimentID: &f.expt, ExperimentRunID: &f.run, ItemID: f.item, TurnID: 0}, &entity.EvalTargetInputData{}) + require.NoError(t, err) + require.Zero(t, produced.ItemVersionID) + stored, err := targetSvc.GetRecordByID(ctx, f.space, produced.ID) + require.NoError(t, err) + require.Zero(t, stored.ItemVersionID) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=0", f.space).Update("target_result_id", produced.ID).Error) + event := f.event() + event.AsyncReportTrigger = true + event.ResumeSource = &entity.ExptResumeSource{Kind: "target", TurnID: 0, RecordID: produced.ID, InvokeID: strconv.FormatInt(produced.ID, 10)} + consumer := &ExptItemEventEvalServiceImpl{exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), evaTargetService: targetSvc} + called := false + require.NoError(t, consumer.HandleCurrentSource(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(ctx, event)) + assert.True(t, called, "real producer V0 must resume versioned current item") + // RunEvaluator also produces V0, via real record repository/DAO. + records := evaluatorrepo.NewEvaluatorRecordRepo(nil, f.provider, evaluatormysql.NewEvaluatorRecordDAO(f.provider), nil) + versions := repomocks.NewMockIEvaluatorRepo(ctrl) + limiter := repomocks.NewMockRateLimiter(ctrl) + execution := svcmocks.NewMockEvaluatorSourceService(ctrl) + ids := idmocks.NewMockIIDGenerator(ctrl) + version := &entity.Evaluator{ID: f.base + 51, SpaceID: f.space, EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: f.base + 51}} + versions.EXPECT().BatchGetEvaluatorByVersionID(gomock.Any(), gomock.Any(), []int64{f.base + 51}, false, false).Return([]*entity.Evaluator{version}, nil) + limiter.EXPECT().AllowInvoke(gomock.Any(), f.space).Return(true) + execution.EXPECT().PreHandle(gomock.Any(), version).Return(nil) + execution.EXPECT().Run(gomock.Any(), version, gomock.Any(), gomock.Any(), f.space, gomock.Any()).Return(&entity.EvaluatorOutputData{}, entity.EvaluatorRunStatusSuccess, "trace") + ids.EXPECT().GenID(gomock.Any()).Return(f.base+902, nil) + plain := repomocks.NewMockIPlainRateLimiter(ctrl) + plain.EXPECT().AllowInvokeWithKeyLimit(gomock.Any(), gomock.Any(), gomock.Any()).Return(true).AnyTimes() + evalSvc := &EvaluatorServiceImpl{plainRateLimiter: plain, evaluatorRepo: versions, evaluatorRecordRepo: records, idgen: ids, limiter: limiter, evaluatorSourceServices: map[entity.EvaluatorType]EvaluatorSourceService{entity.EvaluatorTypePrompt: execution}} + record, err := evalSvc.RunEvaluator(ctx, &entity.RunEvaluatorRequest{SpaceID: f.space, ExperimentID: f.expt, ExperimentRunID: f.run, ItemID: f.item, TurnID: 0, EvaluatorVersionID: f.base + 51, Alias: "alias"}) + require.NoError(t, err) + require.Zero(t, record.ItemVersionID) + loaded, err := records.GetEvaluatorRecord(ctx, record.ID, false) + require.NoError(t, err) + require.Zero(t, loaded.ItemVersionID) + refs, _ := json.Marshal(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: record.ID}}}) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=0", f.space).Update("evaluator_result_ids", refs).Error) + consumer.evaluatorRecordService = &EvaluatorRecordServiceImpl{evaluatorRecordRepo: records} + event.AsyncReportTrigger = false + event.AsyncEvaluatorReportTrigger = true + event.ResumeSource = &entity.ExptResumeSource{Kind: "evaluator", TurnID: 0, RecordID: record.ID, InvokeID: strconv.FormatInt(record.ID, 10)} + called = false + require.NoError(t, consumer.HandleCurrentSource(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(ctx, event)) + require.True(t, called, "real evaluator producer V0 must resume versioned item") +} + +func TestLateCompatRawRetryPreEvalVersion(t *testing.T) { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + f := newLateWriteFixture(t, "") + require.NoError(t, f.conn.Unscoped().Where("space_id=?", f.space).Delete(&model.ExptTurnResultRunLog{}).Error) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("mode", int32(mode)).Error) + ctrl := gomock.NewController(t) + gen := idmocks.NewMockIIDGenerator(ctrl) + gen.EXPECT().GenMultiIDs(gomock.Any(), 2).Return([]int64{f.base + 910, f.base + 911}, nil) + event := f.event() + event.RequestID = "" + event.ExptRunMode = mode + execCtx := &entity.ExptItemEvalCtx{Event: event, EvalSetItem: &entity.EvaluationSetItem{ItemID: f.item, ItemVersionID: &f.version, Turns: []*entity.Turn{{ID: 0}, {ID: f.base + 22}}}, ExistItemEvalResult: &entity.ExptItemEvalResult{ItemResultRunLog: &entity.ExptItemResultRunLog{ItemVersionID: f.version}, TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{}}} + require.NoError(t, (&ExptRecordEvalModeRetryIgnoreResult{exptTurnResultRepo: f.turnRepo(), idgen: gen}).PreEval(context.Background(), execCtx)) + source, err := f.port.GetCurrentItemRunSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + require.Equal(t, f.version, source.Turns[0].ItemVersionID) + result := *source.Turns[0] + result.Status = entity.TurnRunState_Success + ok, err := f.port.StoreCurrentTurnRunResult(context.Background(), source, &result) + require.NoError(t, err) + require.True(t, ok) + }) + } +} + +func TestLateCompatInlineZeroAndVersionFence(t *testing.T) { + for _, kind := range []string{"inline0", "builtin0", "version0", "version_mismatch", "negative_target", "target_mismatch", "inlinekey_mismatch", "record_mismatch", "read_error"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, ExptRunMode: entity.EvaluationModeSubmit, AsyncEvaluatorReportTrigger: true, ResumeSource: &entity.ExptResumeSource{Kind: "evaluator", TurnID: 0, RecordID: 61, InvokeID: "61"}} + current := &entity.ExptItemCurrentRequest{SpaceID: 3, ExptID: 1, RunID: 2, ItemID: 11, ItemVersionID: 31, Mode: entity.EvaluationModeSubmit, Status: entity.ItemRunState_Processing} + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(11)).Return(current, nil) + refs := &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "inline", RecordID: 61}}} + record := &entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 11, ItemVersionID: 31, TurnID: 0, SourceType: entity.EvaluatorRecordSourceTypeInline, InlineKey: "inline", Status: entity.EvaluatorRunStatusSuccess} + var readErr error + switch kind { + case "builtin0": + record.SourceType = entity.EvaluatorRecordSourceTypeBuiltin + record.EvaluatorVersionID = 51 + refs = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, RecordID: 61}}} + case "version0": + record.ItemVersionID = 0 + case "version_mismatch": + record.ItemVersionID = 32 + case "negative_target": + record.TargetRecordID = -1 + case "target_mismatch": + record.TargetRecordID = 42 + case "inlinekey_mismatch": + record.InlineKey = "old" + case "record_mismatch": + refs.Inline[0].RecordID = 62 + case "read_error": + readErr = errors.New("record storage read error") + } + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), int64(11), int64(3)).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 11, ItemVersionID: 31, TurnID: 0, TargetResultID: 41, Status: entity.TurnRunState_Processing, EvaluatorResultIds: refs}}, nil) + records.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(61), false).Return(record, readErr) + svc := &ExptItemEventEvalServiceImpl{exptItemResultRepo: items, exptTurnResultRepo: turns, evaluatorRecordService: records} + called := false + err := svc.HandleCurrentSource(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(context.Background(), event) + require.Equal(t, kind == "inline0" || kind == "builtin0" || kind == "version0", called) + if kind == "read_error" { + require.True(t, errno.NeedMQRetry(err)) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestLateCompatFailRetryDependencyBeforePort(t *testing.T) { + f := newLateWriteFixture(t, "") + source := f.source() + ctrl := gomock.NewController(t) + resultSvc := svcmocks.NewMockExptResultService(ctrl) + fault := errors.New("frozen successful-stage read unavailable") + resultSvc.EXPECT().GetExptItemTurnResults(gomock.Any(), f.expt, f.item, f.space, gomock.Any()).Return(nil, fault) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetErrRetryConf(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.RetryConf{}).AnyTimes() + svc := &ExptItemEventEvalServiceImpl{exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), configer: config, itemLocker: lateWriteLocker(t, ctrl)} + event := f.event() + event.RequestID = "" + event.MaxRetryTimes = 0 + before := f.snapshot() + err := svc.HandleEventErr(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + executionSource(ctx).Source = source + eiec := &entity.ExptItemEvalCtx{Event: event, ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{}}} + return (&ExptRecordEvalModeFailRetry{resultSvc: resultSvc}).PreEval(ctx, eiec) + })(context.Background(), event) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, errRetryStartDependencyFailure) + require.ErrorIs(t, err, fault) + require.Equal(t, before, f.snapshot()) +} + +func TestLateCompatDebtFinishedAndPreExec(t *testing.T) { + for _, kind := range []string{"completed_debt", "preexec_debt", "rq2", "cancelled", "completed_late"} { + t.Run(kind, func(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + source := f.source() + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + manager := svcmocks.NewMockIExptManager(ctrl) + guard := componentmocks.NewMockICentralReservationGuard(ctrl) + config.EXPECT().GetErrRetryConf(gomock.Any(), f.space, gomock.Any()).Return(&entity.RetryConf{RetryTimes: 3, IsInDebt: true}).AnyTimes() + event := f.event() + event.RetryTimes = 3 + ctx := ctxcache.Init(context.Background()) + event.WithCtxCentralQuotaHeld(ctx, "frozen-scope") + svc := &ExptItemEventEvalServiceImpl{experimentRepo: f.experimentRepo(), exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), configer: config, manager: manager, centralGuard: guard, itemLocker: lateWriteLocker(t, ctrl)} + expected := kind == "completed_debt" || kind == "preexec_debt" + finishes, releases := 0, 0 + + svc.manager = &guardSourceTestTerminator{IExptManager: manager, finish: func(ctx context.Context, original *entity.ExptCurrentItemRunSource) (bool, error) { + require.True(t, sameExecutionIdentity(source, original)) + finishes++ + err := guard.Release(ctx, "frozen-scope", f.run, f.item, original.Current.RequestID, "debt") + return true, err + }} + + guard.EXPECT().Release(gomock.Any(), "frozen-scope", f.run, f.item, "rq1", gomock.Any()).DoAndReturn(func(ctx context.Context, _ string, _, _ int64, _, _ string) error { + require.NoError(t, ctx.Err()) + releases++ + return nil + }).Times(map[bool]int{true: 1, false: 0}[expected]) + if kind == "rq2" { + f.token("rq2") + } + if kind == "completed_late" { + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Fail), "result_state": int32(entity.ExptItemResultStateLogged)}).Error) + } + before := f.snapshot() + if kind == "cancelled" { + child, cancel := context.WithCancel(ctx) + cancel() + ctx = child + } + err := svc.HandleEventErr(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + executionSource(ctx).Source = source + debt := errors.New("space in debt") + if kind == "completed_debt" { + executor := &ExptItemEvalCtxExecutor{ItemResultRepo: f.itemRepo(), Configer: config} + return executor.CompleteItemRun(ctx, &entity.ExptItemEvalCtx{Event: event}, debt) + } + return debt + })(ctx, event) + if kind == "cancelled" { + require.True(t, errno.NeedMQRetry(err)) + } else { + require.NoError(t, err) + } + require.Equal(t, map[bool]int{true: 1, false: 0}[expected], finishes) + require.Equal(t, map[bool]int{true: 1, false: 0}[expected], releases) + if kind != "completed_debt" { + require.Equal(t, before, f.snapshot()) + } + }) + } +} + +func TestLateCompatSweepMissingRawLogs(t *testing.T) { + for _, kind := range []string{"zombie_all", "zombie_partial", "sandbox_all", "sandbox_partial", "idgen_error", "insert_error", "second_insert_error", "idgen_cancel", "version0", "manual_missing"} { + t.Run(kind, func(t *testing.T) { + f := newLateWriteFixture(t, "") + ctrl := gomock.NewController(t) + partial := kind == "zombie_partial" || kind == "sandbox_partial" + if partial { + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND turn_id=?", f.space, f.base+22).Delete(&model.ExptTurnResultRunLog{}).Error) + } else { + require.NoError(t, f.conn.Unscoped().Where("space_id=?", f.space).Delete(&model.ExptTurnResultRunLog{}).Error) + } + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("evaluator_result_ids", []byte(`null`)).Error) + if kind == "version0" { + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_item_ref"} { + require.NoError(t, f.conn.Table(table).Where("space_id=?", f.space).Update("item_version_id", 0).Error) + } + f.version = 0 + } + if kind == "manual_missing" { + f.token("rq1") + } + source, sourceErr := f.port.GetCurrentItemRunSource(context.Background(), f.space, f.expt, f.run, f.item) + if kind == "manual_missing" { + require.Error(t, sourceErr) + return + } + require.NoError(t, sourceErr) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetConsumerConf(gomock.Any()).Return(&entity.ExptConsumerConf{ExptExecConf: &entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ZombieSecond: 1}}}).AnyTimes() + ids := idmocks.NewMockIIDGenerator(ctrl) + fault := errors.New("prepared raw ID unavailable") + count := 2 + if partial { + count = 1 + } + calls := 0 + taskCtx, cancelTask := context.WithCancel(context.Background()) + defer cancelTask() + ids.EXPECT().GenMultiIDs(gomock.Any(), count).DoAndReturn(func(context.Context, int) ([]int64, error) { + calls++ + if kind == "idgen_cancel" { + cancelTask() + } + if kind == "idgen_error" { + return nil, fault + } + out := []int64{f.base + 920, f.base + 921} + return out[:count], nil + }).AnyTimes() + target := svcmocks.NewMockIEvalTargetService(ctrl) + records := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return(nil, nil).AnyTimes() + target.EXPECT().GetRecordByID(gomock.Any(), f.space, f.base+41).Return(&entity.EvalTargetRecord{ID: f.base + 41, SpaceID: f.space, ExperimentRunID: f.run, ItemID: f.item, TurnID: 0, Status: gptr.Of(entity.EvalTargetRunStatusFail)}, nil).AnyTimes() + target.EXPECT().TerminateAsyncRecordsAndDestroySandbox(gomock.Any(), f.space, []int64{f.base + 41}, gomock.Any(), gomock.Any(), gomock.Any()).Times(map[bool]int{true: 0, false: 1}[kind == "idgen_error" || kind == "idgen_cancel"]) + if kind == "sandbox_all" || kind == "sandbox_partial" { + target.EXPECT().CheckSandboxTerminated(gomock.Any(), f.space, []int64{f.base + 41}).Return([]int64{f.base + 41}, map[int64]string{f.base + 41: "Failed"}) + } + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: f.itemRepo(), ExptTurnResultRepo: f.turnRepo(), IDGen: ids, itemLocker: lateWriteLocker(t, ctrl), Configer: config, evalTargetService: target, EvaluatorRecordRepo: records} + old := time.Now().Add(-time.Hour) + source.Current.UpdatedAt = &old + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("updated_at", old).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=? AND turn_id<>0", f.space).Update("target_result_id", 0).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id<>0", f.space).Update("target_result_id", 0).Error) + source = f.source() + before := f.snapshot() + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run} + item := &entity.ExptEvalItem{Source: source, ItemID: f.item, ExptID: f.expt, State: entity.ItemRunState_Processing, UpdatedAt: source.Current.UpdatedAt} + inserted := 0 + if kind == "insert_error" || kind == "second_insert_error" { + require.NoError(t, f.conn.Callback().Create().Before("gorm:create").Register("compat_missing_insert_fault", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_turn_result_run_log" { + inserted++ + if kind == "insert_error" || inserted == 2 { + _ = tx.AddError(fault) + } + } + })) + } + var completed []*entity.ExptEvalItem + var err error + if kind == "sandbox_all" || kind == "sandbox_partial" { + _, completed, err = scheduler.sweepCurrentSandboxItems(taskCtx, event, []*entity.ExptEvalItem{item}, &entity.Experiment{}) + } else { + _, completed, err = scheduler.handleZombies(taskCtx, event, []*entity.ExptEvalItem{item}, nil) + } + if kind == "insert_error" || kind == "second_insert_error" { + require.NoError(t, f.conn.Callback().Create().Remove("compat_missing_insert_fault")) + require.Positive(t, inserted) + } + if kind == "idgen_error" || kind == "idgen_cancel" || kind == "insert_error" || kind == "second_insert_error" { + require.True(t, errno.NeedMQRetry(err)) + if kind == "idgen_cancel" { + require.ErrorIs(t, err, context.Canceled) + } else { + require.ErrorIs(t, err, fault) + } + require.Equal(t, before, f.snapshot()) + require.Empty(t, completed) + return + } + require.NoError(t, err) + require.Equal(t, 1, calls) + require.Len(t, completed, 1) + var logs []model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=?", f.space).Find(&logs).Error) + require.Len(t, logs, 2) + for _, log := range logs { + require.Equal(t, f.version, log.ItemVersionID) + require.Equal(t, int32(entity.TurnRunState_Fail), log.Status) + require.NotEmpty(t, gptr.Indirect(log.ErrMsg)) + } + stats, err := experimentrepo.NewExptStatsRepo(experimentmysql.NewExptStatsDAO(f.provider)).Get(context.Background(), f.expt, f.space) + require.NoError(t, err) + require.Equal(t, int32(1), stats.FailItemCnt) + require.Equal(t, int32(1), stats.ProcessingItemCnt) + archive := &ExptResultServiceImpl{ExptItemResultRepo: f.itemRepo(), ExptTurnResultRepo: f.turnRepo(), ExptStatsRepo: experimentrepo.NewExptStatsRepo(experimentmysql.NewExptStatsDAO(f.provider)), resultPublishRepo: experimentrepo.NewExptResultPublishRepo(f.provider), resultPublishLocker: lateWriteLocker(t, ctrl), evalTargetService: target} + expt := &entity.Experiment{ID: f.expt, SpaceID: f.space, EvalConf: &entity.EvaluationConfiguration{}} + _, err = archive.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, err) + after, err := archive.ExptStatsRepo.Get(context.Background(), f.expt, f.space) + require.NoError(t, err) + require.Equal(t, *stats, *after) + turnResults, err := f.itemRepo().GetItemTurnResults(context.Background(), f.space, f.expt, f.item) + require.NoError(t, err) + builder := &ExptResultBuilder{turnResultDO: turnResults, ItemIDTurnID2TurnResultID: map[int64]map[int64]int64{f.item: {}}} + for _, turn := range turnResults { + builder.ItemIDTurnID2TurnResultID[f.item][turn.TurnID] = turn.ID + info := builder.getTurnSystemInfo(context.Background(), f.item, turn.TurnID) + require.NotNil(t, info.Error) + if kind == "sandbox_all" || kind == "sandbox_partial" { + require.Contains(t, gptr.Indirect(info.Error.Detail), "Failed") + } else { + require.Contains(t, gptr.Indirect(info.Error.Detail), "1") + } + } + _, err = archive.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, err) + final, err := archive.ExptStatsRepo.Get(context.Background(), f.expt, f.space) + require.NoError(t, err) + require.Equal(t, *after, *final) + }) + } +} + +func TestLateCompatTurnFailureReasonReplacesOld(t *testing.T) { + for _, terminal := range []bool{false, true} { + t.Run(fmt.Sprint(terminal), func(t *testing.T) { + f := newLateWriteFixture(t, "") + old := []byte(errno.SerializeErr(errno.NewTurnOtherErr("old reason", errors.New("old error")))) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("err_msg", old).Error) + state := entity.ItemRunState_Fail + if terminal { + state = entity.ItemRunState_Terminal + } + source := f.source() + reason := []byte(errno.SerializeErr(errno.NewItemZombieTimeoutErr(1, false))) + ok, err := f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Status: state, ErrMsg: reason, FailPendingTurns: true}) + require.NoError(t, err) + require.True(t, ok) + logs, err := f.turnRepo().GetItemTurnRunLogs(context.Background(), f.expt, f.run, f.item, f.space) + require.NoError(t, err) + for _, log := range logs { + require.NotEqual(t, string(old), log.ErrMsg, "pending turn must show this completion reason") + parsed := errno.DeserializeErr([]byte(log.ErrMsg)) + if terminal { + ok, _ := errno.ParseItemManuallyTerminatedErr(parsed) + require.True(t, ok) + } else { + ok, msg := errno.ParseTurnOtherErr(parsed) + require.True(t, ok) + require.Contains(t, msg, "1") + } + } + }) + } +} + +func TestLateCompatDebtQuotaReleasedOnlyOnce(t *testing.T) { + ctrl := gomock.NewController(t) + guard := componentmocks.NewMockICentralReservationGuard(ctrl) + dispatch := repomocks.NewMockIExptItemDispatchRepo(ctrl) + ctx := ctxcache.Init(context.Background()) + execution := ¤tItemExecution{} + ctx = context.WithValue(ctx, currentItemSourceKey{}, execution) + event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 11} + event.WithCtxCentralQuotaHeld(ctx, "scope") + dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), int64(3), int64(1), int64(2), []int64{11}).Return([]*repo.ExptDispatchObservation{{ItemID: 11, Status: int32(entity.ItemRunState_Fail)}}, nil) + guard.EXPECT().Release(gomock.Any(), "scope", int64(2), int64(11), "", gomock.Any()).Return(nil).Times(1) + svc := &ExptItemEventEvalServiceImpl{centralGuard: guard, dispatchRepo: dispatch} + svc.releaseQuotaIfItemTerminal(ctx, event, "scope", guard, errors.New("debt")) + svc.releaseCentralQuotaOutsideGate(ctx, event, errors.New("debt"), "expt terminated on indebt") +} + +func TestLateCompatCompletionPreservesSuccessfulTurn(t *testing.T) { + for _, kind := range []string{"fail", "terminal", "yield", "rollback"} { + t.Run(kind, func(t *testing.T) { + f := newLateWriteFixture(t, "") + successErr := []byte("successful stage metadata") + oldPending := []byte(errno.SerializeErr(errno.NewTurnOtherErr("old pending", errors.New("previous error")))) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=0", f.space).Updates(map[string]any{"status": int32(entity.TurnRunState_Success), "err_msg": successErr}).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=? AND turn_id=0", f.space).Update("status", int32(entity.TurnRunState_Success)).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=?", f.space, f.base+22).Update("err_msg", oldPending).Error) + source := f.source() + before := f.snapshot() + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, FailPendingTurns: true, ErrMsg: []byte(errno.SerializeErr(errno.NewSandboxTerminatedBeforeReportErr("Failed"))), RetryTimes: 2} + if kind == "terminal" { + write.Status = entity.ItemRunState_Terminal + } + if kind == "yield" { + write.Yield = true + } + fault := errors.New("turn metadata transaction fault") + if kind == "rollback" { + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("compat_turn_metadata_fault", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_stats" { + _ = tx.AddError(fault) + } + })) + } + ok, err := f.port.CompleteCurrentItemRun(context.Background(), source, write) + if kind == "rollback" { + require.NoError(t, f.conn.Callback().Update().Remove("compat_turn_metadata_fault")) + require.ErrorIs(t, err, fault) + require.False(t, ok) + require.Equal(t, before, f.snapshot()) + return + } + require.NoError(t, err) + require.True(t, ok) + logs, err := f.turnRepo().GetItemTurnRunLogs(context.Background(), f.expt, f.run, f.item, f.space) + require.NoError(t, err) + for _, log := range logs { + if log.TurnID == 0 { + require.Equal(t, source.Turns[0], log) + } else if kind == "yield" { + require.Equal(t, string(oldPending), log.ErrMsg) + } else { + require.NotEqual(t, string(oldPending), log.ErrMsg) + } + } + after := f.snapshot() + require.Equal(t, before["expt_turn_evaluator_result_ref"], after["expt_turn_evaluator_result_ref"]) + var canonical model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=? AND turn_id=0", f.space).Take(&canonical).Error) + require.Equal(t, 0.0, gptr.Indirect(canonical.WeightedScore)) + require.Equal(t, f.base+41, canonical.TargetResultID) + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_manage.go b/backend/modules/evaluation/domain/service/expt_manage.go index 619c7b5191..815443b8e1 100644 --- a/backend/modules/evaluation/domain/service/expt_manage.go +++ b/backend/modules/evaluation/domain/service/expt_manage.go @@ -51,6 +51,7 @@ type IExptConfigManager interface { // IExptExecutionManager 实验执行控制接口（负责实验的运行、监控和状态管理） type IExptExecutionManager interface { + RetryInCurrentRun(context.Context, *entity.RetryInCurrentRunRequest) (*entity.ManualRetryReceipt, error) CheckRun(ctx context.Context, expt *entity.Experiment, spaceID int64, session *entity.Session, opts ...entity.ExptRunCheckOptionFn) error Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go index a99201cd5a..21166b4659 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go @@ -20,6 +20,8 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/encoding" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" @@ -320,6 +322,16 @@ func (e *ExptMangerImpl) prepareRun(ctx context.Context, exptID, runID, spaceID } func (e *ExptMangerImpl) cleanupUnscheduledRun(ctx context.Context, exptID, runID, spaceID int64, cause error) { + if e.runFinishRepo != nil { + if err := e.cleanupCurrentUnscheduledRun(ctx, exptID, runID, spaceID, cause); err != nil { + logs.CtxError(ctx, "current unscheduled cleanup failed: %v", err) + } + return + } + e.cleanupLegacyUnscheduledRun(ctx, exptID, runID, spaceID, cause) +} + +func (e *ExptMangerImpl) cleanupLegacyUnscheduledRun(ctx context.Context, exptID, runID, spaceID int64, cause error) { if runID <= 0 { logs.CtxError(ctx, "[ExptEval][RunLock] invalid cleanup run [expt_id=%v run_id=%v]", exptID, runID) return @@ -335,7 +347,13 @@ func (e *ExptMangerImpl) cleanupUnscheduledRun(ctx context.Context, exptID, runI cancelState() unlockCtx, cancelUnlock := context.WithTimeout(ctx, 2*time.Second) - released, unlockErr := e.mutex.UnlockForce(unlockCtx, e.makeExptMutexLockKey(exptID)) + var released bool + var unlockErr error + if e.runFinishRepo != nil { + released, unlockErr = e.mutex.UnlockWithValue(unlockCtx, e.makeExptMutexLockKey(exptID), strconv.FormatInt(runID, 10)) + } else { + released, unlockErr = e.mutex.UnlockForce(unlockCtx, e.makeExptMutexLockKey(exptID)) + } cancelUnlock() if stateErr != nil || unlockErr != nil { logs.CtxError(ctx, "[ExptEval][RunLock] cleanup failed [expt_id=%v run_id=%v state_err=%v unlock_err=%v]", exptID, runID, stateErr, unlockErr) @@ -477,6 +495,42 @@ func (e *ExptMangerImpl) RetryItems(ctx context.Context, exptID, runID, spaceID } func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error { + if e.runFinishRepo == nil { + if _, current := e.itemResultRepo.(repo.IExptCurrentResultRepo); current { + return errno.NewMQRetryErr("current completion provider missing") + } + } + + if e.runFinishRepo != nil { + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), exptID, spaceID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("complete run experiment missing") + } + if current.ExptType == entity.ExptType_Offline { + if current.LatestRunID != exptRunID { + return nil + } + opt := &entity.CompleteExptOption{} + for _, fn := range opts { + fn(opt) + } + if opt.Status == entity.ExptStatus_Terminated || opt.Status == entity.ExptStatus_Failed { + _, err := e.TerminateCurrentRun(ctx, exptID, exptRunID, spaceID, session, nil, opts...) + return err + } + result, err := e.FinishCurrentRun(ctx, &entity.ExptScheduleEvent{ExptID: exptID, ExptRunID: exptRunID, SpaceID: spaceID, Session: session}) + if err != nil { + return err + } + if result == nil || result.Disposition == entity.ExptRunFinishPending { + return errno.NewMQRetryErr("current run completion pending") + } + return nil + } + } const idemKeyPrefix = "CompleteRun:" opt := &entity.CompleteExptOption{} @@ -508,10 +562,6 @@ func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID, spa return err } - if _, err := e.mutex.UnlockForce(ctx, e.makeExptMutexLockKey(exptID)); err != nil { - return err - } - if opt.Status > 0 { runLog.Status = int64(opt.Status) } @@ -521,7 +571,10 @@ func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID, spa logs.CtxInfo(ctx, "[ExptEval] CompleteRun, expt_id: %v, expt_run_id: %v, status: %v, msg: %v", exptID, exptRunID, runLog.Status, opt.StatusMessage) - if err := e.runLogRepo.Save(ctx, runLog); err != nil { + if err := e.runLogRepo.Update(ctx, exptID, exptRunID, spaceID, runLogMutableFields(runLog)); err != nil { + return err + } + if _, err := e.mutex.UnlockForce(ctx, e.makeExptMutexLockKey(exptID)); err != nil { return err } @@ -534,6 +587,16 @@ func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID, spa return nil } +func runLogMutableFields(run *entity.ExptRunLog) map[string]any { + return map[string]any{ + "status": run.Status, "status_message": run.StatusMessage, + "pending_cnt": run.PendingCnt, "processing_cnt": run.ProcessingCnt, + "success_cnt": run.SuccessCnt, "fail_cnt": run.FailCnt, + "terminated_cnt": run.TerminatedCnt, "credit_cost": run.CreditCost, + "token_cost": run.TokenCost, + } +} + func (e *ExptMangerImpl) calculateRunLogStats(ctx context.Context, exptID, exptRunID int64, runLog *entity.ExptRunLog, spaceID int64, session *entity.Session) error { var ( maxLoop = 10000 @@ -603,6 +666,36 @@ func (e *ExptMangerImpl) calculateRunLogStats(ctx context.Context, exptID, exptR } func (e *ExptMangerImpl) CompleteExpt(ctx context.Context, exptID int64, exptRunID *int64, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error { + if e.runFinishRepo == nil { + if _, current := e.itemResultRepo.(repo.IExptCurrentResultRepo); current { + return errno.NewMQRetryErr("current completion provider missing") + } + } + + if e.runFinishRepo != nil { + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), exptID, spaceID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("complete experiment missing") + } + if current.ExptType == entity.ExptType_Offline { + if exptRunID == nil || *exptRunID <= 0 { + return errno.NewMQRetryErr("offline completion requires original run identity") + } + opt := &entity.CompleteExptOption{} + for _, fn := range opts { + fn(opt) + } + if opt.Status == entity.ExptStatus_Terminated || opt.Status == entity.ExptStatus_Failed { + _, err := e.TerminateCurrentRun(ctx, exptID, *exptRunID, spaceID, session, nil, opts...) + return err + } + _, err := e.FinishCurrentRun(ctx, &entity.ExptScheduleEvent{ExptID: exptID, ExptRunID: *exptRunID, SpaceID: spaceID, Session: session}) + return err + } + } const idemKeyPrefix = "CompleteExpt:" opt := &entity.CompleteExptOption{} @@ -991,27 +1084,49 @@ func (e *ExptMangerImpl) releaseCentralQuotaForRun( // （判据只能用预占时刻，分不出"卡死很久"与"刚被接管"，按它释放会真超发）； // zombie 清理只扫 Processing，但实验已终态、daemon 不再跳。 // 三条路都不接 ⇒ 永久泄漏。 - itemIDs, err := e.incompleteItemIDsForRelease(ctx, expt.ID, runID, expt.SpaceID) + observations, err := e.incompleteItemObservationsForRelease(ctx, expt.ID, runID, expt.SpaceID) if err != nil { // 查不到就不猜：宁可这次不释放（留给对账/人工），也不能凭空构造 item 列表。 logs.CtxError(ctx, "[CentralReservation] scan incomplete items fail on %s, skip release, expt_id: %v, expt_run_id: %v, err: %v", occasion, expt.ID, runID, err) return } - if len(itemIDs) == 0 { + if len(observations) == 0 { return } failed := 0 - for _, itemID := range itemIDs { - if err := e.centralGuard.Release(ctx, expt.SchedulerScope, runID, itemID, reason); err != nil { + relCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) + defer cancel() + for _, observation := range observations { + itemID := observation.ItemID + if err := e.centralGuard.Release(relCtx, expt.SchedulerScope, runID, itemID, observation.RequestID, reason); err != nil { failed++ logs.CtxWarn(ctx, "[CentralReservation] release quota fail on %s, scope: %v, expt_run_id: %v, item_id: %v, err: %v", occasion, expt.SchedulerScope, runID, itemID, err) } } logs.CtxInfo(ctx, "[CentralReservation] quota released for incomplete items on %s, scope: %v, expt_id: %v, expt_run_id: %v, items: %v, failed: %v, reason: %v", - occasion, expt.SchedulerScope, expt.ID, runID, len(itemIDs), failed, reason) + occasion, expt.SchedulerScope, expt.ID, runID, len(observations), failed, reason) +} + +func (e *ExptMangerImpl) incompleteItemObservationsForRelease(ctx context.Context, exptID, runID, spaceID int64) ([]*repo.ExptDispatchObservation, error) { + observations, err := e.itemResultRepo.ScanIncompleteDispatchObservations(ctx, spaceID, exptID, runID) + if err != nil { + return nil, err + } + seen := make(map[int64]bool, len(observations)) + observed := make([]*repo.ExptDispatchObservation, 0, len(observations)) + for _, observation := range observations { + if observation == nil || observation.ItemID <= 0 || seen[observation.ItemID] { + return nil, fmt.Errorf("incomplete or foreign quota release observation") + } + seen[observation.ItemID] = true + // The primary query already restricts the original run log to incomplete + // rows. An absorbing terminal canonical projection still owns that token. + observed = append(observed, observation) + } + return observed, nil } // releaseSupersededRunQuota 在重试新建 run 之前，释放**被顶替的上一个 run** 仍挂在账本上的预占。 @@ -1210,6 +1325,24 @@ func (e *ExptMangerImpl) terminateItemTurns(ctx context.Context, exptID int64, i // 因此调用方（前端重试 / 重复点击）可安全重放。 // 沙箱销毁与 CK 读侧刷新为 best-effort，失败仅记日志、不影响返回值。 func (e *ExptMangerImpl) TerminateItems(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64, session *entity.Session) error { + if len(itemIDs) == 0 { + return nil + } + if e.runFinishRepo != nil { + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), exptID, spaceID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("termination experiment missing") + } + if current.ExptType == entity.ExptType_Offline { + return e.terminateCurrentItems(ctx, exptID, exptRunID, spaceID, itemIDs, session) + } + } + if _, current := e.itemResultRepo.(repo.IExptCurrentResultRepo); current && e.runFinishRepo == nil { + return errno.NewMQRetryErr("termination provider missing") + } if len(itemIDs) == 0 { logs.CtxInfo(ctx, "[ExptItemTerminate] skip with empty item_ids, expt_id: %v, expt_run_id: %v", exptID, exptRunID) return nil @@ -1673,10 +1806,6 @@ func (e *ExptMangerImpl) RecordExptData(ctx context.Context, exptID, exptRunID, runLog.Status = int64(entity.ExptStatus_Processing) logs.CtxInfo(ctx, "[ExptEval] RecordExptData run_log, expt_id: %v, expt_run_id: %v, status: %v", exptID, exptRunID, runLog.Status) - if err := e.runLogRepo.Save(ctx, runLog); err != nil { - return err - } - stats, err := e.exptResultService.CalculateStats(ctx, exptID, spaceID, session) if err != nil { return err @@ -1690,11 +1819,8 @@ func (e *ExptMangerImpl) RecordExptData(ctx context.Context, exptID, exptRunID, TerminatedItemCnt: int32(stats.TerminatedItemCnt), } - if err := e.statsRepo.UpdateByExptID(ctx, exptID, spaceID, exptStats); err != nil { - return err - } - - return nil + // The online scheduler holds the run tick lock throughout calibration. + return e.runLogRepo.UpdateRunStats(ctx, exptID, exptRunID, spaceID, runLog, exptStats) } func (e *ExptMangerImpl) ExistCompletingRunLock(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) { @@ -1727,7 +1853,14 @@ func (e *ExptMangerImpl) unlockCompletingRun(ctx context.Context, exptID, exptRu func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (err error) { duration := time.Duration(e.configer.GetExptExecConf(ctx, spaceID).GetZombieIntervalSecond()) * time.Second - locked, err := e.mutex.LockBackoff(ctx, e.makeExptMutexLockKey(exptID), duration, time.Second) + var locked bool + if e.runFinishRepo != nil { + locked, _, err = e.mutex.BackoffLockWithValue(ctx, e.makeExptMutexLockKey(exptID), strconv.FormatInt(exptRunID, 10), duration, time.Second) + } else { + // Existing constructor callers retain the old lock contract. Production + // uses NewExptManagerWithRunFinish and binds the lifecycle owner to runID. + locked, err = e.mutex.LockBackoff(ctx, e.makeExptMutexLockKey(exptID), duration, time.Second) + } if err != nil { return err } @@ -1762,7 +1895,13 @@ func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mo } // 重试顶替旧 run：必须赶在下面改写 LatestRunID 之前释放旧 run 的残留预占。 - e.releaseSupersededRunQuota(ctx, exptID, spaceID, exptRunID, mode) + if e.runFinishRepo != nil { + if err := e.releaseCurrentSupersededRunQuota(ctx, exptID, spaceID, exptRunID, mode); err != nil { + return err + } + } else { + e.releaseSupersededRunQuota(ctx, exptID, spaceID, exptRunID, mode) + } if err := e.exptRepo.Update(ctx, &entity.Experiment{ ID: exptID, @@ -1804,6 +1943,15 @@ func (e *ExptMangerImpl) LogRetryItemsRun(ctx context.Context, exptID int64, mod logs.CtxInfo(ctx, "[ExptEval][RunLock] lock occupied [expt_id=%v holder=%s]", exptID, existedRunID) return 0, false, errorx.NewByCode(errno.ExperimentRunningExistedCode) } + if e.runFinishRepo != nil { + existing, readErr := e.runLogRepo.Get(ctx, exptID, runID) + if readErr != nil { + return 0, false, readErr + } + if existing == nil || existing.SpaceID != spaceID || existing.ExptID != exptID || existing.ExptRunID != runID || entity.ExptRunMode(existing.Mode) != entity.EvaluationModeRetryItems || (existing.Status != int64(entity.ExptStatus_Pending) && existing.Status != int64(entity.ExptStatus_Processing)) { + return 0, false, errorx.NewByCode(errno.ExperimentRunningExistedCode) + } + } logs.CtxDebug(ctx, "[ExptEval][RunLock] joining existing run [expt_id=%v run_id=%v]", exptID, runID) completing, err := e.ExistCompletingRunLock(ctx, exptID, runID, spaceID) @@ -1814,16 +1962,7 @@ func (e *ExptMangerImpl) LogRetryItemsRun(ctx context.Context, exptID int64, mod return 0, false, errorx.NewByCode(errno.ExperimentIsCompletingCode) } - rl, err = e.runLogRepo.Get(ctx, exptID, runID) - if err != nil { - return 0, false, err - } - - if rl == nil { - return 0, false, errorx.New("target runlog %v not found, expt_id: %v", runID, exptID) - } - - if err := rl.AppendItemIDs(itemIDs); err != nil { + if err := e.runLogRepo.AppendRunItemIDs(ctx, exptID, runID, spaceID, itemIDs); err != nil { return 0, false, err } } else { @@ -1839,16 +1978,21 @@ func (e *ExptMangerImpl) LogRetryItemsRun(ctx context.Context, exptID int64, mod if len(itemIDs) > 0 { rl.ItemIds = []entity.ExptRunLogItems{{ItemIDs: itemIDs, CreateAt: gptr.Of(time.Now().Unix())}} } - } - - if err := e.runLogRepo.Save(ctx, rl); err != nil { - return 0, false, err + if err := e.runLogRepo.Create(ctx, rl); err != nil { + return 0, false, err + } } // retried 分支刻意不释放：那说明锁被一个**活着的 run** 持有，runID 是它的号、 // LatestRunID 也还是它 —— 此时释放等于把正在跑的 item 的额度还回去。 if !retried { - e.releaseSupersededRunQuota(ctx, exptID, spaceID, runID, mode) + if e.runFinishRepo != nil { + if err := e.releaseCurrentSupersededRunQuota(ctx, exptID, spaceID, runID, mode); err != nil { + return 0, false, err + } + } else { + e.releaseSupersededRunQuota(ctx, exptID, spaceID, runID, mode) + } if err := e.exptRepo.Update(ctx, &entity.Experiment{ID: exptID, LatestRunID: runID}); err != nil { return 0, false, err @@ -1864,6 +2008,21 @@ func (e *ExptMangerImpl) GetRunLog(ctx context.Context, exptID, exptRunID, space } func (e *ExptMangerImpl) SetExptTerminating(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { + if e.runFinishRepo != nil { + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), exptID, spaceID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("termination experiment missing") + } + if current.ExptType == entity.ExptType_Offline { + return e.beginCurrentRunTermination(ctx, exptID, exptRunID, spaceID) + } + } + if _, current := e.itemResultRepo.(repo.IExptCurrentResultRepo); current && e.runFinishRepo == nil { + return errno.NewMQRetryErr("termination provider missing") + } if err := e.runLogRepo.Update(ctx, exptID, exptRunID, spaceID, map[string]any{"status": int64(entity.ExptStatus_Terminating)}); err != nil { return err } diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index 7598824975..09f3edd7ce 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -282,7 +282,7 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). - Save(ctx, gomock.Any()). + Update(ctx, int64(123), int64(456), int64(789), gomock.Any()). Return(nil) }, wantErr: false, @@ -327,7 +327,7 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). - Save(ctx, gomock.Any()). + Update(ctx, int64(123), int64(456), int64(789), gomock.Any()). Return(nil) mgr.idem.(*idemMocks.MockIdempotentService). @@ -388,7 +388,7 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). - Save(ctx, gomock.Any()). + Update(ctx, int64(123), int64(456), int64(789), gomock.Any()). Return(nil) }, wantErr: false, @@ -708,7 +708,7 @@ func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { BackoffLockWithValue(ctx, gomock.Any(), "1001", gomock.Any(), time.Second). Return(true, "1001", nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT().Save(ctx, gomock.Any()). + EXPECT().Create(ctx, gomock.Any()). Do(func(ctx context.Context, rl *entity.ExptRunLog) { assert.Equal(t, int64(1001), rl.ExptRunID) assert.Len(t, rl.ItemIds, 1) @@ -735,12 +735,8 @@ func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { Return(false, "1001", nil) mgr.mutex.(*lockMocks.MockILocker). EXPECT().Exists(ctx, "expt_completing_mutex_lock:123:1001").Return(false, nil) - existingLog := &entity.ExptRunLog{ID: 1001, ExptID: exptID, ExptRunID: 1001} mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT().Get(ctx, exptID, int64(1001)). - Return(existingLog, nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT().Save(ctx, gomock.Any()).Return(nil) + EXPECT().AppendRunItemIDs(ctx, exptID, int64(1001), spaceID, itemIDs).Return(nil) }, wantRunID: 1001, wantRetry: true, @@ -790,8 +786,8 @@ func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker). EXPECT().Exists(ctx, "expt_completing_mutex_lock:123:1001").Return(false, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT().Get(ctx, exptID, int64(1001)). - Return(nil, errors.New("get run log failed")) + EXPECT().AppendRunItemIDs(ctx, exptID, int64(1001), spaceID, itemIDs). + Return(errors.New("get run log failed")) }, wantErr: true, }, @@ -805,7 +801,7 @@ func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { BackoffLockWithValue(ctx, gomock.Any(), "1006", gomock.Any(), time.Second). Return(true, "1006", nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT().Save(ctx, gomock.Any()).Return(errors.New("save failed")) + EXPECT().Create(ctx, gomock.Any()).Return(errors.New("save failed")) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Update(gomock.Any(), exptID, int64(1006), spaceID, gomock.Any()).Return(nil) mgr.mutex.(*lockMocks.MockILocker).EXPECT().UnlockForce(gomock.Any(), mgr.makeExptMutexLockKey(exptID)).Return(true, nil) }, @@ -3417,14 +3413,6 @@ func TestExptMangerImpl_RecordExptData(t *testing.T) { {Status: int32(entity.TurnRunState_Processing)}, }, int64(2), nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT(). - Save(ctx, gomock.Any()). - Do(func(_ context.Context, rl *entity.ExptRunLog) { - assert.Equal(t, int64(entity.ExptStatus_Processing), rl.Status) - }). - Return(nil) - mgr.exptResultService.(*svcMocks.MockExptResultService). EXPECT(). CalculateStats(ctx, int64(123), int64(789), session). @@ -3436,10 +3424,10 @@ func TestExptMangerImpl_RecordExptData(t *testing.T) { TerminatedItemCnt: 0, }, nil) - mgr.statsRepo.(*repoMocks.MockIExptStatsRepo). + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). - UpdateByExptID(ctx, int64(123), int64(789), gomock.Any()). - Do(func(_ context.Context, _, _ int64, stats *entity.ExptStats) { + UpdateRunStats(ctx, int64(123), int64(456), int64(789), gomock.Any(), gomock.Any()). + Do(func(_ context.Context, _, _, _ int64, run *entity.ExptRunLog, stats *entity.ExptStats) { assert.Equal(t, int32(3), stats.SuccessItemCnt) assert.Equal(t, int32(1), stats.PendingItemCnt) assert.Equal(t, int32(0), stats.FailItemCnt) @@ -3514,9 +3502,10 @@ func TestExptMangerImpl_RecordExptData(t *testing.T) { ListTurnResult(ctx, int64(789), int64(123), nil, gomock.Any(), false). Return([]*entity.ExptTurnResult{}, int64(0), nil) + mgr.exptResultService.(*svcMocks.MockExptResultService).EXPECT().CalculateStats(ctx, int64(123), int64(789), session).Return(&entity.ExptCalculateStats{}, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). - Save(ctx, gomock.Any()). + UpdateRunStats(ctx, int64(123), int64(456), int64(789), gomock.Any(), gomock.Any()). Return(errors.New("save run log error")) err := mgr.RecordExptData(ctx, 123, 456, 789, session) @@ -3545,11 +3534,6 @@ func TestExptMangerImpl_RecordExptData(t *testing.T) { ListTurnResult(ctx, int64(789), int64(123), nil, gomock.Any(), false). Return([]*entity.ExptTurnResult{}, int64(0), nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT(). - Save(ctx, gomock.Any()). - Return(nil) - mgr.exptResultService.(*svcMocks.MockExptResultService). EXPECT(). CalculateStats(ctx, int64(123), int64(789), session). @@ -3581,11 +3565,6 @@ func TestExptMangerImpl_RecordExptData(t *testing.T) { ListTurnResult(ctx, int64(789), int64(123), nil, gomock.Any(), false). Return([]*entity.ExptTurnResult{}, int64(0), nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). - EXPECT(). - Save(ctx, gomock.Any()). - Return(nil) - mgr.exptResultService.(*svcMocks.MockExptResultService). EXPECT(). CalculateStats(ctx, int64(123), int64(789), session). @@ -3593,9 +3572,9 @@ func TestExptMangerImpl_RecordExptData(t *testing.T) { SuccessItemCnt: 5, }, nil) - mgr.statsRepo.(*repoMocks.MockIExptStatsRepo). + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). - UpdateByExptID(ctx, int64(123), int64(789), gomock.Any()). + UpdateRunStats(ctx, int64(123), int64(456), int64(789), gomock.Any(), gomock.Any()). Return(errors.New("update stats error")) err := mgr.RecordExptData(ctx, 123, 456, 789, session) diff --git a/backend/modules/evaluation/domain/service/expt_manage_impl.go b/backend/modules/evaluation/domain/service/expt_manage_impl.go index f6e998b127..ee07476153 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_impl.go @@ -76,6 +76,10 @@ func NewExptManager( // centralGuard 额度闸。CompleteExpt 用它释放实验终态时仍未跑完的 item 预占。 // 开源部署注入 noop（Release 直接返回 nil）。 centralGuard component.ICentralReservationGuard, + manualRetryRepo repo.IExptManualRetryRepo, + evaluatorRecords EvaluatorRecordService, + currentRunWaker component.ICurrentRunSchedulerWaker, + manualRetryLocker ManualRetryLockerFactory, ) IExptManager { return &ExptMangerImpl{ // tupleSvc: tupleSvc, @@ -111,10 +115,21 @@ func NewExptManager( centralScopeProvider: centralScopeProvider, centralAdmissionPolicy: centralAdmissionPolicy, centralGuard: centralGuard, + manualRetryRepo: manualRetryRepo, + evaluatorRecords: evaluatorRecords, + currentRunWaker: currentRunWaker, + manualRetryLocker: manualRetryLocker, } } +type ManualRetryLockerFactory func() lock.ILocker + type ExptMangerImpl struct { + runFinishRepo repo.IExptRunFinishRepo + manualRetryLocker ManualRetryLockerFactory + manualRetryRepo repo.IExptManualRetryRepo + evaluatorRecords EvaluatorRecordService + currentRunWaker component.ICurrentRunSchedulerWaker // tupleSvc IExptTupleService exptResultService ExptResultService exptAggrResultService ExptAggrResultService diff --git a/backend/modules/evaluation/domain/service/expt_manage_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_impl_test.go index a4625bdeb8..5ed3ef4601 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_impl_test.go @@ -53,13 +53,15 @@ func newTestExptManager(ctrl *gomock.Controller) *ExptMangerImpl { // 让位降权灰度开关: 运行发起处(Run/RetryItems/Invoke)会读一次写入 ext, 默认关闭。 configer.EXPECT().GetRetryYieldEnabled(gomock.Any(), gomock.Any()).Return(false).AnyTimes() + items := repoMocks.NewMockIExptItemResultRepo(ctrl) + stubOriginalQuotaObservations(items) return &ExptMangerImpl{ exptResultService: svcMocks.NewMockExptResultService(ctrl), exptAggrResultService: svcMocks.NewMockExptAggrResultService(ctrl), exptRepo: repoMocks.NewMockIExperimentRepo(ctrl), runLogRepo: repoMocks.NewMockIExptRunLogRepo(ctrl), statsRepo: repoMocks.NewMockIExptStatsRepo(ctrl), - itemResultRepo: repoMocks.NewMockIExptItemResultRepo(ctrl), + itemResultRepo: items, turnResultRepo: repoMocks.NewMockIExptTurnResultRepo(ctrl), configer: configer, quotaRepo: repoMocks.NewMockQuotaRepo(ctrl), @@ -1791,6 +1793,7 @@ func TestNewExptManager(t *testing.T) { component.NewNoopCentralSchedulerScopeProvider(), component.NewNoopCentralAdmissionPolicy(), component.NewNoopCentralReservationGuard(), + nil, nil, component.NewNoopCurrentRunSchedulerWaker(), nil, ) impl, ok := mgr.(*ExptMangerImpl) diff --git a/backend/modules/evaluation/domain/service/expt_manual_retry_admission_test.go b/backend/modules/evaluation/domain/service/expt_manual_retry_admission_test.go new file mode 100644 index 0000000000..12fa2186ad --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_manual_retry_admission_test.go @@ -0,0 +1,509 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/cloudwego/kitex/pkg/kerrors" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + + "github.com/stretchr/testify/require" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +func TestManualRetryAdmissionReceiptBeforeDependencies(t *testing.T) { + req := admissionRequest() + req.ItemIDs = []int64{9223372036854775806, 11, 11} + _, hash, err := normalizeCurrentRunRetry(req) + require.NoError(t, err) + raw := `[{"ItemIDs":[11],"manual_retry_receipt":{"request_id":"72cc8921-40f4-4a43-b772-aaf2b5dcdf05","payload_hash":"` + hash + `","accepted_count":1,"rejected_items":[{"item_id":9223372036854775806,"reason":"busy"}],"snapshot_at_ms":1700000000123}},{"manual_retry_receipt":{"request_id":"later","payload_hash":"other","accepted_count":0}}]` + port := &admissionMemoryRepo{} + require.NoError(t, json.Unmarshal([]byte(raw), &port.chunks)) + // All record/config/experiment dependencies absent: replay must precede them. + manager := &ExptMangerImpl{manualRetryRepo: port} + got, err := manager.RetryInCurrentRun(context.Background(), req) + require.NoError(t, err) + require.Equal(t, int32(1), got.AcceptedCount) + require.Equal(t, int64(1700000000123), got.SnapshotAtMS) + require.Equal(t, int64(9223372036854775806), got.RejectedItems[0].ItemID) + req.ItemIDs = []int64{11, 9223372036854775806} + again, err := manager.RetryInCurrentRun(context.Background(), req) + require.NoError(t, err) + require.Equal(t, got, again) + req.Ext = map[string]string{"business": "changed"} + _, err = manager.RetryInCurrentRun(context.Background(), req) + require.ErrorContains(t, err, "request_conflict") +} + +type admissionReceiptRepo struct{} + +func (*admissionReceiptRepo) FindReceipt(context.Context, int64, int64, int64, string, string) (*entity.ManualRetryReceipt, error) { + return nil, nil +} + +func (*admissionReceiptRepo) RequeueTerminalItems(context.Context, *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryReceipt, error) { + return nil, errors.New("unexpected write") +} + +func admissionRequest() *entity.RetryInCurrentRunRequest { + return &entity.RetryInCurrentRunRequest{SpaceID: 9, ExptID: 7, ExpectedRunID: 8, RequestID: "72cc8921-40f4-4a43-b772-aaf2b5dcdf05", Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{11}, Session: &entity.Session{UserID: "operator"}} +} + +func TestManualRetryAdmissionGuards(t *testing.T) { + for _, kind := range []string{"disabled", "bad_uuid", "malformed_uuid", "internal_ext", "failure_selection", "empty_selection", "retry_all", "negative_item", "foreign_run", "unknown_run", "pending", "terminating", "draining", "online", "run_busy", "lost_run_lock"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + expts := repomocks.NewMockIExperimentRepo(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + mutex := lockmocks.NewMockILocker(ctrl) + port := &admissionReceiptRepo{} + manager := &ExptMangerImpl{manualRetryRepo: port, configer: config, exptRepo: expts, itemResultRepo: items, mutex: mutex, manualRetryLocker: func() lock.ILocker { return mutex }} + req := admissionRequest() + got := &entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline} + shape := false + switch kind { + case "bad_uuid": + req.RequestID = "not-uuid" + shape = true + case "malformed_uuid": + req.RequestID = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + shape = true + case "internal_ext": + req.Ext = map[string]string{entity.ManualRetryExtKey: "forged"} + shape = true + case "failure_selection": + req.Mode = entity.EvaluationModeFailRetry + shape = true + case "empty_selection": + req.ItemIDs = nil + shape = true + case "retry_all": + req.Mode = entity.EvaluationModeRetryAll + shape = true + case "negative_item": + req.ItemIDs = []int64{-1} + shape = true + case "foreign_run": + got.LatestRunID = 18 + case "unknown_run": + req.ExpectedRunID = got.LatestRunID + 1 + case "pending": + got.Status = entity.ExptStatus_Pending + case "terminating": + got.Status = entity.ExptStatus_Terminating + case "draining": + got.Status = entity.ExptStatus_Draining + case "online": + got.ExptType = entity.ExptType_Online + } + if !shape { + config.EXPECT().GetExptExecConf(gomock.Any(), int64(9)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(kind != "disabled")}) + if kind != "disabled" { + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).Return(got, nil) + } + } + if kind == "run_busy" || kind == "lost_run_lock" { + items.EXPECT().BatchGet(gomock.Any(), int64(9), int64(7), []int64{11}).Return(nil, nil) + lockCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + if kind == "lost_run_lock" { + cancel() + } + mutex.EXPECT().LockWithRenew(gomock.Any(), "expt_run_exec_lock:7:8", gomock.Any(), gomock.Any()).Return(kind != "run_busy", lockCtx, func() {}, nil) + } + receipt, err := manager.RetryInCurrentRun(context.Background(), req) + require.Error(t, err) + require.Nil(t, receipt) + if kind == "run_busy" { + require.ErrorContains(t, err, "retry_busy") + } + if kind == "foreign_run" || kind == "unknown_run" { + require.ErrorContains(t, err, "run_conflict") + } + if kind == "lost_run_lock" { + require.ErrorIs(t, err, context.Canceled) + } + if kind != "lost_run_lock" { + biz, ok := kerrors.FromBizStatusError(err) + require.True(t, ok, "admission error must survive the PacketAdapter BizStatus boundary") + expected := int32(601204102) + switch { + case shape: + expected = 601200202 + case kind == "disabled": + expected = 601204106 + case kind == "foreign_run" || kind == "unknown_run": + expected = 601204101 + case kind == "run_busy": + expected = 601204105 + } + require.Equal(t, expected, biz.BizStatusCode()) + } + }) + } +} + +func TestManualRetryPayloadIdentity(t *testing.T) { + a := admissionRequest() + a.ItemIDs = []int64{9223372036854775806, 11, 11} + a.Ext = map[string]string{"z": "2", "a": "1"} + normalized, hash, err := normalizeCurrentRunRetry(a) + require.NoError(t, err) + require.Equal(t, []int64{11, 9223372036854775806}, normalized.ItemIDs) + b := admissionRequest() + b.ItemIDs = []int64{11, 9223372036854775806} + b.Ext = map[string]string{"a": "1", "z": "2"} + _, same, err := normalizeCurrentRunRetry(b) + require.NoError(t, err) + require.Equal(t, hash, same) + b.ItemIDs[1]-- + _, different, err := normalizeCurrentRunRetry(b) + require.NoError(t, err) + require.NotEqual(t, hash, different) + b.ItemIDs = a.ItemIDs + b.Ext["z"] = "3" + _, different, err = normalizeCurrentRunRetry(b) + require.NoError(t, err) + require.NotEqual(t, hash, different) + a.Ext["a"] = "changed" + require.Equal(t, "1", normalized.Ext["a"]) + require.Len(t, hash, 64) +} + +func (r *admissionReceiptRepo) PreflightTerminalItems(context.Context, *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) { + return nil, errors.New("unexpected preflight") +} + +type admissionMemoryRepo struct { + chunks []entity.ExptRunLogItems + calls []string + preflightError error + commitError error + preflightRejected bool + afterPreflight func() + beforeCommit func() +} + +func (r *admissionMemoryRepo) FindReceipt(_ context.Context, _, _, _ int64, id, hash string) (*entity.ManualRetryReceipt, error) { + r.calls = append(r.calls, "receipt") + for _, c := range r.chunks { + if p := c.ManualRetryReceipt; p != nil && p.RequestID == id { + if p.PayloadHash != hash { + return nil, errors.New("request_conflict") + } + return p, nil + } + } + return nil, nil +} + +func (r *admissionMemoryRepo) PreflightTerminalItems(_ context.Context, req *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) { + r.calls = append(r.calls, "preflight") + if r.afterPreflight != nil { + defer r.afterPreflight() + } + if r.preflightError != nil { + return nil, r.preflightError + } + plan := &entity.ManualRetryPreparation{Items: req.Items, RejectedItems: req.RejectedItems, SnapshotAtMS: req.SnapshotAtMS} + if r.preflightRejected { + plan.Items = nil + for _, item := range req.Items { + plan.RejectedItems = append(plan.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.SourceItem.ItemID, Reason: "source_changed"}) + } + } + return plan, nil +} + +func (r *admissionMemoryRepo) RequeueTerminalItems(ctx context.Context, req *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryReceipt, error) { + r.calls = append(r.calls, "commit") + if r.beforeCommit != nil { + r.beforeCommit() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if r.commitError != nil { + return nil, r.commitError + } + receipt := &entity.ManualRetryReceipt{RequestID: req.RequestID, PayloadHash: req.PayloadHash, AcceptedCount: int32(len(req.Items)), RejectedItems: req.RejectedItems, SnapshotAtMS: req.SnapshotAtMS} + // Encode and decode the storage representation; assertions cannot pass through pointer identity. + raw, err := json.Marshal(append(r.chunks, entity.ExptRunLogItems{ManualRetryReceipt: receipt})) + if err != nil { + return nil, err + } + requireDecode := json.Unmarshal(raw, &r.chunks) + if requireDecode != nil { + return nil, requireDecode + } + return r.chunks[len(r.chunks)-1].ManualRetryReceipt, nil +} + +type admissionWaker struct { + calls int + err error +} + +func (w *admissionWaker) WakeCurrentRun(context.Context, string) error { w.calls++; return w.err } + +func TestManualRetryAdmissionCommitOrdering(t *testing.T) { + for _, kind := range []string{"legacy", "item_busy", "lost_item_lock", "lock_state_changed", "locked_receipt", "released", "absent", "mismatch", "release_error", "preflight_capacity", "preflight_source_changed", "db_error", "wake_error", "receipt_after_close", "receipt_after_replacement", "lost_during_preflight", "lost_after_release"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + expts := repomocks.NewMockIExperimentRepo(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + refs := repomocks.NewMockIExptItemRefRepo(ctrl) + runs := repomocks.NewMockIExptRunLogRepo(ctrl) + mutex := lockmocks.NewMockILocker(ctrl) + guard := componentmocks.NewMockICentralReservationGuard(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + waker := &admissionWaker{} + port := &admissionMemoryRepo{} + req := admissionRequest() + req.Ext = map[string]string{"business": "unchanged"} + manager := &ExptMangerImpl{manualRetryRepo: port, configer: config, exptRepo: expts, itemResultRepo: items, turnResultRepo: turns, itemRefRepo: refs, runLogRepo: runs, mutex: mutex, centralGuard: guard, publisher: publisher, currentRunWaker: waker, manualRetryLocker: func() lock.ILocker { return mutex }} + expt := &entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline, EvalSetSourceType: entity.ExptEvalSetSourceType_MultiSetConfig, EvalConf: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3), ItemConcurNum: gptr.Of(4)}} + enforce := kind == "released" || kind == "absent" || kind == "mismatch" || kind == "release_error" || kind == "preflight_capacity" || kind == "preflight_source_changed" || kind == "lost_after_release" || kind == "wake_error" + if enforce { + expt.ExptDispatchMode = entity.ExptDispatchModeEnforce + expt.SchedulerScope = "test-scope" + } + run := &entity.ExptRunLog{ID: 8, SpaceID: 9, ExptID: 7, ExptRunID: 8, Status: int64(entity.ExptStatus_Processing), Mode: int32(entity.EvaluationModeSubmit), CreatedBy: "original"} + config.EXPECT().GetExptExecConf(gomock.Any(), int64(9)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}) + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).Return(expt, nil) + oldRun := int64(6) + oldToken := "prior-request" + ext := map[string]string{"other": "keep", entity.ManualRetryExtKey: `{"run_id":6,"request_id":"prior-request","mode":5,"operator":"old","accepted_at_ms":1}`} + source := &entity.ExptItemResult{ID: 101, SpaceID: 9, ExptID: 7, ExptRunID: oldRun, ItemID: 11, ItemVersionID: 91, Status: entity.ItemRunState_Fail, Ext: ext} + items.EXPECT().BatchGet(gomock.Any(), int64(9), int64(7), []int64{11}).Return([]*entity.ExptItemResult{source}, nil) + items.EXPECT().MGetItemRunLog(gomock.Any(), int64(7), int64(8), []int64{11}, int64(9)).Return(nil, nil) + refs.EXPECT().GetByExptIDAndItemID(gomock.Any(), int64(9), int64(7), int64(11)).Return(&entity.ExptItemRef{SpaceID: 9, ExptID: 7, ItemID: 11, ItemVersionID: 91}, nil) + turns.EXPECT().GetItemTurnResults(gomock.Any(), int64(7), int64(11), int64(9)).Return([]*entity.ExptTurnResult{{ID: 301, SpaceID: 9, ExptID: 7, ExptRunID: 6, ItemID: 11, ItemVersionID: 91, TurnID: 21, Status: 2}}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(7), int64(8), int64(11), int64(9)).Return(nil, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(9), []int64{301}).Return(nil, nil) + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + if kind == "lost_during_preflight" { + port.afterPreflight = runCancel + } + mutex.EXPECT().LockWithRenew(gomock.Any(), "expt_run_exec_lock:7:8", 5*time.Second, 5*time.Minute).DoAndReturn(func(context.Context, string, time.Duration, time.Duration) (bool, context.Context, func(), error) { + if kind == "locked_receipt" { + _, hash, _ := normalizeCurrentRunRetry(req) + port.chunks = []entity.ExptRunLogItems{{ManualRetryReceipt: &entity.ManualRetryReceipt{RequestID: req.RequestID, PayloadHash: hash, AcceptedCount: 0, SnapshotAtMS: 77}}} + } + return true, runCtx, runCancel, nil + }) + if kind != "locked_receipt" { + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).DoAndReturn(func(context.Context, int64, int64) (*entity.Experiment, error) { + if kind == "lock_state_changed" { + copy := *expt + copy.LatestRunID = 18 + return ©, nil + } + return expt, nil + }) + } + if kind != "locked_receipt" && kind != "lock_state_changed" { + runs.EXPECT().Get(gomock.Any(), int64(7), int64(8)).Return(run, nil) + itemCtx, itemCancel := context.WithCancel(runCtx) + defer itemCancel() + mutex.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:7:11", 5*time.Second, 5*time.Minute).DoAndReturn(func(context.Context, string, time.Duration, time.Duration) (bool, context.Context, func(), error) { + if kind == "lost_item_lock" { + itemCancel() + } + return kind != "item_busy", itemCtx, itemCancel, nil + }) + } + if kind == "preflight_capacity" { + port.preflightError = errors.New("receipt capacity exceeded") + } + if kind == "preflight_source_changed" { + port.preflightRejected = true + } + if kind == "db_error" { + port.commitError = errors.New("database commit failed") + } + if kind == "wake_error" { + waker.err = errors.New("wake unavailable") + } + if enforce && kind != "preflight_capacity" && kind != "preflight_source_changed" { + guard.EXPECT().ReleaseTerminalReservation(gomock.Any(), "test-scope", oldRun, int64(11), oldToken).DoAndReturn(func(context.Context, string, int64, int64, string) (component.TerminalReservationReleaseResult, error) { + require.Equal(t, "preflight", port.calls[len(port.calls)-1]) + port.calls = append(port.calls, "release") + switch kind { + case "absent": + return component.TerminalReservationAbsent, nil + case "mismatch": + return component.TerminalReservationMismatch, nil + case "release_error": + return 0, errors.New("redis unavailable") + case "lost_after_release": + runCancel() + return component.TerminalReservationReleased, nil + } + return component.TerminalReservationReleased, nil + }) + } + failures := kind == "lost_item_lock" || kind == "lock_state_changed" || kind == "release_error" || kind == "preflight_capacity" || kind == "db_error" || kind == "lost_during_preflight" || kind == "lost_after_release" + publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + callerCtx, callerCancel := context.WithCancel(context.Background()) + defer callerCancel() + got, err := manager.RetryInCurrentRun(callerCtx, req) + require.NoError(t, callerCtx.Err()) + if failures { + require.Error(t, err) + require.Nil(t, got) + require.Empty(t, port.chunks) + return + } + require.NoError(t, err) + require.Equal(t, req.RequestID, got.RequestID) + if kind == "locked_receipt" { + require.Equal(t, int64(77), got.SnapshotAtMS) + require.Equal(t, []string{"receipt", "receipt"}, port.calls) + return + } + expected := int32(1) + if kind == "item_busy" || kind == "mismatch" || kind == "preflight_source_changed" { + expected = 0 + } + require.Equal(t, expected, got.AcceptedCount) + require.Equal(t, "commit", port.calls[len(port.calls)-1]) + require.Equal(t, got, port.chunks[len(port.chunks)-1].ManualRetryReceipt) + require.Equal(t, int32(entity.EvaluationModeSubmit), run.Mode) + require.Equal(t, "original", run.CreatedBy) + if expected == 0 { + require.Len(t, got.RejectedItems, 1) + } + require.Equal(t, entity.ExptStatus_Processing, expt.Status) + require.Equal(t, int64(8), expt.LatestRunID) + require.Equal(t, 4, *expt.EvalConf.ItemConcurNum) + if enforce { + require.Equal(t, 1, waker.calls) + } + if kind == "receipt_after_close" || kind == "receipt_after_replacement" { + expt.Status = entity.ExptStatus_Success + if kind == "receipt_after_replacement" { + expt.LatestRunID = 18 + expt.Status = entity.ExptStatus_Processing + } + manager.configer = nil + manager.exptRepo = nil + manager.evaluatorRecords = nil + } + before := len(port.calls) + again, err := manager.RetryInCurrentRun(context.Background(), req) + require.NoError(t, err) + require.Equal(t, got, again) + require.Equal(t, before+1, len(port.calls)) + conflict := *req + conflict.Ext = map[string]string{"business": "changed"} + _, err = manager.RetryInCurrentRun(context.Background(), &conflict) + require.ErrorContains(t, err, "request_conflict") + }) + } +} + +func TestManualRetryAdmissionUsesIndependentLocker(t *testing.T) { + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + expts := repomocks.NewMockIExperimentRepo(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + shared := lockmocks.NewMockILocker(ctrl) + owned := lockmocks.NewMockILocker(ctrl) + manager := &ExptMangerImpl{manualRetryRepo: &admissionReceiptRepo{}, configer: config, exptRepo: expts, itemResultRepo: items, mutex: shared} + req := admissionRequest() + config.EXPECT().GetExptExecConf(gomock.Any(), int64(9)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}) + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).Return(&entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline}, nil) + items.EXPECT().BatchGet(gomock.Any(), int64(9), int64(7), []int64{11}).Return(nil, nil) + owned.EXPECT().LockWithRenew(gomock.Any(), "expt_run_exec_lock:7:8", gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + // Use a factory so an old renewal can never share ownership with the next admission. + manager.manualRetryLocker = func() lock.ILocker { return owned } + _, err := manager.RetryInCurrentRun(context.Background(), req) + require.ErrorContains(t, err, "retry_busy") +} + +func TestManualRetryLegacyWakeWithoutSnapshot(t *testing.T) { + ctrl := gomock.NewController(t) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + manager := &ExptMangerImpl{publisher: publisher} + expt := &entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline, EvalConf: &entity.EvaluationConfiguration{ItemRetryNum: gptr.Of(3), ItemConcurNum: gptr.Of(4)}} + run := &entity.ExptRunLog{ID: 8, SpaceID: 9, ExptID: 7, ExptRunID: 8, Mode: int32(entity.EvaluationModeSubmit), Status: int64(entity.ExptStatus_Processing), CreatedBy: "original"} + exptBefore, err := json.Marshal(expt) + require.NoError(t, err) + runBefore, err := json.Marshal(run) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err = manager.wakeCurrentRetryRun(ctx, expt, run) + require.ErrorContains(t, err, "original schedule event snapshot unavailable") + require.NoError(t, ctx.Err()) + exptAfter, err := json.Marshal(expt) + require.NoError(t, err) + require.Equal(t, exptBefore, exptAfter) + runAfter, err := json.Marshal(run) + require.NoError(t, err) + require.Equal(t, runBefore, runAfter) +} + +func TestManualRetryLegacyWakePreservesPeriodicTick(t *testing.T) { + ctrl := gomock.NewController(t) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + manager := &ExptMangerImpl{publisher: publisher} + event := &entity.ExptScheduleEvent{SpaceID: 9, ExptID: 7, ExptRunID: 8, ExptRunMode: entity.EvaluationModeSubmit, ExptType: entity.ExptType_Offline, CreatedAt: 1700000000, Session: &entity.Session{UserID: "original", AppID: 566307}, ItemRetryTimes: 3, Ext: map[string]string{entity.RetryYieldExtKey: "true", "business": "frozen"}} + before, err := json.Marshal(event) + require.NoError(t, err) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(9)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 5}) + publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), event, gptr.Of(5*time.Second)).DoAndReturn(func(_ context.Context, got *entity.ExptScheduleEvent, _ *time.Duration) error { + require.Same(t, event, got) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Equal(t, before, encoded) + return nil + }) + require.ErrorContains(t, manager.wakeCurrentRetryRun(context.Background(), &entity.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8}, &entity.ExptRunLog{ExptRunID: 8}), "original schedule event snapshot unavailable") + mode := &ExptSubmitExec{publisher: publisher, configer: config} + require.NoError(t, mode.NextTick(context.Background(), event, true)) + after, err := json.Marshal(event) + require.NoError(t, err) + require.Equal(t, before, after) +} + +func TestManualRetryDependencyErrorRetainsCause(t *testing.T) { + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + expts := repomocks.NewMockIExperimentRepo(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(9)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}).Times(2) + cause := errors.New("RPC read unavailable") + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).Return(nil, cause) + expts.EXPECT().GetByID(gomock.Any(), int64(7), int64(9)).Return(nil, nil) + manager := &ExptMangerImpl{manualRetryRepo: &admissionReceiptRepo{}, configer: config, exptRepo: expts} + _, err := manager.RetryInCurrentRun(context.Background(), admissionRequest()) + require.Same(t, cause, err) + _, ok := kerrors.FromBizStatusError(err) + require.False(t, ok) + _, err = manager.RetryInCurrentRun(context.Background(), admissionRequest()) + require.ErrorContains(t, err, "experiment missing") + _, ok = kerrors.FromBizStatusError(err) + require.False(t, ok) +} diff --git a/backend/modules/evaluation/domain/service/expt_manual_retry_impl.go b/backend/modules/evaluation/domain/service/expt_manual_retry_impl.go new file mode 100644 index 0000000000..c38503ea96 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_manual_retry_impl.go @@ -0,0 +1,577 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "maps" + "sort" + "time" + + "github.com/google/uuid" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/errorx" + "github.com/coze-dev/coze-loop/backend/pkg/logs" +) + +// 与 FinishCurrentRun / 终止链路共用 expt_run_exec_lock，持锁上限必须一致： +// 1 分钟到期会 cancel opCtx，失败项一多整批重试就会回滚。 +const manualRetryLockMaxHold = 5 * time.Minute + +func (m *ExptMangerImpl) prepareTerminalCandidatesAndReusableRefs(ctx context.Context, expt *entity.Experiment, runID int64, mode entity.ExptRunMode, itemIDs []int64, records EvaluatorRecordService) (*entity.ManualRetryPreparation, error) { + ctx = contexts.WithCtxWriteDB(ctx) + plan := &entity.ManualRetryPreparation{SnapshotAtMS: time.Now().UnixMilli()} + var items []*entity.ExptItemResult + if mode == entity.EvaluationModeFailRetry { + if len(itemIDs) != 0 { + return nil, fmt.Errorf("failure retry cannot select items") + } + var cursor int64 + for { + page, next, err := m.itemResultRepo.ScanItemResults(ctx, expt.ID, cursor, 500, []int32{int32(entity.ItemRunState_Fail)}, expt.SpaceID) + if err != nil { + return nil, err + } + items = append(items, page...) + if next == 0 || len(page) == 0 { + break + } + if next <= cursor { + return nil, fmt.Errorf("invalid failure retry cursor") + } + cursor = next + } + } else if mode == entity.EvaluationModeRetryItems { + if len(itemIDs) == 0 { + return nil, fmt.Errorf("manual retry selection is empty") + } + selected := make(map[int64]bool, len(itemIDs)) + for _, id := range itemIDs { + if id <= 0 { + return nil, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("invalid retry item id")) + } + selected[id] = true + } + itemIDs = make([]int64, 0, len(selected)) + for id := range selected { + itemIDs = append(itemIDs, id) + } + sort.Slice(itemIDs, func(i, j int) bool { return itemIDs[i] < itemIDs[j] }) + var err error + items, err = m.itemResultRepo.BatchGet(ctx, expt.SpaceID, expt.ID, itemIDs) + if err != nil { + return nil, err + } + found := make(map[int64]bool) + for _, item := range items { + if item != nil { + found[item.ItemID] = true + } + } + for _, id := range itemIDs { + if !found[id] { + plan.RejectedItems = append(plan.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: id, Reason: "not_found"}) + } + } + } else { + return nil, fmt.Errorf("unsupported manual retry mode %d", mode) + } + for _, item := range items { + if item == nil { + return nil, fmt.Errorf("nil retry item") + } + } + sort.Slice(items, func(i, j int) bool { return items[i].ItemID < items[j].ItemID }) + for _, item := range items { + reason := manualRetryTerminalReason(item.Status) + if reason != "" { + plan.RejectedItems = append(plan.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.ItemID, Reason: reason}) + continue + } + if item.SpaceID != expt.SpaceID || item.ExptID != expt.ID || item.ExptRunID <= 0 { + return nil, fmt.Errorf("invalid retry item source") + } + previous, err := entity.ParseManualRetryContext(item.Ext) + if err != nil { + return nil, err + } + prepared := &entity.PreparedRetryItem{SourceItem: item, PreviousRunID: item.ExptRunID} + if previous != nil { + if previous.RunID != item.ExptRunID { + return nil, fmt.Errorf("manual retry source run mismatch") + } + prepared.PreviousRequestID = previous.RequestID + } + itemLogs, err := m.itemResultRepo.MGetItemRunLog(ctx, expt.ID, runID, []int64{item.ItemID}, expt.SpaceID) + if err != nil { + return nil, err + } + if len(itemLogs) > 1 { + return nil, fmt.Errorf("duplicate current item log") + } + if len(itemLogs) == 1 { + prepared.CurrentItemLog = itemLogs[0] + } + + if log := prepared.CurrentItemLog; log != nil { + if log.SpaceID != expt.SpaceID || log.ExptID != expt.ID || log.ExptRunID != runID || log.ItemID != item.ItemID { + return nil, fmt.Errorf("invalid current item log") + } + reason = manualRetryTerminalReason(entity.ItemRunState(log.Status)) + if reason == "" && log.ResultState != int32(entity.ExptItemResultStateResulted) { + reason = "result_not_ready" + } + if reason != "" { + plan.RejectedItems = append(plan.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.ItemID, Reason: reason}) + continue + } + } else if item.ExptRunID == runID { + return nil, fmt.Errorf("current retry item log missing") + } + switch expt.EvalSetSourceType { + case 0, entity.ExptEvalSetSourceType_SingleSet: + // A committed experiment binding and the existing item/turn rows are + // the frozen single-set source. Never fall back to reading a live draft. + if expt.EvalSetID <= 0 || resolveLoadSpaceID(expt.SpaceID, expt.EvalSetSpaceID) <= 0 || resolveSetReadVersionID(expt.EvalSetID, expt.EvalSetVersionID) == nil || item.ItemVersionID < 0 { + return nil, fmt.Errorf("frozen single-set retry source missing") + } + if log := prepared.CurrentItemLog; log != nil && log.ItemVersionID != item.ItemVersionID { + return nil, fmt.Errorf("frozen retry item version mismatch") + } + prepared.SingleSetSource = &entity.PreparedRetrySingleSetSource{ + SourceType: expt.EvalSetSourceType, EvalSetID: expt.EvalSetID, + EvalSetVersionID: expt.EvalSetVersionID, EvalSetSpaceID: expt.EvalSetSpaceID, + TargetID: expt.TargetID, TargetVersionID: expt.TargetVersionID, + TargetSpaceID: expt.TargetSpaceID, TargetType: expt.TargetType, + } + case entity.ExptEvalSetSourceType_MultiSetConfig: + prepared.FrozenItemRef, err = m.itemRefRepo.GetByExptIDAndItemID(ctx, expt.SpaceID, expt.ID, item.ItemID) + if err != nil { + return nil, err + } + ref := prepared.FrozenItemRef + if ref == nil || ref.SpaceID != expt.SpaceID || ref.ExptID != expt.ID || ref.ItemID != item.ItemID || ref.ItemVersionID != item.ItemVersionID { + return nil, fmt.Errorf("frozen retry item version mismatch") + } + default: + return nil, fmt.Errorf("invalid retry evaluation set source type") + } + prepared.ItemVersionID = item.ItemVersionID + if err := m.prepareManualRetryTurns(ctx, expt, runID, mode, prepared, records); err != nil { + return nil, err + } + plan.Items = append(plan.Items, prepared) + } + sort.Slice(plan.RejectedItems, func(i, j int) bool { return plan.RejectedItems[i].ItemID < plan.RejectedItems[j].ItemID }) + return plan, nil +} + +func manualRetryTerminalReason(status entity.ItemRunState) string { + switch status { + case entity.ItemRunState_Success, entity.ItemRunState_Fail, entity.ItemRunState_Terminal: + return "" + case entity.ItemRunState_Queueing: + return "queueing" + case entity.ItemRunState_Processing: + return "processing" + default: + return "not_terminal" + } +} + +func (m *ExptMangerImpl) prepareManualRetryTurns(ctx context.Context, expt *entity.Experiment, runID int64, mode entity.ExptRunMode, item *entity.PreparedRetryItem, records EvaluatorRecordService) error { + turns, err := m.turnResultRepo.GetItemTurnResults(ctx, expt.ID, item.SourceItem.ItemID, expt.SpaceID) + if err != nil { + return err + } + if len(turns) == 0 { + return fmt.Errorf("frozen retry turns missing") + } + logs, err := m.turnResultRepo.GetItemTurnRunLogs(ctx, expt.ID, runID, item.SourceItem.ItemID, expt.SpaceID) + if err != nil { + return err + } + byTurn := make(map[int64]*entity.ExptTurnResultRunLog) + for _, log := range logs { + if log == nil || log.SpaceID != expt.SpaceID || log.ExptID != expt.ID || log.ExptRunID != runID || log.ItemID != item.SourceItem.ItemID || byTurn[log.TurnID] != nil { + return fmt.Errorf("invalid retry turn log") + } + byTurn[log.TurnID] = log + } + ids := make([]int64, 0, len(turns)) + for _, turn := range turns { + if turn == nil || turn.SpaceID != expt.SpaceID || turn.ExptID != expt.ID || turn.ItemID != item.SourceItem.ItemID || turn.ExptRunID != item.PreviousRunID || turn.ItemVersionID != item.ItemVersionID { + return fmt.Errorf("invalid frozen retry turn") + } + ids = append(ids, turn.ID) + } + refs, err := m.turnResultRepo.BatchGetTurnEvaluatorResultRef(ctx, expt.SpaceID, ids) + if err != nil { + return err + } + refsByTurn := make(map[int64][]*entity.ExptTurnEvaluatorResultRef) + for _, ref := range refs { + if ref == nil || ref.SpaceID != expt.SpaceID || ref.ExptID != expt.ID { + return fmt.Errorf("invalid retry evaluator reference") + } + refsByTurn[ref.ExptTurnResultID] = append(refsByTurn[ref.ExptTurnResultID], ref) + } + source := &retryEvaluatorSourceLogs{repo: m.turnResultRepo, event: &entity.ExptItemEvalEvent{SpaceID: expt.SpaceID, ExptID: expt.ID, ExptRunID: runID, EvalSetItemID: item.SourceItem.ItemID}, logsByRun: map[int64]map[int64]*entity.ExptTurnResultRunLog{runID: byTurn}} + for _, turn := range turns { + log := byTurn[turn.TurnID] + if item.CurrentItemLog != nil && log == nil { + return fmt.Errorf("frozen current retry turn log missing") + } + if log != nil && log.ItemVersionID != item.ItemVersionID { + return fmt.Errorf("retry turn log version mismatch") + } + prepared := &entity.PreparedRetryTurn{SourceTurn: turn, CurrentRunLog: log, SourceRefs: refsByTurn[turn.ID]} + if mode == entity.EvaluationModeFailRetry { + if entity.TurnRunState(turn.Status) == entity.TurnRunState_Success { + prepared.PreserveSuccess = true + prepared.TargetResultID = turn.TargetResultID + prepared.EvaluatorResults = retryEvaluatorResultsFromRefs(turn.ID, prepared.SourceRefs) + } else { + targetSpace := expt.TargetSpaceID + if item.FrozenItemRef != nil { + if cfg := item.FrozenItemRef.ItemConfig; cfg != nil && cfg.TargetSourceSpaceID > 0 { + targetSpace = cfg.TargetSourceSpaceID + } + } + targetVersion := expt.TargetVersionID + targetRequired := !shouldSkipTargetNode(expt) + targetReader := &manualRetryTargetReader{IEvalTargetService: m.evalTargetService, versionID: targetVersion, source: source, turn: turn} + prepared.TargetResultID, prepared.EvaluatorResults, err = failRetrySelectTurnRunLogRefs(ctx, resolveLoadSpaceID(expt.SpaceID, targetSpace), targetRequired, turn, targetReader, records, prepared.SourceRefs, source) + if err != nil { + return err + } + } + for _, ref := range prepared.SourceRefs { + if prepared.PreserveSuccess || manualRetryResultContains(prepared.EvaluatorResults, ref) { + prepared.KeepRefIDs = append(prepared.KeepRefIDs, ref.ID) + } + } + } + item.Turns = append(item.Turns, prepared) + } + for _, byTurn := range source.logsByRun { + for _, log := range byTurn { + item.SourceTurnLogs = append(item.SourceTurnLogs, log) + } + } + sort.Slice(item.SourceTurnLogs, func(i, j int) bool { + a, b := item.SourceTurnLogs[i], item.SourceTurnLogs[j] + if a.ExptRunID != b.ExptRunID { + return a.ExptRunID < b.ExptRunID + } + return a.TurnID < b.TurnID + }) + return nil +} + +func manualRetryResultContains(results *entity.EvaluatorResults, ref *entity.ExptTurnEvaluatorResultRef) bool { + if results == nil { + return false + } + if ref.SourceType == int32(entity.EvaluatorRecordSourceTypeInline) { + for _, r := range results.Inline { + if r != nil && r.RecordID == ref.EvaluatorResultID && r.InlineKey == ref.InlineKey { + return true + } + } + return false + } + for _, r := range results.Registered { + if r != nil && r.RecordID == ref.EvaluatorResultID && r.VersionID == ref.EvaluatorVersionID && r.Alias == ref.Alias { + return true + } + } + return ref.Alias == "" && results.EvalVerIDToResID[ref.EvaluatorVersionID] == ref.EvaluatorResultID +} + +type manualRetryTargetReader struct { + IEvalTargetService + versionID int64 + source *retryEvaluatorSourceLogs + turn *entity.ExptTurnResult +} + +func (r *manualRetryTargetReader) GetRecordByID(ctx context.Context, spaceID, recordID int64) (*entity.EvalTargetRecord, error) { + if r.IEvalTargetService == nil { + return nil, fmt.Errorf("manual retry target service missing") + } + record, err := r.IEvalTargetService.GetRecordByID(ctx, spaceID, recordID) + if err != nil || record == nil { + return record, err + } + if record.SpaceID != spaceID || record.ID != recordID || record.TargetVersionID != r.versionID || record.ExperimentRunID <= 0 || record.ItemID != r.turn.ItemID || record.TurnID != r.turn.TurnID { + return nil, fmt.Errorf("manual retry target source mismatch") + } + if record.Status != nil && *record.Status == entity.EvalTargetRunStatusSuccess { + byTurn, err := r.source.loadRun(ctx, record.ExperimentRunID) + if err != nil { + return nil, err + } + origin := byTurn[r.turn.TurnID] + if origin == nil || origin.TargetResultID != recordID { + return nil, fmt.Errorf("manual retry target source log mismatch") + } + } + return record, nil +} + +func normalizeCurrentRunRetry(req *entity.RetryInCurrentRunRequest) (*entity.RetryInCurrentRunRequest, string, error) { + if req == nil { + return nil, "", errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("invalid manual retry request")) + } + normalized := *req + normalized.Ext = maps.Clone(req.Ext) + selected := make(map[int64]bool, len(req.ItemIDs)) + for _, id := range req.ItemIDs { + selected[id] = true + } + normalized.ItemIDs = make([]int64, 0, len(selected)) + for id := range selected { + normalized.ItemIDs = append(normalized.ItemIDs, id) + } + sort.Slice(normalized.ItemIDs, func(i, j int) bool { return normalized.ItemIDs[i] < normalized.ItemIDs[j] }) + operator := "" + if req.Session != nil { + operator = req.Session.UserID + } + payload := struct { + SpaceID int64 `json:"space_id"` + ExptID int64 `json:"expt_id"` + RunID int64 `json:"run_id"` + Operator string `json:"operator"` + Mode entity.ExptRunMode `json:"mode"` + ItemIDs []int64 `json:"item_ids"` + Ext map[string]string `json:"ext,omitempty"` + }{normalized.SpaceID, normalized.ExptID, normalized.ExpectedRunID, operator, normalized.Mode, normalized.ItemIDs, normalized.Ext} + raw, err := json.Marshal(payload) + if err != nil { + return nil, "", err + } + hash := sha256.Sum256(raw) + return &normalized, hex.EncodeToString(hash[:]), nil +} + +func (m *ExptMangerImpl) RetryInCurrentRun(ctx context.Context, req *entity.RetryInCurrentRunRequest) (*entity.ManualRetryReceipt, error) { + normalized, hash, err := normalizeCurrentRunRetry(req) + if err != nil { + return nil, err + } + if m.manualRetryRepo == nil { + return nil, fmt.Errorf("manual retry repository unavailable") + } + receipt, err := m.manualRetryRepo.FindReceipt(ctx, normalized.SpaceID, normalized.ExptID, normalized.ExpectedRunID, normalized.RequestID, hash) + if err != nil || receipt != nil { + return receipt, err + } + if err := validateCurrentRunRetry(normalized); err != nil { + return nil, err + } + if m.configer == nil { + return nil, fmt.Errorf("manual retry configer unavailable") + } + if !m.configer.GetExptExecConf(ctx, normalized.SpaceID).IsRunningRetryEnabled() { + return nil, errorx.NewByCode(errno.ManualRetryDisabledCode) + } + ctx = contexts.WithCtxWriteDB(ctx) + expt, err := m.exptRepo.GetByID(ctx, normalized.ExptID, normalized.SpaceID) + if err != nil { + return nil, err + } + if err := validateCurrentRetryExperiment(expt, normalized); err != nil { + return nil, err + } + plan, err := m.prepareTerminalCandidatesAndReusableRefs(ctx, expt, normalized.ExpectedRunID, normalized.Mode, normalized.ItemIDs, m.evaluatorRecords) + if err != nil { + return nil, err + } + if m.manualRetryLocker == nil { + return nil, fmt.Errorf("manual retry locker unavailable") + } + locker := m.manualRetryLocker() + if locker == nil { + return nil, fmt.Errorf("manual retry locker unavailable") + } + locked, lockCtx, cancel, err := locker.LockWithRenew(ctx, fmt.Sprintf("expt_run_exec_lock:%d:%d", expt.ID, normalized.ExpectedRunID), 5*time.Second, manualRetryLockMaxHold) + if err != nil { + return nil, err + } + if !locked { + return nil, errorx.NewByCode(errno.ManualRetryBusyCode) + } + defer cancel() + if err := lockCtx.Err(); err != nil { + return nil, err + } + receipt, err = m.manualRetryRepo.FindReceipt(lockCtx, normalized.SpaceID, normalized.ExptID, normalized.ExpectedRunID, normalized.RequestID, hash) + if err != nil || receipt != nil { + return receipt, err + } + expt, err = m.exptRepo.GetByID(lockCtx, normalized.ExptID, normalized.SpaceID) + if err != nil { + return nil, err + } + if err := validateCurrentRetryExperiment(expt, normalized); err != nil { + return nil, err + } + run, err := m.runLogRepo.Get(lockCtx, expt.ID, normalized.ExpectedRunID) + if err != nil { + return nil, err + } + if run == nil { + return nil, fmt.Errorf("manual retry run missing") + } + if run.SpaceID != expt.SpaceID || run.ExptID != expt.ID || run.ExptRunID != normalized.ExpectedRunID { + return nil, errorx.NewByCode(errno.ManualRetryRunConflictCode) + } + if run.Status != int64(entity.ExptStatus_Pending) && run.Status != int64(entity.ExptStatus_Processing) { + return nil, errorx.NewByCode(errno.ManualRetryRunFinishedCode) + } + opCtx, stop := context.WithCancel(lockCtx) + defer stop() + held := make([]context.Context, 0, len(plan.Items)) + candidates := make([]*entity.PreparedRetryItem, 0, len(plan.Items)) + for _, item := range plan.Items { + if err := opCtx.Err(); err != nil { + return nil, err + } + acquired, itemCtx, itemCancel, err := locker.LockWithRenew(opCtx, fmt.Sprintf("expt_item_eval_run_lock:%d:%d", expt.ID, item.SourceItem.ItemID), 5*time.Second, manualRetryLockMaxHold) + if err != nil { + return nil, err + } + if !acquired { + plan.RejectedItems = append(plan.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.SourceItem.ItemID, Reason: "busy"}) + continue + } + defer itemCancel() + unregister := context.AfterFunc(itemCtx, stop) + defer unregister() + held = append(held, itemCtx) + if err := itemCtx.Err(); err != nil { + return nil, err + } + candidates = append(candidates, item) + } + checkLocks := func() error { + if err := opCtx.Err(); err != nil { + return err + } + for _, ctx := range held { + if err := ctx.Err(); err != nil { + return err + } + } + return nil + } + if err := checkLocks(); err != nil { + return nil, err + } + operator := "" + if normalized.Session != nil { + operator = normalized.Session.UserID + } + commit := &entity.RequeueTerminalItemsRequest{SpaceID: expt.SpaceID, ExptID: expt.ID, RunID: normalized.ExpectedRunID, RequestID: normalized.RequestID, PayloadHash: hash, Operator: operator, Mode: normalized.Mode, SnapshotAtMS: plan.SnapshotAtMS, Items: candidates, RejectedItems: plan.RejectedItems} + checked, err := m.manualRetryRepo.PreflightTerminalItems(opCtx, commit) + if err != nil { + return nil, err + } + commit.Items = nil + commit.RejectedItems = checked.RejectedItems + for _, item := range checked.Items { + if err := checkLocks(); err != nil { + return nil, err + } + if expt.ExptDispatchMode == entity.ExptDispatchModeEnforce { + if m.centralGuard == nil || expt.SchedulerScope == "" { + return nil, fmt.Errorf("terminal reservation release unavailable") + } + released, err := m.centralGuard.ReleaseTerminalReservation(opCtx, expt.SchedulerScope, item.PreviousRunID, item.SourceItem.ItemID, item.PreviousRequestID) + if err != nil { + return nil, err + } + switch released { + case component.TerminalReservationReleased, component.TerminalReservationAbsent: + case component.TerminalReservationMismatch: + commit.RejectedItems = append(commit.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.SourceItem.ItemID, Reason: "reservation_mismatch"}) + continue + default: + return nil, fmt.Errorf("invalid terminal reservation release result") + } + } + commit.Items = append(commit.Items, item) + } + if err := checkLocks(); err != nil { + return nil, err + } + receipt, err = m.manualRetryRepo.RequeueTerminalItems(opCtx, commit) + if err != nil { + return nil, err + } + cancel() + stop() + if err := m.wakeCurrentRetryRun(ctx, expt, run); err != nil { + logs.CtxWarn(ctx, "wake current retry run failed, expt_id=%d, run_id=%d, err=%v", expt.ID, run.ExptRunID, err) + } + return receipt, nil +} + +func validateCurrentRunRetry(req *entity.RetryInCurrentRunRequest) error { + if req.SpaceID <= 0 || req.ExptID <= 0 || req.ExpectedRunID <= 0 || len(req.RequestID) != 36 { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("invalid manual retry identity")) + } + if _, err := uuid.Parse(req.RequestID); err != nil { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("invalid manual retry request UUID")) + } + if req.Mode != entity.EvaluationModeFailRetry && req.Mode != entity.EvaluationModeRetryItems { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("unsupported manual retry mode")) + } + if (req.Mode == entity.EvaluationModeFailRetry && len(req.ItemIDs) != 0) || (req.Mode == entity.EvaluationModeRetryItems && len(req.ItemIDs) == 0) { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("invalid manual retry selection")) + } + for _, id := range req.ItemIDs { + if id <= 0 { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("invalid retry item id")) + } + } + if _, exists := req.Ext[entity.ManualRetryExtKey]; exists { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("manual retry ext is internal")) + } + return nil +} + +func validateCurrentRetryExperiment(expt *entity.Experiment, req *entity.RetryInCurrentRunRequest) error { + if expt == nil { + return fmt.Errorf("manual retry experiment missing") + } + if expt.ID != req.ExptID || expt.SpaceID != req.SpaceID || expt.LatestRunID != req.ExpectedRunID { + return errorx.NewByCode(errno.ManualRetryRunConflictCode) + } + if expt.Status != entity.ExptStatus_Processing || expt.ExptType != entity.ExptType_Offline { + return errorx.NewByCode(errno.ManualRetryRunFinishedCode) + } + return nil +} + +func (m *ExptMangerImpl) wakeCurrentRetryRun(ctx context.Context, expt *entity.Experiment, _ *entity.ExptRunLog) error { + if expt.ExptDispatchMode == entity.ExptDispatchModeEnforce { + if m.currentRunWaker == nil { + return fmt.Errorf("central current run wake unavailable") + } + return m.currentRunWaker.WakeCurrentRun(ctx, expt.SchedulerScope) + } + // The original periodic tick retains the frozen Ext and Session. + return fmt.Errorf("original schedule event snapshot unavailable") +} diff --git a/backend/modules/evaluation/domain/service/expt_manual_retry_prepare_test.go b/backend/modules/evaluation/domain/service/expt_manual_retry_prepare_test.go new file mode 100644 index 0000000000..8b3307c092 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_manual_retry_prepare_test.go @@ -0,0 +1,136 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" +) + +type terminalRetryPreparer interface { + prepareTerminalCandidatesAndReusableRefs(context.Context, *entity.Experiment, int64, entity.ExptRunMode, []int64, EvaluatorRecordService) (*entity.ManualRetryPreparation, error) +} + +func TestManualRetryPrepareSuccessfulStages(t *testing.T) { + for _, tc := range []struct { + name string + perSetVersion *int64 + wrongTopVersion bool + wrongEvalSource bool + }{ + {name: "success_without_score"}, + {name: "rpc_error_no_write"}, + {name: "wrong_target_source"}, + {name: "target_source_top41_cfg99", perSetVersion: gptr.Of(int64(99))}, + {name: "target_source_top41_cfg0", perSetVersion: gptr.Of(int64(0))}, + {name: "target_source_wrong_top_record_cfg99", perSetVersion: gptr.Of(int64(99)), wrongTopVersion: true}, + {name: "target_source_wrong_top_record_cfg0", perSetVersion: gptr.Of(int64(0)), wrongTopVersion: true}, + {name: "target_source_wrong_evaluator_cfg0", perSetVersion: gptr.Of(int64(0)), wrongEvalSource: true}, + } { + kind := tc.name + dependencyError := kind == "rpc_error_no_write" + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + refs := repomocks.NewMockIExptItemRefRepo(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + eval := svcmocks.NewMockEvaluatorRecordService(ctrl) + manager := &ExptMangerImpl{itemResultRepo: items, turnResultRepo: turns, itemRefRepo: refs, evalTargetService: target} + preparer, ok := any(manager).(terminalRetryPreparer) + require.True(t, ok, "manual terminal preparation must freeze stages before resetting references") + items.EXPECT().ScanItemResults(gomock.Any(), int64(7), int64(0), int64(500), []int32{3}, int64(9)).DoAndReturn(func(ctx context.Context, _, _, _ int64, _ []int32, _ int64) ([]*entity.ExptItemResult, int64, error) { + require.True(t, contexts.CtxWriteDB(ctx)) + return []*entity.ExptItemResult{{ID: 101, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, Status: entity.ItemRunState_Fail, Ext: map[string]string{"other": "keep"}}}, 0, nil + }) + items.EXPECT().MGetItemRunLog(gomock.Any(), int64(7), int64(8), []int64{11}, int64(9)).Return([]*entity.ExptItemResultRunLog{{ID: 201, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, Status: 3, ResultState: 1, RetryTimes: 3}}, nil) + itemConfig := &entity.ExptItemConfig{TargetSourceSpaceID: 19} + if tc.perSetVersion != nil { + itemConfig.EvalTargetConf = &entity.ItemTargetConf{TargetVersionID: *tc.perSetVersion} + } + refs.EXPECT().GetByExptIDAndItemID(gomock.Any(), int64(9), int64(7), int64(11)).Return(&entity.ExptItemRef{ID: 501, SpaceID: 9, ExptID: 7, ItemID: 11, ItemVersionID: 91, ItemConfig: itemConfig}, nil) + turns.EXPECT().GetItemTurnResults(gomock.Any(), int64(7), int64(11), int64(9)).Return([]*entity.ExptTurnResult{{ID: 301, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, TurnID: 21, Status: 2, TargetResultID: 31, WeightedScore: gptr.Of(0.8)}, {ID: 302, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, TurnID: 22, Status: 1, WeightedScore: gptr.Of(0.0)}}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(7), int64(8), int64(11), int64(9)).Return([]*entity.ExptTurnResultRunLog{{ID: 401, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, TurnID: 21, TargetResultID: 31, EvaluatorResultIds: &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 61, 52: 62}}}, {ID: 402, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, TurnID: 22, Status: entity.TurnRunState_Success}}, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(9), []int64{301, 302}).Return([]*entity.ExptTurnEvaluatorResultRef{{ID: 701, SpaceID: 9, ExptID: 7, ExptTurnResultID: 301, EvaluatorVersionID: 51, EvaluatorResultID: 61}, {ID: 702, SpaceID: 9, ExptID: 7, ExptTurnResultID: 301, EvaluatorVersionID: 52, EvaluatorResultID: 62}}, nil) + var rpcErr error + if dependencyError { + rpcErr = errors.New("record unavailable") + } + targetItem := int64(11) + if kind == "wrong_target_source" { + targetItem = 99 + } + recordVersion := int64(41) + if tc.wrongTopVersion { + recordVersion = 99 + } + target.EXPECT().GetRecordByID(gomock.Any(), int64(19), int64(31)).Return(&entity.EvalTargetRecord{ID: 31, SpaceID: 19, TargetVersionID: recordVersion, ExperimentRunID: 8, ItemID: targetItem, TurnID: 21, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, rpcErr) + if !dependencyError { + evalItem := int64(11) + if tc.wrongEvalSource { + evalItem = 99 + } + eval.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{{ID: 61, ExperimentID: 7, ExperimentRunID: 8, ItemID: evalItem, TurnID: 21, EvaluatorVersionID: 51, Status: entity.EvaluatorRunStatusSuccess}, {ID: 62, Status: entity.EvaluatorRunStatusFail}}, nil).AnyTimes() + } + got, err := preparer.prepareTerminalCandidatesAndReusableRefs(context.Background(), &entity.Experiment{ID: 7, SpaceID: 9, TargetVersionID: 41, EvalSetSourceType: entity.ExptEvalSetSourceType_MultiSetConfig, ExptType: entity.ExptType_Offline}, 8, entity.EvaluationModeFailRetry, nil, eval) + if kind == "wrong_target_source" || tc.wrongTopVersion { + require.ErrorContains(t, err, "target source") + require.Nil(t, got) + return + } + if dependencyError { + require.ErrorContains(t, err, "record unavailable") + require.Nil(t, got) + return + } + require.NoError(t, err) + require.Len(t, got.Items, 1) + item := got.Items[0] + require.Equal(t, int64(8), item.PreviousRunID) + require.Equal(t, int32(3), item.CurrentItemLog.RetryTimes) + require.Equal(t, int64(31), item.Turns[0].TargetResultID) + if tc.wrongEvalSource { + require.Nil(t, item.Turns[0].EvaluatorResults) + require.Empty(t, item.Turns[0].KeepRefIDs) + return + } + require.Equal(t, map[int64]int64{51: 61}, item.Turns[0].EvaluatorResults.EvalVerIDToResID) + require.Equal(t, []int64{701}, item.Turns[0].KeepRefIDs) + require.True(t, item.Turns[1].PreserveSuccess) + require.Equal(t, 0.0, *item.Turns[1].SourceTurn.WeightedScore) + require.Equal(t, "keep", item.SourceItem.Ext["other"]) + }) + } +} + +func TestManualRetryPrepareManualSelection(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + refs := repomocks.NewMockIExptItemRefRepo(ctrl) + preparer, ok := any(&ExptMangerImpl{itemResultRepo: items, turnResultRepo: turns, itemRefRepo: refs}).(terminalRetryPreparer) + require.True(t, ok) + items.EXPECT().BatchGet(gomock.Any(), int64(9), int64(7), []int64{11, 12, 13}).Return([]*entity.ExptItemResult{{ID: 101, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, Status: entity.ItemRunState_Success}, {ItemID: 12, Status: entity.ItemRunState_Processing}, {ItemID: 13, Status: entity.ItemRunState_Queueing}}, nil) + items.EXPECT().MGetItemRunLog(gomock.Any(), int64(7), int64(8), []int64{11}, int64(9)).Return([]*entity.ExptItemResultRunLog{{ID: 201, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, Status: 2, ResultState: 1}}, nil) + refs.EXPECT().GetByExptIDAndItemID(gomock.Any(), int64(9), int64(7), int64(11)).Return(&entity.ExptItemRef{SpaceID: 9, ExptID: 7, ItemID: 11, ItemVersionID: 91}, nil) + turns.EXPECT().GetItemTurnResults(gomock.Any(), int64(7), int64(11), int64(9)).Return([]*entity.ExptTurnResult{{ID: 301, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, TurnID: 21, Status: 1, TargetResultID: 31}}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(7), int64(8), int64(11), int64(9)).Return([]*entity.ExptTurnResultRunLog{{ID: 401, SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, ItemVersionID: 91, TurnID: 21}}, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(9), []int64{301}).Return(nil, nil) + got, err := preparer.prepareTerminalCandidatesAndReusableRefs(context.Background(), &entity.Experiment{ID: 7, SpaceID: 9, EvalSetSourceType: entity.ExptEvalSetSourceType_MultiSetConfig}, 8, entity.EvaluationModeRetryItems, []int64{13, 11, 12, 11}, nil) + require.NoError(t, err) + require.Len(t, got.Items, 1) + require.Equal(t, []*entity.ManualRetryRejectedItem{{ItemID: 12, Reason: "processing"}, {ItemID: 13, Reason: "queueing"}}, got.RejectedItems) + require.Zero(t, got.Items[0].Turns[0].TargetResultID) + require.False(t, got.Items[0].Turns[0].PreserveSuccess) +} diff --git a/backend/modules/evaluation/domain/service/expt_manual_retry_schedule_test.go b/backend/modules/evaluation/domain/service/expt_manual_retry_schedule_test.go new file mode 100644 index 0000000000..b0f4a5dde9 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_manual_retry_schedule_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" +) + +func TestManualRetryScheduleDoesNotResetMaterializedItems(t *testing.T) { + ctrl := gomock.NewController(t) + runs := repomocks.NewMockIExptRunLogRepo(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + stats := repomocks.NewMockIExptStatsRepo(ctrl) + stats.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, int64, int64) (*entity.ExptStats, error) { + require.FailNow(t, "ScheduleStart must not count/reset manually materialized items again") + return nil, nil + }).AnyTimes() + runs.EXPECT().Get(gomock.Any(), int64(7), int64(8)).Return(&entity.ExptRunLog{SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemIds: []entity.ExptRunLogItems{{ItemIDs: []int64{12, 13}}, {ItemIDs: []int64{11}}, {ManualRetryReceipt: &entity.ManualRetryReceipt{RequestID: "rq1", AcceptedCount: 1}}}}, nil).AnyTimes() + items.EXPECT().BatchGet(gomock.Any(), int64(9), int64(7), []int64{11}).Return([]*entity.ExptItemResult{{SpaceID: 9, ExptID: 7, ExptRunID: 8, ItemID: 11, Ext: map[string]string{"manual_retry": `{"run_id":8,"request_id":"rq1","mode":5}`}}}, nil).AnyTimes() + exec := &ExptRetryItemsExec{exptRunLogRepo: runs, exptItemResultRepo: items, exptStatsRepo: stats} + event := &entity.ExptScheduleEvent{SpaceID: 9, ExptID: 7, ExptRunID: 8, ExecEvalSetItemIDs: []int64{12, 13}} + require.NoError(t, exec.ScheduleStart(context.Background(), event, &entity.Experiment{ID: 7, SpaceID: 9})) + require.Equal(t, []int64{12, 13, 11}, event.ExecEvalSetItemIDs) + require.NoError(t, exec.ScheduleStart(context.Background(), event, &entity.Experiment{ID: 7, SpaceID: 9})) + require.Equal(t, []int64{12, 13, 11}, event.ExecEvalSetItemIDs) +} diff --git a/backend/modules/evaluation/domain/service/expt_orphaned_item_requeue_test.go b/backend/modules/evaluation/domain/service/expt_orphaned_item_requeue_test.go index b9bae08504..e11ed167ae 100644 --- a/backend/modules/evaluation/domain/service/expt_orphaned_item_requeue_test.go +++ b/backend/modules/evaluation/domain/service/expt_orphaned_item_requeue_test.go @@ -5,9 +5,11 @@ package service import ( "context" "errors" + "fmt" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" @@ -34,119 +36,32 @@ func admittedEnforceCtx(event *entity.ExptItemEvalEvent) context.Context { return ctx } -// TestReservationAbsent_RequeuesOrphanedProcessingItem 主场景：Processing 且账本无 reservation -// ⇒ 必须退回 Queueing，并把 stats 的 Processing 计数与主表状态一起退回。 -func TestReservationAbsent_RequeuesOrphanedProcessingItem(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - dispatchRepo := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - dispatchRepo.EXPECT().MGetDispatchObservations(gomock.Any(), int64(3), int64(1), int64(2), []int64{4}). - Return([]*repo.ExptDispatchObservation{ - {ItemID: 4, Status: int32(entity.ItemRunState_Processing), QuotaReservationState: entity.QuotaReservationStateNone}, - }, nil) - // ★ 核心断言：退回动作真的发生在这个 item 上。 - dispatchRepo.EXPECT().RequeueProcessingItem(gomock.Any(), int64(3), int64(1), int64(2), int64(4)). - Return(true, nil) - - var gotStatsOp *entity.StatsCntArithOp - statsRepo := repoMocks.NewMockIExptStatsRepo(ctrl) - statsRepo.EXPECT().ArithOperateCount(gomock.Any(), int64(1), int64(3), gomock.Any()). - DoAndReturn(func(_ context.Context, _, _ int64, op *entity.StatsCntArithOp) error { - gotStatsOp = op - return nil +// Requeue owns canonical/turn/log/stats atomically. The consumer refreshes +// only the committed filter and never repeats the database writes. +func TestReservationAbsent_RequeueAtomicAndFilter(t *testing.T) { + for _, changed := range []bool{false, true} { + t.Run(fmt.Sprint(changed), func(t *testing.T) { + ctrl := gomock.NewController(t) + dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) + dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), int64(3), int64(1), int64(2), []int64{4}).Return([]*repo.ExptDispatchObservation{{ItemID: 4, Status: int32(entity.ItemRunState_Processing), RequestID: "rq2"}}, nil) + // Deliberately observe RQ2 while the event is RQ1: never substitute latest. + dispatch.EXPECT().RequeueProcessingItem(gomock.Any(), int64(3), int64(1), int64(2), int64(4), "rq1").Return(changed, nil) + result := svcMocks.NewMockExptResultService(ctrl) + if changed { + result.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), []int64{4}).Return(nil) + } + svc := &ExptItemEventEvalServiceImpl{ + centralGuard: &fakeGuard{confirmResult: false}, dispatchRepo: dispatch, resultSvc: result, + exptItemResultRepo: repoMocks.NewMockIExptItemResultRepo(ctrl), exptTurnResultRepo: repoMocks.NewMockIExptTurnResultRepo(ctrl), exptStatsRepo: repoMocks.NewMockIExptStatsRepo(ctrl), + } + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"} + svc.exptItemResultRepo.(*repoMocks.MockIExptItemResultRepo).EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(4)).Return(&entity.ExptItemCurrentRequest{RunID: 2, ItemID: 4, RequestID: "rq1", Status: entity.ItemRunState_Processing}, nil) + require.NoError(t, svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { + t.Fatal("absent reservation must stop") + return nil + })(admittedEnforceCtx(event), event)) }) - - var gotFields map[string]any - itemResultRepo := repoMocks.NewMockIExptItemResultRepo(ctrl) - itemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), int64(3), int64(1), []int64{4}, gomock.Any()). - DoAndReturn(func(_ context.Context, _, _ int64, _ []int64, ufields map[string]any) error { - gotFields = ufields - return nil - }) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: &fakeGuard{confirmResult: false}, - dispatchRepo: dispatchRepo, - exptStatsRepo: statsRepo, - exptItemResultRepo: itemResultRepo, } - - event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4} - ctx := admittedEnforceCtx(event) - - nextCalled := false - err := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { - nextCalled = true - return nil - })(ctx, event) - - assert.NoError(t, err, "这条消息注定不执行，返回 error 只会让 MQ 无休止重投") - assert.False(t, nextCalled, "reservation 不存在时不得执行 item") - - assert.Equal(t, -1, gotStatsOp.OpStatusCnt[entity.ItemRunState_Processing], - "Processing 桶必须减 1，否则 processing_turn_count 只增不减") - assert.Equal(t, 1, gotStatsOp.OpStatusCnt[entity.ItemRunState_Queueing], - "item 回到队列，Queueing 桶要加回来") - assert.Equal(t, int32(entity.ItemRunState_Queueing), gotFields["status"], - "主表是详情页的数据源，不退回会一直显示成执行中") -} - -// TestReservationAbsent_RollsBackTurnTableAndFilter 派发侧推进五项（run log / 主表 / stats / -// turn 主表 / CK 加速表），回退必须同样是五项。只退前三项会留下「item 已回队列、turn 与 CK -// 仍显示执行中」的错位，而且不自愈 —— 实验若以 Failed 收口，CompleteExpt 的 default 分支 -// 只销毁沙箱、不动 turn 状态。 -func TestReservationAbsent_RollsBackTurnTableAndFilter(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - dispatchRepo := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - dispatchRepo.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*repo.ExptDispatchObservation{ - {ItemID: 4, Status: int32(entity.ItemRunState_Processing), QuotaReservationState: entity.QuotaReservationStateNone}, - }, nil) - dispatchRepo.EXPECT().RequeueProcessingItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(true, nil) - - statsRepo := repoMocks.NewMockIExptStatsRepo(ctrl) - statsRepo.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - itemResultRepo := repoMocks.NewMockIExptItemResultRepo(ctrl) - itemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - - var gotTurnFields map[string]any - turnResultRepo := repoMocks.NewMockIExptTurnResultRepo(ctrl) - turnResultRepo.EXPECT().UpdateTurnResultsWithItemIDs(gomock.Any(), int64(1), []int64{4}, int64(3), gomock.Any()). - DoAndReturn(func(_ context.Context, _ int64, _ []int64, _ int64, ufields map[string]any) error { - gotTurnFields = ufields - return nil - }) - - var gotFilterItemIDs []int64 - resultSvc := svcMocks.NewMockExptResultService(ctrl) - resultSvc.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), gomock.Any()). - DoAndReturn(func(_ context.Context, _, _ int64, itemIDs []int64) error { - gotFilterItemIDs = itemIDs - return nil - }) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: &fakeGuard{confirmResult: false}, - dispatchRepo: dispatchRepo, - exptStatsRepo: statsRepo, - exptItemResultRepo: itemResultRepo, - exptTurnResultRepo: turnResultRepo, - resultSvc: resultSvc, - } - - event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4} - assert.NoError(t, svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { - return nil - })(admittedEnforceCtx(event), event)) - - assert.Equal(t, int32(entity.TurnRunState_Queueing), gotTurnFields["status"], - "turn 主表不退回，详情页 turn 会一直显示运行中而 item 显示排队中") - assert.Equal(t, []int64{4}, gotFilterItemIDs, - "CK 不刷新，开加速器时按「运行中」筛会筛出实际在排队的 item") } // TestReservationAbsent_LeavesTerminalItemUntouched 迟到消息：item 已终态， @@ -187,14 +102,14 @@ func TestReservationAbsent_ResetsStaleQueueingReserved(t *testing.T) { Return([]*repo.ExptDispatchObservation{ {ItemID: 4, Status: int32(entity.ItemRunState_Queueing), QuotaReservationState: entity.QuotaReservationStateReserved}, }, nil) - dispatchRepo.EXPECT().ResetQuotaReserved(gomock.Any(), int64(3), int64(1), int64(2), []int64{4}). + dispatchRepo.EXPECT().ResetQuotaReserved(gomock.Any(), int64(3), int64(1), int64(2), []int64{4}, map[int64]string{4: "rq1"}). Return([]int64{4}, nil) svc := &ExptItemEventEvalServiceImpl{ centralGuard: &fakeGuard{confirmResult: false}, dispatchRepo: dispatchRepo, } - event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4} + event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4, RequestID: "rq1"} assert.NoError(t, svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { t.Fatal("Queueing/reserved 的 item 本轮不得执行") return nil @@ -241,7 +156,7 @@ func TestReservationAbsent_CASMissLeavesStatsAndMainTableUntouched(t *testing.T) {ItemID: 4, Status: int32(entity.ItemRunState_Processing), QuotaReservationState: entity.QuotaReservationStateNone}, }, nil) // CAS 未命中：并发路径已经把这条 run log 改走了。 - dispatchRepo.EXPECT().RequeueProcessingItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + dispatchRepo.EXPECT().RequeueProcessingItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), ""). Return(false, nil) // 两个 mock 都不 EXPECT 任何调用 —— gomock 会让意外调用直接失败，这正是本用例的断言。 @@ -254,6 +169,7 @@ func TestReservationAbsent_CASMissLeavesStatsAndMainTableUntouched(t *testing.T) exptStatsRepo: statsRepo, exptItemResultRepo: itemResultRepo, } + itemResultRepo.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(4)).Return(&entity.ExptItemCurrentRequest{RunID: 2, ItemID: 4, Status: entity.ItemRunState_Processing}, nil) event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4} assert.NoError(t, svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { t.Fatal("CAS 未命中时不得执行 item") diff --git a/backend/modules/evaluation/domain/service/expt_result_filter_sync.go b/backend/modules/evaluation/domain/service/expt_result_filter_sync.go new file mode 100644 index 0000000000..8fe0971944 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_result_filter_sync.go @@ -0,0 +1,367 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "fmt" + "maps" + "reflect" + "slices" + "strconv" + "time" + + "github.com/bytedance/gg/gptr" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/logs" +) + +// ExptManualRetryFilterService checks materialized data, including Resulted items. +// It remains separate from the older result client contract. +// +//go:generate mockgen -destination=mocks/expt_result_filter_sync.go -package=mocks . ExptManualRetryFilterService +type ExptManualRetryFilterService interface { + ReconcileManualRetryFilterBeforeFinish(context.Context, int64, int64, int64) (bool, error) +} + +// Each refresh owns an independent holder for the existing execution item key. +// Callbacks already holding this key call only the private locked helpers. +func (e ExptResultServiceImpl) withFilterItemLock(ctx context.Context, exptID, itemID int64, fn func(context.Context) error) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if e.resultPublishLocker == nil { + if _, current := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo); current { + return errno.NewMQRetryErr("filter independent item locker missing") + } + return fn(ctx) // Compatibility for older unit-test doubles only. + } + locker := e.resultPublishLocker() + if locker == nil { + return errno.NewMQRetryErr("filter item locker missing") + } + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", exptID, itemID) + locked, lockCtx, cancel, err := locker.LockWithRenew(ctx, key, 5*time.Second, time.Minute) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !locked { + return errno.NewMQRetryErr("filter item lock busy") + } + defer cancel() + defer func() { + if _, err := locker.Unlock(key); err != nil { + logs.CtxWarn(ctx, "unlock filter item: %v", err) + } + }() + if err := lockCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if err := fn(contexts.WithCtxWriteDB(lockCtx)); err != nil { + return errno.WrapMQRetryErr(err) + } + if err := lockCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + return nil +} + +func sortedFilterItems(ids []int64) []int64 { + ids = slices.Clone(ids) + slices.Sort(ids) + return slices.Compact(ids) +} + +func (e ExptResultServiceImpl) filterCandidateItems(ctx context.Context, spaceID, exptID int64, ids []int64) ([]int64, error) { + if len(ids) > 0 { + return sortedFilterItems(ids), nil + } + // Only IDs escape this enumeration. Payloads are read again under each lock. + items, _, err := e.ExptItemResultRepo.ListItemResultsByExptID(contexts.WithCtxWriteDB(ctx), exptID, spaceID, entity.Page{}, false) + if err != nil { + return nil, err + } + for _, item := range items { + if item != nil { + ids = append(ids, item.ItemID) + } + } + return sortedFilterItems(ids), nil +} + +func (e ExptResultServiceImpl) UpsertExptTurnResultFilter(ctx context.Context, spaceID, exptID int64, ids []int64) error { + if spaceID == 0 || exptID == 0 { + return fmt.Errorf("UpsertExptTurnResultFilter: invalid space_id or expt_id") + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if e.resultPublishLocker == nil { + if _, current := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo); !current { + return e.upsertExptTurnResultFilterLocked(ctx, spaceID, exptID, ids) + } + } + ids, err := e.filterCandidateItems(ctx, spaceID, exptID, ids) + if err != nil { + return err + } + var firstErr error + for _, item := range ids { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if err := e.withFilterItemLock(ctx, exptID, item, func(ctx context.Context) error { + return e.upsertExptTurnResultFilterLocked(ctx, spaceID, exptID, []int64{item}) + }); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return errno.WrapMQRetryErr(ctxErr) + } + if firstErr == nil { + firstErr = err + logs.CtxWarn(ctx, "UpsertExptTurnResultFilter item failed, continuing batch, expt_id: %d, item_id: %d, err: %v", exptID, item, err) + } + } + } + return firstErr +} + +func (e ExptResultServiceImpl) CompareExptTurnResultFilters(ctx context.Context, spaceID, exptID int64, ids []int64, retry int32) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if e.resultPublishLocker == nil { + if _, current := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo); !current { + return e.compareExptTurnResultFiltersLegacy(ctx, spaceID, exptID, ids, retry) + } + } + ids, err := e.filterCandidateItems(ctx, spaceID, exptID, ids) + if err != nil { + return err + } + var firstErr error + for _, item := range ids { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if err := e.withFilterItemLock(ctx, exptID, item, func(ctx context.Context) error { + return e.compareCurrentItemFilterLocked(ctx, spaceID, exptID, item, retry) + }); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return errno.WrapMQRetryErr(ctxErr) + } + if firstErr == nil { + firstErr = err + logs.CtxWarn(ctx, "CompareExptTurnResultFilters item failed, continuing batch, expt_id: %d, item_id: %d, err: %v", exptID, item, err) + } + } + } + return firstErr +} + +func (e ExptResultServiceImpl) ReconcileManualRetryFilterBeforeFinish(ctx context.Context, space, expt, run int64) (bool, error) { + scanner, ok := e.ExptItemResultRepo.(repo.IExptManualRetryFilterRepo) + if !ok { + return false, errno.NewMQRetryErr("manual retry filter enumeration missing") + } + ids := []int64{} + cursor := int64(0) + for { + if err := ctx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + page, next, err := scanner.ScanManualRetryFilterItems(ctx, space, expt, run, cursor, 200) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + ids = append(ids, page...) + if next == 0 { + break + } + if next <= cursor { + return false, errno.NewMQRetryErr("filter enumeration cursor stalled") + } + cursor = next + } + for _, item := range sortedFilterItems(ids) { + if err := e.withFilterItemLock(ctx, expt, item, func(ctx context.Context) error { + if e.resultPublishRepo == nil || e.exptTurnResultFilterRepo == nil { + return errno.NewMQRetryErr("filter calibration dependencies missing") + } + // Publication's source reader accepts Resulted and finished runs. Execution's + // active-only source reader would silently lose precisely these candidates. + source, err := e.resultPublishRepo.GetItemResultPublishSource(ctx, space, expt, run, item) + if err != nil { + return err + } + if source == nil { + return errno.NewMQRetryErr("filter calibration current binding unavailable") + } + manual, err := entity.ParseManualRetryContext(source.Item.Ext) + if err != nil { + return err + } + if manual == nil || manual.RunID != run || manual.RequestID != source.Source.Current.RequestID { + return errno.NewMQRetryErr("filter calibration request changed") + } + if !entity.IsItemRunFinished(source.Item.Status) || !entity.IsItemRunFinished(entity.ItemRunState(source.ItemLog.Status)) || source.ItemLog.ResultState != int32(entity.ExptItemResultStateResulted) { + return errno.NewMQRetryErr("filter calibration publication pending") + } + filters, err := e.buildCurrentItemFilterLocked(ctx, space, expt, item) + if err != nil { + return err + } + if len(filters) == 0 { + return errno.NewMQRetryErr("filter calibration source turns missing") + } + if err := ctx.Err(); err != nil { + return err + } + if err := e.exptTurnResultFilterRepo.Save(ctx, filters); err != nil { + return err + } + return ctx.Err() + }); err != nil { + return false, errno.WrapMQRetryErr(err) + } + } + return true, nil +} + +func (e ExptResultServiceImpl) materializedFilterMatchesLocked(ctx context.Context, space, expt, item int64) (bool, error) { + filters, err := e.buildCurrentItemFilterLocked(ctx, space, expt, item) + if err != nil { + return false, err + } + return e.compareMaterializedFilterLocked(ctx, space, expt, item, filters) +} + +func (e ExptResultServiceImpl) compareMaterializedFilterLocked(ctx context.Context, space, expt, item int64, expected []*entity.ExptTurnResultFilterEntity) (bool, error) { + if len(expected) == 0 { + return false, nil + } + actual, fullRead, err := e.readMaterializedItemFilterLocked(ctx, space, expt, item, expected[0].CreatedDate) + if err != nil { + return false, err + } + if len(actual) != len(expected) { + return false, nil + } + byTurn := e.createTurnKeyToFilterMap(actual) + compareItemVersion := true + if storage, ok := e.exptTurnResultFilterRepo.(repo.IExptMaterializedItemVersionRepo); ok { + compareItemVersion = storage.PersistsMaterializedItemVersion() + } + for _, want := range expected { + got := byTurn[GenerateTurnKey(space, expt, item, want.TurnID)] + // Both repositories' current compare contract returns these filter fields. + // No future BMQ source guard or ingestion ordering is implied by equality. + if got == nil || got.ItemIdx != want.ItemIdx || got.Status != want.Status || got.EvaluatorScoreCorrected != want.EvaluatorScoreCorrected || !maps.Equal(got.EvaluatorScore, want.EvaluatorScore) || got.EvalTargetData["actual_output"] != want.EvalTargetData["actual_output"] { + return false, nil + } + if fullRead && (compareItemVersion && got.ItemVersionID != want.ItemVersionID || got.EvalSetID != want.EvalSetID || got.EvalSetVersionID != want.EvalSetVersionID || !maps.Equal(got.EvalTargetData, want.EvalTargetData) || !maps.Equal(got.AnnotationFloat, want.AnnotationFloat) || !maps.Equal(got.AnnotationBool, want.AnnotationBool) || !maps.Equal(got.AnnotationString, want.AnnotationString) || !maps.Equal(got.EvalTargetMetrics, want.EvalTargetMetrics) || !filterWeightedScoreEqual(got.EvaluatorWeightedScore, want.EvaluatorWeightedScore)) { + return false, nil + } + } + if !fullRead { + return false, nil + } // Existing async readers cannot prove complete projection equality. + return true, ctx.Err() +} + +func (e ExptResultServiceImpl) buildCurrentItemFilterLocked(ctx context.Context, space, expt, item int64) ([]*entity.ExptTurnResultFilterEntity, error) { + return e.buildExptTurnResultFiltersLocked(contexts.WithCtxWriteDB(ctx), space, expt, []int64{item}) +} + +// The existing CK schema serializes a nil weighted score as non-null zero. +// Per-evaluator maps retain key presence, so missing scores never become 0. +func filterWeightedScoreEqual(got, want *float64) bool { + if want == nil { + return got == nil || *got == 0 + } + return reflect.DeepEqual(got, want) +} + +// Called only by local paths whose execution mutex still encloses the call. +func refreshFilterWithHeldItemLock(ctx context.Context, svc ExptResultService, space, expt, item int64) error { + if local, ok := svc.(*ExptResultServiceImpl); ok { + return local.upsertExptTurnResultFilterLocked(contexts.WithCtxWriteDB(ctx), space, expt, []int64{item}) + } + return svc.UpsertExptTurnResultFilter(ctx, space, expt, []int64{item}) +} + +// Recover only the final calibration boundary so runtime failures cannot reach +// the scheduler's CompleteRun/Failed fallback. +func reconcileManualRetryFilterBeforeFinish(ctx context.Context, service ExptManualRetryFilterService, space, expt, run int64) (ready bool, err error) { + defer func() { + if failure := recover(); failure != nil { + ready = false + err = errno.NewMQRetryErr(fmt.Sprintf("filter calibration runtime failure: %v", failure)) + } + }() + return service.ReconcileManualRetryFilterBeforeFinish(ctx, space, expt, run) +} + +func (e ExptResultServiceImpl) readMaterializedItemFilterLocked(ctx context.Context, space, expt, item int64, date time.Time) ([]*entity.ExptTurnResultFilterEntity, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + started := time.Now() + full, complete := e.exptTurnResultFilterRepo.(repo.IExptMaterializedFilterRepo) + var rows []*entity.ExptTurnResultFilterEntity + var err error + if complete { + rows, err = full.GetMaterializedByExptIDItemIDs(ctx, strconv.FormatInt(space, 10), strconv.FormatInt(expt, 10), date.Format("2006-01-02"), []string{strconv.FormatInt(item, 10)}) + } else { + rows, err = e.exptTurnResultFilterRepo.GetByExptIDItemIDs(ctx, strconv.FormatInt(space, 10), strconv.FormatInt(expt, 10), date.Format("2006-01-02"), []string{strconv.FormatInt(item, 10)}) + } + if e.Metric != nil { + e.Metric.EmitExptTurnResultFilterQueryLatency(space, started.Unix(), err != nil) + } + return rows, complete, err +} + +func (e ExptResultServiceImpl) compareCurrentItemFilterLocked(ctx context.Context, space, expt, item int64, retry int32) error { + expected, err := e.buildCurrentItemFilterLocked(ctx, space, expt, item) + if err != nil { + return err + } + if len(expected) == 0 || expected[0].CreatedDate.IsZero() { + return nil + } + actual, _, err := e.readMaterializedItemFilterLocked(ctx, space, expt, item, expected[0].CreatedDate) + if err != nil { + return err + } + byTurn := e.createTurnKeyToFilterMap(actual) + for _, want := range expected { + got := byTurn[GenerateTurnKey(space, expt, item, want.TurnID)] + missing := got == nil + scoreDiff, outputDiff, diff := false, false, missing + if !missing { + scoreDiff = !maps.Equal(want.EvaluatorScore, got.EvaluatorScore) + outputDiff = want.EvalTargetData["actual_output"] != got.EvalTargetData["actual_output"] + diff = scoreDiff || outputDiff || want.Status != got.Status || want.ItemIdx != got.ItemIdx || want.EvaluatorScoreCorrected != got.EvaluatorScoreCorrected + } + if !diff || retry >= 3 { + if e.Metric != nil { + e.Metric.EmitExptTurnResultFilterCheck(space, scoreDiff, outputDiff, diff, missing) + } + if diff { + logs.CtxError(ctx, "CompareExptTurnResultFilters finish, diff exist, retryTimes >= maxRetryTimes, itemID: %d, turnID: %d, retryTimes: %d", item, want.TurnID, retry) + } + continue + } + if err := ctx.Err(); err != nil { + return err + } + if err := e.publisher.PublishExptTurnResultFilterEvent(ctx, &entity.ExptTurnResultFilterEvent{ExperimentID: expt, SpaceID: space, ItemID: []int64{item}, RetryTimes: gptr.Of(retry + 1), FilterType: gptr.Of(entity.UpsertExptTurnResultFilterTypeCheck)}, gptr.Of(10*time.Second)); err != nil { + return err + } + } + return nil +} diff --git a/backend/modules/evaluation/domain/service/expt_result_impl.go b/backend/modules/evaluation/domain/service/expt_result_impl.go index 20f1ad21bc..8c8b4a2ab1 100644 --- a/backend/modules/evaluation/domain/service/expt_result_impl.go +++ b/backend/modules/evaluation/domain/service/expt_result_impl.go @@ -93,6 +93,8 @@ func NewExptResultService( } type ExptResultServiceImpl struct { + resultPublishRepo repo.IExptResultPublishRepo + resultPublishLocker ManualRetryLockerFactory ExptItemResultRepo repo.IExptItemResultRepo ExptTurnResultRepo repo.IExptTurnResultRepo ExptStatsRepo repo.IExptStatsRepo @@ -156,6 +158,27 @@ func (e ExptResultServiceImpl) GetExptItemTurnResults(ctx context.Context, exptI } func (e ExptResultServiceImpl) RecordItemRunLogs(ctx context.Context, exptID, exptRunID, itemID, spaceID int64, expt *entity.Experiment) ([]*entity.ExptTurnEvaluatorResultRef, error) { + if e.resultPublishRepo != nil { + source, err := e.GetItemResultPublishSource(ctx, spaceID, exptID, exptRunID, itemID) + if err != nil { + return nil, err + } + if !publicationReady(source) { + return nil, nil + } + publication, err := e.PrepareItemResultPublication(ctx, source, expt) + if err != nil { + return nil, err + } + applied, err := e.PublishPreparedItemResult(ctx, publication, nil) + if err != nil || !applied { + return nil, err + } + return publication.Refs, nil + } + if _, actual := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo); actual { + return nil, errno.NewMQRetryErr("real result repository missing atomic publication wiring") + } itemRunLog, err := e.ExptItemResultRepo.GetItemRunLog(ctx, exptID, exptRunID, itemID, spaceID) if err != nil { return nil, err @@ -2876,13 +2899,33 @@ func (e ExptResultServiceImpl) ManualUpsertExptTurnResultFilter(ctx context.Cont logs.CtxError(ctx, "Failed to send ExptTurnResultFilterEvent, err: %v", err) } + if e.resultPublishLocker != nil { + return e.UpsertExptTurnResultFilter(ctx, spaceID, exptID, itemIDs) + } + if _, current := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo); current { + return errno.NewMQRetryErr("manual filter independent item locker missing") + } return nil } -func (e ExptResultServiceImpl) UpsertExptTurnResultFilter(ctx context.Context, spaceID, exptID int64, itemIDs []int64) error { +func (e ExptResultServiceImpl) upsertExptTurnResultFilterLocked(ctx context.Context, space, expt int64, ids []int64) error { + filters, err := e.buildExptTurnResultFiltersLocked(ctx, space, expt, ids) + if err != nil { + return err + } + if len(filters) == 0 { + return nil + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + return e.exptTurnResultFilterRepo.Save(ctx, filters) +} + +func (e ExptResultServiceImpl) buildExptTurnResultFiltersLocked(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptTurnResultFilterEntity, error) { // 当前方法中space_id和expt_id必填，item_ids选填 if spaceID == 0 || exptID == 0 { - return fmt.Errorf("UpsertExptTurnResultFilter: invalid space_id or expt_id") + return nil, fmt.Errorf("UpsertExptTurnResultFilter: invalid space_id or expt_id") } ctx = contexts.WithCtxWriteDB(ctx) // 更新result时需要取最新的result @@ -2893,11 +2936,11 @@ func (e ExptResultServiceImpl) UpsertExptTurnResultFilter(ctx context.Context, s var allTurnResults []*entity.ExptTurnResult for { if loopCnt >= maxLoop { - return fmt.Errorf("UpsertExptTurnResultFilter: 超过最大循环次数，可能存在死循环，已查%d条", len(allTurnResults)) + return nil, fmt.Errorf("UpsertExptTurnResultFilter: 超过最大循环次数，可能存在死循环，已查%d条", len(allTurnResults)) } turnResults, total, err := e.ExptTurnResultRepo.ListTurnResultByItemIDs(ctx, spaceID, exptID, itemIDs, entity.NewPage(offset, limit), false) if err != nil { - return err + return nil, err } if len(turnResults) == 0 { break @@ -2910,7 +2953,7 @@ func (e ExptResultServiceImpl) UpsertExptTurnResultFilter(ctx context.Context, s loopCnt++ } if len(allTurnResults) == 0 { - return nil + return nil, nil } itemIDMap := make(map[int64]bool) for _, turnResult := range allTurnResults { @@ -2921,11 +2964,11 @@ func (e ExptResultServiceImpl) UpsertExptTurnResultFilter(ctx context.Context, s }) itemResults, err := e.ExptItemResultRepo.BatchGet(ctx, spaceID, exptID, itemIDs) if err != nil { - return err + return nil, err } exptTurnResultFilterKeyMappings, err := e.exptTurnResultFilterRepo.GetExptTurnResultFilterKeyMappings(ctx, spaceID, exptID) if err != nil { - return err + return nil, err } exptTurnResultFilterKeyMappingEvaluatorMap := make(map[string]*entity.ExptTurnResultFilterKeyMapping) exptTurnResultFilterKeyMappingAnnotationMap := make(map[string]*entity.ExptTurnResultFilterKeyMapping) @@ -2947,19 +2990,28 @@ func (e ExptResultServiceImpl) UpsertExptTurnResultFilter(ctx context.Context, s e.ExptTurnResultRepo, e.ExptItemResultRepo, e.ExptAnnotateRepo, e.evalTargetRepo, e.evalTargetService, e.evaluatorRecordService, e.evaluationSetItemService, e.evaluationSetVersionService, e.evaluationSetService, e.analysisService, exptTurnResultFilterKeyMappingEvaluatorMap, exptTurnResultFilterKeyMappingAnnotationMap, make(map[int64]entity.ItemRunState), e.fileProvider, e.scoreCalculator) // ★ 多评测集实验: 逐 item 取归属评测集(含版本), 供写 expt_turn_result_filter 时按行回填。 - // 拿不到(老调用方未注入 repo / 查询失败)时只告警, 退化为按主集写, 不阻塞写入。 - payloadBuilder.ItemID2EvalSetRef = e.loadItemEvalSetRefs(ctx, spaceID, exptID, itemIDs) + // 正式刷新在锁内读取，错误上抛；未注入 factory 的旧测试保留兼容路径。 + if e.resultPublishLocker != nil && e.exptItemRefRepo != nil { + refs, err := e.exptItemRefRepo.MGetByExptIDAndItemIDs(ctx, spaceID, exptID, itemIDs) + if err != nil { + return nil, err + } + payloadBuilder.ItemID2EvalSetRef = make(map[int64]*entity.ExptItemRef, len(refs)) + for _, ref := range refs { + if ref != nil { + payloadBuilder.ItemID2EvalSetRef[ref.ItemID] = ref + } + } + } else { + payloadBuilder.ItemID2EvalSetRef = e.loadItemEvalSetRefs(ctx, spaceID, exptID, itemIDs) + } exptTurnResultFilters, err := payloadBuilder.BuildTurnResultFilter(ctx) if err != nil { - return err - } - - if err = e.exptTurnResultFilterRepo.Save(ctx, exptTurnResultFilters); err != nil { - return err + return nil, err } - return nil + return exptTurnResultFilters, nil } // loadItemEvalSetRefs 取 item -> expt_item_ref 映射 (多评测集实验的评测集归属真值源)。 @@ -3253,7 +3305,7 @@ func (e ExptResultServiceImpl) InsertExptTurnResultFilterKeyMappings(ctx context return e.exptTurnResultFilterRepo.InsertExptTurnResultFilterKeyMappings(ctx, mappings) } -func (e ExptResultServiceImpl) CompareExptTurnResultFilters(ctx context.Context, spaceID, exptID int64, itemIDs []int64, retryTimes int32) error { +func (e ExptResultServiceImpl) compareExptTurnResultFiltersLegacy(ctx context.Context, spaceID, exptID int64, itemIDs []int64, retryTimes int32) error { ctx = contexts.WithCtxWriteDB(ctx) // 更新result时需要取最新的result exptDO, err := e.ExperimentRepo.MGetByID(ctx, []int64{exptID}, spaceID) if err != nil { diff --git a/backend/modules/evaluation/domain/service/expt_result_publish.go b/backend/modules/evaluation/domain/service/expt_result_publish.go new file mode 100644 index 0000000000..c17d529c40 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_result_publish.go @@ -0,0 +1,358 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/bytedance/gg/gptr" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/logs" +) + +// ExptResultPublicationService keeps external record reads and publication outside SQL. +// The callback is invoked after an item lock and source recheck, immediately before commit. +// +//go:generate mockgen -destination=mocks/expt_result_publish.go -package=mocks . ExptResultPublicationService +type ExptResultPublicationService interface { + GetItemResultPublishSource(context.Context, int64, int64, int64, int64) (*entity.ExptItemResultPublishSource, error) + PrepareItemResultPublication(context.Context, *entity.ExptItemResultPublishSource, *entity.Experiment) (*entity.ExptItemResultPublication, error) + PublishPreparedItemResult(context.Context, *entity.ExptItemResultPublication, func(context.Context, *entity.ExptItemResultPublishSource) error) (bool, error) +} + +func (e *ExptResultServiceImpl) GetItemResultPublishSource(ctx context.Context, space, expt, run, item int64) (*entity.ExptItemResultPublishSource, error) { + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if e.resultPublishRepo == nil { + return nil, errno.NewMQRetryErr("result publication repository missing") + } + source, err := e.resultPublishRepo.GetItemResultPublishSource(ctx, space, expt, run, item) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + return source, nil +} + +func publicationReady(source *entity.ExptItemResultPublishSource) bool { + return source != nil && source.Source != nil && source.ItemLog != nil && source.ItemLog.ResultState == int32(entity.ExptItemResultStateLogged) && entity.IsItemRunFinished(entity.ItemRunState(source.ItemLog.Status)) +} + +func (e *ExptResultServiceImpl) PrepareItemResultPublication(ctx context.Context, source *entity.ExptItemResultPublishSource, expt *entity.Experiment) (*entity.ExptItemResultPublication, error) { + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if !publicationReady(source) { + return nil, nil + } + if source.Item == nil || len(source.Source.CanonicalTurns) == 0 || len(source.Source.CanonicalTurns) != len(source.Source.Turns) { + return nil, errno.NewMQRetryErr("publication source incomplete") + } + ctx = contexts.WithCtxWriteDB(ctx) + c := source.Source.Current + if expt == nil || expt.ID != c.ExptID || expt.SpaceID != c.SpaceID { + return nil, errno.NewMQRetryErr("publication experiment source mismatch") + } + publication := &entity.ExptItemResultPublication{Source: source} + weights := buildScoreWeights(expt) + for _, old := range source.Source.CanonicalTurns { + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if old == nil { + return nil, errno.NewMQRetryErr("publication canonical turn missing") + } + log := source.Source.Turns[old.TurnID] + if log == nil { + return nil, errno.NewMQRetryErr("publication frozen turn missing") + } + turn := *old + turn.Status = int32(log.Status) + turn.TargetResultID = log.TargetResultID + turn.TraceID = log.TraceID + turn.LogID = log.LogID + turn.ErrMsg = log.ErrMsg + turn.ExptRunID = c.RunID + turn.WeightedScore = nil + if log.TargetResultID > 0 { + targetSpace := resolveLoadSpaceID(c.SpaceID, expt.TargetSpaceID) + if expt.EvalSetSourceType == entity.ExptEvalSetSourceType_MultiSetConfig { + if source.ItemRef == nil || source.ItemRef.ItemConfig == nil { + return nil, errno.NewMQRetryErr("publication frozen item configuration missing") + } + if source.ItemRef.ItemConfig.TargetSourceSpaceID > 0 { + targetSpace = source.ItemRef.ItemConfig.TargetSourceSpaceID + } + } + if e.evalTargetService == nil { + return nil, errno.NewMQRetryErr("publication target reader missing") + } + record, err := e.evalTargetService.GetRecordByID(ctx, targetSpace, log.TargetResultID) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if record == nil || record.ID != log.TargetResultID || record.ItemID != c.ItemID || record.TurnID != log.TurnID || record.SpaceID != targetSpace || record.ExperimentRunID <= 0 || record.Status == nil || *record.Status < entity.EvalTargetRunStatusSuccess || *record.Status > entity.EvalTargetRunStatusAsyncInvoking || record.ItemVersionID != 0 && record.ItemVersionID != c.ItemVersionID { + return nil, errno.NewMQRetryErr("publication target source record incomplete or foreign") + } + } + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + refs := NewTurnEvaluatorResultRefs(0, c.ExptID, old.ID, c.SpaceID, log.EvaluatorResultIds) + ids := make([]int64, 0, len(refs)) + seen := map[int64]bool{} + for _, ref := range refs { + if ref == nil || ref.EvaluatorResultID <= 0 { + return nil, errno.NewMQRetryErr("publication evaluator ref incomplete") + } + if !seen[ref.EvaluatorResultID] { + ids = append(ids, ref.EvaluatorResultID) + seen[ref.EvaluatorResultID] = true + } + } + scoreRecords := map[string]*entity.EvaluatorRecord{} + if len(ids) > 0 { + if e.evaluatorRecordService == nil { + return nil, errno.NewMQRetryErr("publication evaluator reader missing") + } + records, err := e.evaluatorRecordService.BatchGetEvaluatorRecord(ctx, ids, false, false) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + byID := map[int64]*entity.EvaluatorRecord{} + for _, record := range records { + if record == nil || byID[record.ID] != nil { + return nil, errno.NewMQRetryErr("publication evaluator records incomplete or duplicated") + } + byID[record.ID] = record + } + for _, ref := range refs { + record := byID[ref.EvaluatorResultID] + if record == nil || record.ItemID != c.ItemID || record.TurnID != log.TurnID || record.ExperimentID != c.ExptID || record.ExperimentRunID <= 0 || record.SpaceID <= 0 || record.Status < entity.EvaluatorRunStatusSuccess || record.Status > entity.EvaluatorRunStatusSkipped || record.ItemVersionID != 0 && record.ItemVersionID != c.ItemVersionID || record.TargetRecordID != 0 && record.TargetRecordID != log.TargetResultID { + return nil, errno.NewMQRetryErr("publication evaluator source record incomplete or foreign") + } + if ref.SourceType == int32(entity.EvaluatorRecordSourceTypeInline) { + if record.SourceType != entity.EvaluatorRecordSourceTypeInline || record.InlineKey != ref.InlineKey { + return nil, errno.NewMQRetryErr("publication inline source mismatch") + } + } else { + if record.SourceType == entity.EvaluatorRecordSourceTypeInline || record.EvaluatorVersionID != ref.EvaluatorVersionID || record.Alias != ref.Alias { + return nil, errno.NewMQRetryErr("publication registered source mismatch") + } + if record.Status == entity.EvaluatorRunStatusSuccess { + scoreRecords[entity.EncodeEvaluatorInstanceKey(record.EvaluatorVersionID, record.Alias)] = record + } + } + } + } + if len(scoreRecords) > 0 { + if e.scoreCalculator != nil { + turn.WeightedScore = e.scoreCalculator.CalculateWeightedScore(ctx, expt, scoreRecords, weights) + } else { + turn.WeightedScore = calculateWeightedScore(scoreRecords, weights) + } + } + publication.Turns = append(publication.Turns, &turn) + publication.Refs = append(publication.Refs, refs...) + } + sort.Slice(publication.Refs, func(i, j int) bool { + a, b := publication.Refs[i], publication.Refs[j] + if a.ExptTurnResultID != b.ExptTurnResultID { + return a.ExptTurnResultID < b.ExptTurnResultID + } + if a.EvaluatorVersionID != b.EvaluatorVersionID { + return a.EvaluatorVersionID < b.EvaluatorVersionID + } + if a.Alias != b.Alias { + return a.Alias < b.Alias + } + return a.InlineKey < b.InlineKey + }) + var missing []*entity.ExptTurnEvaluatorResultRef + for _, ref := range publication.Refs { + for _, old := range source.Source.CurrentRefs { + if old.ExptTurnResultID == ref.ExptTurnResultID && old.EvaluatorVersionID == ref.EvaluatorVersionID && old.EvaluatorResultID == ref.EvaluatorResultID && old.Alias == ref.Alias && old.InlineKey == ref.InlineKey && old.SourceType == ref.SourceType { + ref.ID = old.ID + break + } + } + if ref.ID == 0 { + missing = append(missing, ref) + } + } + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if len(missing) > 0 { + if e.idgen == nil { + return nil, errno.NewMQRetryErr("publication reference ID generator missing") + } + ids, err := e.idgen.GenMultiIDs(ctx, len(missing)) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if len(ids) != len(missing) { + return nil, errno.NewMQRetryErr("publication ref IDs incomplete") + } + for i, ref := range missing { + if ids[i] <= 0 { + return nil, errno.NewMQRetryErr("publication ref ID invalid") + } + ref.ID = ids[i] + } + } + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + return publication, nil +} + +func (e *ExptResultServiceImpl) PublishPreparedItemResult(ctx context.Context, publication *entity.ExptItemResultPublication, beforeCommit func(context.Context, *entity.ExptItemResultPublishSource) error) (bool, error) { + if err := ctx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + if publication == nil { + return false, nil + } + if !publicationReady(publication.Source) { + return false, errno.NewMQRetryErr("publication prepared source invalid") + } + if e.resultPublishRepo == nil || e.resultPublishLocker == nil { + return false, errno.NewMQRetryErr("publication repository or independent item locker missing") + } + c := publication.Source.Source.Current + locker := e.resultPublishLocker() + if locker == nil { + return false, errno.NewMQRetryErr("publication item locker missing") + } + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", c.ExptID, c.ItemID) + locked, lockCtx, cancel, err := locker.LockWithRenew(ctx, key, 5*time.Second, time.Minute) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if !locked { + return false, errno.NewMQRetryErr("publication item lock busy") + } + defer cancel() + defer func() { + if _, err := locker.Unlock(key); err != nil { + logs.CtxWarn(ctx, "unlock publication item: %v", err) + } + }() + valid, err := e.resultPublishRepo.IsItemResultPublishSourceCurrent(lockCtx, publication.Source) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if !valid { + return false, nil + } + if err := lockCtx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + if beforeCommit != nil { + if err := beforeCommit(lockCtx, publication.Source); err != nil { + return false, err + } + } + if err := lockCtx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } + applied, err := e.resultPublishRepo.CommitItemResult(lockCtx, publication) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + return applied, nil +} + +func (e *ExptSchedulerImpl) recordCurrentItemResults(ctx context.Context, event *entity.ExptScheduleEvent, items []*entity.ExptEvalItem, mode entity.ExptSchedulerMode, expt *entity.Experiment, service ExptResultPublicationService) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + var itemMeta map[int64]*entity.EvaluationSetItem + var itemVer map[int64]int64 + if e.itemCompletePublisher != nil && len(items) > 0 { + itemMeta, itemVer = e.resolveItemCompleteMeta(ctx, event, items, expt) + } + appliedItems := []int64{} + for _, item := range items { + if item == nil { + return errno.NewMQRetryErr("nil scanned publication item") + } + source := item.Publication + if source == nil && item.PublicationObserved { + continue + } + if source == nil { + // Legacy callers do not carry the new snapshot. Freeze before preparation, + // matching the original scanned token/status; never attach a newer request. + var err error + source, err = service.GetItemResultPublishSource(ctx, event.SpaceID, event.ExptID, event.ExptRunID, item.ItemID) + if err != nil { + return err + } + if source == nil || source.Source.Current.RequestID != item.RequestID || source.ItemLog.Status != int32(item.State) { + continue + } + if item.Source != nil && !sameExecutionIdentity(item.Source, source.Source) { + continue + } + } + if !publicationReady(source) { + continue + } + current := source.Source.Current + if current.SpaceID != event.SpaceID || current.ExptID != event.ExptID || current.RunID != event.ExptRunID || current.ItemID != item.ItemID || current.RequestID != item.RequestID { + return errno.NewMQRetryErr("publication execution scope mismatch") + } + publication, err := service.PrepareItemResultPublication(ctx, source, expt) + if err != nil { + return err + } + applied, err := service.PublishPreparedItemResult(ctx, publication, func(lockCtx context.Context, source *entity.ExptItemResultPublishSource) error { + if e.itemCompletePublisher == nil || entity.ItemRunState(source.ItemLog.Status) != entity.ItemRunState_Success { + return nil + } + current := *item + current.State = entity.ItemRunState(source.ItemLog.Status) + current.RequestID = source.Source.Current.RequestID + return e.sendItemComplete(lockCtx, event, expt, ¤t, itemMeta[item.ItemID], itemVer[item.ItemID]) + }) + if err != nil { + return err + } + if !applied { + continue + } + appliedItems = append(appliedItems, item.ItemID) + // Existing PublishResult failure remains best effort after commit. Task4.3 + // handles the final projection repair; Resulted is not scanned as Logged. + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if err := mode.PublishResult(ctx, publication.Refs, event); err != nil { + logs.CtxError(ctx, "publish online result after commit: %v", err) + } + } + if len(appliedItems) == 0 { + return nil + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if err := e.ResultSvc.UpsertExptTurnResultFilter(ctx, event.SpaceID, event.ExptID, appliedItems); err != nil { + logs.CtxError(ctx, "UpsertExptTurnResultFilter after commit: %v", err) + } + if e.Publisher == nil { + return nil + } + return e.Publisher.PublishExptTurnResultFilterEvent(ctx, &entity.ExptTurnResultFilterEvent{ExperimentID: event.ExptID, SpaceID: event.SpaceID, ItemID: appliedItems, RetryTimes: gptr.Of(int32(0)), FilterType: gptr.Of(entity.UpsertExptTurnResultFilterTypeCheck)}, gptr.Of(10*time.Second)) +} diff --git a/backend/modules/evaluation/domain/service/expt_result_publish_provider.go b/backend/modules/evaluation/domain/service/expt_result_publish_provider.go new file mode 100644 index 0000000000..149dd4c26e --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_result_publish_provider.go @@ -0,0 +1,48 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "github.com/coze-dev/coze-loop/backend/infra/idgen" + "github.com/coze-dev/coze-loop/backend/infra/platestwrite" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" +) + +// NewExptResultServiceWithPublication is the production Wire provider. +// NewExptResultService stays compatible with existing callers and legacy test doubles. +func NewExptResultServiceWithPublication( + exptItemResultRepo repo.IExptItemResultRepo, + exptTurnResultRepo repo.IExptTurnResultRepo, + exptAnnotateRepo repo.IExptAnnotateRepo, + exptStatsRepo repo.IExptStatsRepo, + experimentRepo repo.IExperimentRepo, + metric metrics.ExptMetric, + lwt platestwrite.ILatestWriteTracker, + idgen idgen.IIDGenerator, + exptTurnResultFilterRepo repo.IExptTurnResultFilterRepo, + evaluatorService EvaluatorService, + evalTargetService IEvalTargetService, + evalTargetRepo repo.IEvalTargetRepo, + evaluationSetVersionService EvaluationSetVersionService, + evaluationSetService IEvaluationSetService, + evaluatorRecordService EvaluatorRecordService, + evaluationSetItemService EvaluationSetItemService, + publisher events.ExptEventPublisher, + tagRPCAdapter rpc.ITagRPCAdapter, + analysisService IEvaluationAnalysisService, + fileProvider rpc.IFileProvider, + scoreCalculator IEvaluatorScoreCalculator, + // exptItemRefRepo ★ MultiSetConfig 实验按 item 回填 eval_set_id/eval_set_version_id 用。 + // variadic 保持既有调用方（含商业化 wire）向后兼容。 + resultPublishRepo repo.IExptResultPublishRepo, + resultPublishLocker ManualRetryLockerFactory, + exptItemRefRepo ...repo.IExptItemRefRepo, +) ExptResultService { + result := NewExptResultService(exptItemResultRepo, exptTurnResultRepo, exptAnnotateRepo, exptStatsRepo, experimentRepo, metric, lwt, idgen, exptTurnResultFilterRepo, evaluatorService, evalTargetService, evalTargetRepo, evaluationSetVersionService, evaluationSetService, evaluatorRecordService, evaluationSetItemService, publisher, tagRPCAdapter, analysisService, fileProvider, scoreCalculator, exptItemRefRepo...).(*ExptResultServiceImpl) + result.resultPublishRepo = resultPublishRepo + result.resultPublishLocker = resultPublishLocker + return result +} diff --git a/backend/modules/evaluation/domain/service/expt_result_publish_test.go b/backend/modules/evaluation/domain/service/expt_result_publish_test.go new file mode 100644 index 0000000000..42590ab9da --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_result_publish_test.go @@ -0,0 +1,589 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "encoding/json" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + + "github.com/bytedance/gg/gptr" + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + entitymocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity/mocks" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/gorm" +) + +func publicationFixture(t *testing.T) (*lateWriteFixture, *ExptResultServiceImpl, *svcmocks.MockEvaluatorRecordService, *entity.Experiment) { + t.Helper() + f := newLateWriteFixture(t, "rq1") + // Keep one legal turn0; the version-zero record remains legacy-compatible. + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND turn_id<>0", f.space).Delete(&model.ExptTurnResultRunLog{}).Error) + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND turn_id<>0", f.space).Delete(&model.ExptTurnResult{}).Error) + require.NoError(t, f.conn.Where("space_id=?", f.space).Delete(&model.ExptTurnEvaluatorResultRef{}).Error) + refs, err := json.Marshal(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: f.base + 61}}}) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.TurnRunState_Success), "evaluator_result_ids": refs}).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": int32(entity.ExptItemResultStateLogged)}).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=?", f.space).Update("weighted_score", 77).Error) + ctrl := gomock.NewController(t) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + gen := idmocks.NewMockIIDGenerator(ctrl) + var seq atomic.Int64 + gen.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, n int) ([]int64, error) { + out := make([]int64, n) + for i := range out { + out[i] = f.base + 3000 + seq.Add(1) + } + return out, nil + }).AnyTimes() + target.EXPECT().GetRecordByID(gomock.Any(), f.space, f.base+41).Return(&entity.EvalTargetRecord{ID: f.base + 41, SpaceID: f.space, ExperimentRunID: f.run, ItemID: f.item, TurnID: 0, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil).AnyTimes() + e := &ExptResultServiceImpl{ExptItemResultRepo: f.itemRepo(), ExptTurnResultRepo: f.turnRepo(), ExptStatsRepo: experimentrepo.NewExptStatsRepo(experimentmysql.NewExptStatsDAO(f.provider)), idgen: gen, evaluatorRecordService: records, evalTargetService: target, scoreCalculator: NewEvaluatorScoreCalculator(nil, nil), resultPublishRepo: experimentrepo.NewExptResultPublishRepo(f.provider), resultPublishLocker: lateWriteLocker(t, ctrl)} + expt := &entity.Experiment{ID: f.expt, SpaceID: f.space, LatestRunID: f.run, EvalConf: &entity.EvaluationConfiguration{}} + return f, e, records, expt +} + +func publicationRecord(f *lateWriteFixture, score *float64) *entity.EvaluatorRecord { + return &entity.EvaluatorRecord{ID: f.base + 61, SpaceID: f.space, ExperimentID: f.expt, ExperimentRunID: f.run, ItemID: f.item, TurnID: 0, EvaluatorVersionID: f.base + 51, Alias: "alias", Status: entity.EvaluatorRunStatusSuccess, EvaluatorOutputData: &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: score}}} +} + +func TestResultPublishDirectRealRollback(t *testing.T) { + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_stats"} { + t.Run(table, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil).AnyTimes() + before := f.snapshot() + fault := errors.New("publication write fault") + hits := 0 + hook := "publish_fault_" + table + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(hook, func(tx *gorm.DB) { + if tx.Statement.Table == table { + hits++ + _ = tx.AddError(fault) + } + })) + _, err := svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, f.conn.Callback().Update().Remove(hook)) + require.Positive(t, hits) + require.ErrorIs(t, err, fault) + require.Equal(t, before, f.snapshot(), "publication must roll back refs, score, item/resulted and stats together") + }) + } +} + +func TestResultPublishDirectStalePreparation(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + var afterMutation map[string]string + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).DoAndReturn(func(context.Context, []int64, bool, bool) ([]*entity.EvaluatorRecord, error) { + f.token("rq2") + afterMutation = f.snapshot() + return []*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil + }).AnyTimes() + _, err := svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, err) + require.Equal(t, afterMutation, f.snapshot(), "old prepared record must not overwrite new request") +} + +func TestResultPublishDirectNoOldScore(t *testing.T) { + for _, kind := range []string{"no_score", "read_error", "missing_record"} { + t.Run(kind, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + before := f.snapshot() + var result []*entity.EvaluatorRecord + var fault error + switch kind { + case "read_error": + fault = errors.New("evaluator storage unavailable") + case "no_score": + result = []*entity.EvaluatorRecord{publicationRecord(f, nil)} + } + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return(result, fault).AnyTimes() + _, err := svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + if kind != "no_score" { + require.Error(t, err) + require.Equal(t, before, f.snapshot()) + } else { + require.NoError(t, err) + var turn model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&turn).Error) + assert.Nil(t, turn.WeightedScore, "no new score writes SQL NULL, never old77") + } + }) + } +} + +func TestResultPublishSchedulerStaleNoNotification(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + ctrl := gomock.NewController(t) + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 70, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 71}} + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, EvaluationSetID: f.base + 70}}, nil).AnyTimes() + publisher := &stubItemCompletePublisher{} + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + refsService := &publicationSchedulerResult{ExptResultServiceImpl: svc} + eventPublisher := eventmocks.NewMockExptEventPublisher(ctrl) + eventPublisher.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + // Preparing from source rq1 must not send a success notification after rq2 changes it. + var afterMutation map[string]string + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).DoAndReturn(func(context.Context, []int64, bool, bool) ([]*entity.EvaluatorRecord, error) { + f.token("rq2") + afterMutation = f.snapshot() + return []*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil + }).AnyTimes() + scheduler := &ExptSchedulerImpl{ResultSvc: refsService, ExptItemResultRepo: f.itemRepo(), itemCompletePublisher: publisher, evaluationSetItemService: dataset, Publisher: eventPublisher} + err := scheduler.recordEvalItemRunLogs(context.Background(), &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run}, []*entity.ExptEvalItem{{ItemID: f.item, State: entity.ItemRunState_Success, RequestID: "rq1"}}, mode, expt) + require.NoError(t, err) + require.Empty(t, publisher.events, "stale prepared result cannot publish success") + require.Equal(t, afterMutation, f.snapshot()) +} + +// Publication stays real; the unrelated filter projection is outside task4.1. +type publicationSchedulerResult struct{ *ExptResultServiceImpl } + +func (*publicationSchedulerResult) UpsertExptTurnResultFilter(context.Context, int64, int64, []int64) error { + return nil +} + +func TestResultPublishPreparedRealWritepoints(t *testing.T) { + for _, point := range []string{"delete_ref", "insert_ref", "turn", "item", "resulted", "stats"} { + t.Run(point, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil) + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + publication, err := svc.PrepareItemResultPublication(context.Background(), source, expt) + require.NoError(t, err) + before := f.snapshot() + fault := errors.New("fault " + point) + hits := 0 + hook := "publication_point_" + point + callback := func(tx *gorm.DB) { + table := tx.Statement.Table + if point == "delete_ref" && table == "expt_turn_evaluator_result_ref" || point == "insert_ref" && table == "expt_turn_evaluator_result_ref" || point == "turn" && table == "expt_turn_result" || point == "item" && table == "expt_item_result" || point == "resulted" && table == "expt_item_result_run_log" || point == "stats" && table == "expt_stats" { + hits++ + _ = tx.AddError(fault) + } + } + switch point { + case "delete_ref": + require.NoError(t, f.conn.Callback().Delete().Before("gorm:delete").Register(hook, callback)) + case "insert_ref": + require.NoError(t, f.conn.Callback().Create().Before("gorm:create").Register(hook, callback)) + default: + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(hook, callback)) + } + applied, err := svc.PublishPreparedItemResult(context.Background(), publication, nil) + switch point { + case "delete_ref": + require.NoError(t, f.conn.Callback().Delete().Remove(hook)) + case "insert_ref": + require.NoError(t, f.conn.Callback().Create().Remove(hook)) + default: + require.NoError(t, f.conn.Callback().Update().Remove(hook)) + } + require.Positive(t, hits) + require.ErrorIs(t, err, fault) + require.False(t, applied) + require.Equal(t, before, f.snapshot(), "all eleven tables including receipt must roll back") + }) + } +} + +func TestResultPublishCurrentOnceAndFirstFailure(t *testing.T) { + for _, kind := range []string{"canonical_processing", "canonical_success", "first_failed", "registered_inline", "old_map"} { + t.Run(kind, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + if kind == "canonical_success" { + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Success)).Error) + require.NoError(t, f.conn.Model(&model.ExptStats{}).Where("space_id=?", f.space).Updates(map[string]any{"processing_cnt": 1, "success_cnt": 1}).Error) + } + switch kind { + case "first_failed": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Fail)).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.TurnRunState_Fail), "evaluator_result_ids": []byte(`null`)}).Error) + require.NoError(t, f.conn.Create(&model.ExptTurnResult{ID: f.base + 420, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: f.base + 22, Status: int32(entity.TurnRunState_Processing), WeightedScore: gptr.Of(88.0)}).Error) + require.NoError(t, f.conn.Create(&model.ExptTurnResultRunLog{ID: f.base + 520, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: f.base + 22, Status: int32(entity.TurnRunState_Queueing)}).Error) + case "old_map": + raw, _ := json.Marshal(&entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{f.base + 51: f.base + 61}}) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("evaluator_result_ids", raw).Error) + } + result := publicationRecord(f, gptr.Of(0.0)) + if kind == "old_map" { + result.Alias = "" + } + if kind == "registered_inline" { + raw, _ := json.Marshal(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: f.base + 61}}, Inline: []*entity.InlineEvalResult{{InlineKey: "inline", RecordID: f.base + 62}}}) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("evaluator_result_ids", raw).Error) + inline := *result + inline.ID = f.base + 62 + inline.SourceType = entity.EvaluatorRecordSourceTypeInline + inline.InlineKey = "inline" + inline.EvaluatorVersionID = 0 + inline.Alias = "" + inline.EvaluatorOutputData = &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(100.0)}} + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{result, &inline}, nil).Times(1) + } else if kind != "first_failed" { + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{result}, nil).Times(1) + } + before := f.snapshot() + refs, err := svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, err) + stats, err := svc.ExptStatsRepo.Get(context.Background(), f.expt, f.space) + require.NoError(t, err) + require.Equal(t, int32(1), stats.ProcessingItemCnt) + if kind == "first_failed" { + require.Equal(t, int32(1), stats.FailItemCnt) + } else { + require.Equal(t, int32(1), stats.SuccessItemCnt) + } + after := f.snapshot() + for _, table := range []string{"experiment", "expt_run_log", "eval_target_record", "evaluator_record", "expt_item_ref", "expt_turn_result_run_log"} { + require.Equal(t, before[table], after[table], table) + } + turns, err := svc.ExptItemResultRepo.GetItemTurnResults(context.Background(), f.space, f.expt, f.item) + require.NoError(t, err) + for _, turn := range turns { + if kind == "first_failed" { + require.Nil(t, turn.WeightedScore) + } else { + require.Equal(t, 0.0, gptr.Indirect(turn.WeightedScore)) + } + } + if kind == "registered_inline" { + require.Len(t, refs, 2) + var inline bool + for _, ref := range refs { + inline = inline || ref.InlineKey == "inline" + } + require.True(t, inline) + } + _, err = svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, err) + require.Equal(t, after, f.snapshot(), "Resulted repetition must make no writes, records rereads or counters") + }) + } +} + +func TestResultPublishPreparedSourceChange(t *testing.T) { + for _, kind := range []string{"request", "targetref", "evaluatorref", "canonical", "missing_turn", "corrupt_json", "cancelled"} { + t.Run(kind, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil) + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + publication, err := svc.PrepareItemResultPublication(context.Background(), source, expt) + require.NoError(t, err) + ctx := context.Background() + switch kind { + case "request": + f.token("rq2") + case "targetref": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("target_result_id", f.base+42).Error) + case "evaluatorref": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("evaluator_result_ids", []byte(`{"registered":[{"version_id":51,"alias":"new","record_id":62}]}`)).Error) + case "canonical": + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=?", f.space).Update("weighted_score", 66).Error) + case "missing_turn": + require.NoError(t, f.conn.Where("space_id=?", f.space).Delete(&model.ExptTurnResultRunLog{}).Error) + case "corrupt_json": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("evaluator_result_ids", []byte(`{"registered":`)).Error) + case "cancelled": + child, cancel := context.WithCancel(ctx) + cancel() + ctx = child + } + before := f.snapshot() + sent := false + applied, err := svc.PublishPreparedItemResult(ctx, publication, func(context.Context, *entity.ExptItemResultPublishSource) error { sent = true; return nil }) + if kind == "missing_turn" || kind == "corrupt_json" || kind == "cancelled" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.False(t, applied) + require.False(t, sent) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestResultPublishCanonicalTerminalAbsorbsLateLog(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Terminal)).Error) + before := f.snapshot() + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil).AnyTimes() + _, err := svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, err) + require.Equal(t, before, f.snapshot(), "late successful log cannot reverse canonical Terminal") +} + +func TestResultPublishSchedulerSendCommitRetry(t *testing.T) { + for _, kind := range []string{"send_failure", "commit_failure", "mode_failure"} { + t.Run(kind, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + ctrl := gomock.NewController(t) + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 70, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 71}} + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil).AnyTimes() + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, EvaluationSetID: f.base + 70}}, nil).AnyTimes() + notify := &stubItemCompletePublisher{} + fault := errors.New("publish source fault") + if kind == "send_failure" { + notify.err = fault + } + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + modeErr := error(nil) + if kind == "mode_failure" { + modeErr = fault + } + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(modeErr).Times(1) + events := eventmocks.NewMockExptEventPublisher(ctrl) + events.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + scheduler := &ExptSchedulerImpl{ResultSvc: &publicationSchedulerResult{svc}, ExptItemResultRepo: f.itemRepo(), itemCompletePublisher: notify, evaluationSetItemService: dataset, Publisher: events} + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run} + item := &entity.ExptEvalItem{ItemID: f.item, State: entity.ItemRunState_Success, RequestID: "rq1", Publication: source} + before := f.snapshot() + if kind == "commit_failure" { + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("send_commit_fault", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_result_run_log" { + _ = tx.AddError(fault) + } + })) + } + err = scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt) + if kind == "commit_failure" { + require.NoError(t, f.conn.Callback().Update().Remove("send_commit_fault")) + } + if kind != "mode_failure" { + require.ErrorIs(t, err, fault) + require.Equal(t, before, f.snapshot()) + notify.err = nil + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt)) + require.Len(t, notify.events, 2) + } else { + require.NoError(t, err) + require.Len(t, notify.events, 1) + } + after := f.snapshot() + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, []*entity.ExptEvalItem{item}, mode, expt)) + require.Equal(t, after, f.snapshot()) + require.Len(t, notify.events, map[bool]int{true: 1, false: 2}[kind == "mode_failure"], "duplicate source after Resulted must not notify") + }) + } +} + +func TestResultPublishLockCancellationNoWrites(t *testing.T) { + for _, kind := range []string{"cancelled", "lost_lease", "busy"} { + t.Run(kind, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil) + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + p, err := svc.PrepareItemResultPublication(context.Background(), source, expt) + require.NoError(t, err) + ctx := context.Background() + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + if kind == "cancelled" { + child, cancel := context.WithCancel(ctx) + cancel() + ctx = child + } else { + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ string, _ time.Duration, _ time.Duration) (bool, context.Context, func(), error) { + if kind == "busy" { + return false, ctx, func() {}, nil + } + child, cancel := context.WithCancel(ctx) + cancel() + return true, child, cancel, nil + }) + if kind != "busy" { + locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + } + } + svc.resultPublishLocker = func() lock.ILocker { return locker } + before := f.snapshot() + sent := false + applied, err := svc.PublishPreparedItemResult(ctx, p, func(context.Context, *entity.ExptItemResultPublishSource) error { sent = true; return nil }) + require.True(t, errno.NeedMQRetry(err)) + require.False(t, applied) + require.False(t, sent) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestResultPublishRealRepoMissingPortDoesNotHalfWrite(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil).AnyTimes() + svc.resultPublishRepo = nil + before := f.snapshot() + _, err := svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.True(t, errno.NeedMQRetry(err), "real repository without publication wiring must fail closed") + require.Equal(t, before, f.snapshot()) +} + +func TestResultPublishSQLConcurrentCommitOnce(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil) + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + p, err := svc.PrepareItemResultPublication(context.Background(), source, expt) + require.NoError(t, err) + start := make(chan struct{}) + var count atomic.Int32 + var wg sync.WaitGroup + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + applied, err := svc.resultPublishRepo.CommitItemResult(context.Background(), p) + if applied { + count.Add(1) + } + errs <- err + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + require.Equal(t, int32(1), count.Load()) + stats, err := svc.ExptStatsRepo.Get(context.Background(), f.expt, f.space) + require.NoError(t, err) + require.Equal(t, int32(1), stats.SuccessItemCnt) + require.Equal(t, int32(1), stats.ProcessingItemCnt) + var refs []model.ExptTurnEvaluatorResultRef + require.NoError(t, f.conn.Where("space_id=?", f.space).Find(&refs).Error) + require.Len(t, refs, 1) +} + +func TestResultPublishScanFreezesNotificationSource(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), f.space).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).AnyTimes() + expt.EvalSet = &entity.EvaluationSet{ID: f.base + 70, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 71}} + scan := &exptBaseExec{exptItemResultRepo: f.itemRepo(), configer: config} + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run} + _, _, items, err := scan.ScanEvalItems(context.Background(), event, expt) + require.NoError(t, err) + require.Len(t, items, 1) + require.True(t, items[0].PublicationObserved) + require.NotNil(t, items[0].Publication) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{publicationRecord(f, gptr.Of(0.0))}, nil) + // Same request, new invocation references after the original primary scan. + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("target_result_id", f.base+42).Error) + before := f.snapshot() + notify := &stubItemCompletePublisher{} + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item}}, nil) + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + scheduler := &ExptSchedulerImpl{ResultSvc: &publicationSchedulerResult{svc}, ExptItemResultRepo: f.itemRepo(), itemCompletePublisher: notify, evaluationSetItemService: dataset} + require.NoError(t, scheduler.recordEvalItemRunLogs(context.Background(), event, items, mode, expt)) + require.Empty(t, notify.events) + require.Equal(t, before, f.snapshot()) +} + +func TestResultPublishFailedEvaluatorCannotPublishScore(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + record := publicationRecord(f, gptr.Of(99.0)) + record.Status = entity.EvaluatorRunStatusFail + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{record}, nil) + _, err := svc.RecordItemRunLogs(context.Background(), f.expt, f.run, f.item, f.space, expt) + require.NoError(t, err) + var turn model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&turn).Error) + require.Nil(t, turn.WeightedScore, "failed evaluator's score is not a current successful result") +} + +func TestResultPublishConfigSourceFrozenAndReadErrors(t *testing.T) { + for _, kind := range []string{"config_change", "read_error", "missing_record", "space_mismatch", "unknown_status"} { + t.Run(kind, func(t *testing.T) { + f, svc, records, expt := publicationFixture(t) + record := publicationRecord(f, gptr.Of(0.0)) + if kind == "space_mismatch" { + record.SpaceID = 0 + } + if kind == "unknown_status" { + record.Status = entity.EvaluatorRunStatusUnknown + } + source, err := svc.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + if kind == "read_error" || kind == "missing_record" || kind == "space_mismatch" || kind == "unknown_status" { + fault := error(nil) + result := []*entity.EvaluatorRecord{record} + if kind == "read_error" { + fault = errors.New("record source read failure") + } + if kind == "missing_record" { + result = nil + } + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return(result, fault) + before := f.snapshot() + _, err = svc.PrepareItemResultPublication(context.Background(), source, expt) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, before, f.snapshot()) + return + } + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return([]*entity.EvaluatorRecord{record}, nil) + p, err := svc.PrepareItemResultPublication(context.Background(), source, expt) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&model.ExptItemRef{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_config", []byte(`{"target_source_space_id":99}`)).Error) + before := f.snapshot() + sent := false + ok, err := svc.PublishPreparedItemResult(context.Background(), p, func(context.Context, *entity.ExptItemResultPublishSource) error { sent = true; return nil }) + require.NoError(t, err) + require.False(t, ok) + require.False(t, sent) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestResultPublishPreparationStopsAfterCancelledRecordRead(t *testing.T) { + f, svc, _, expt := publicationFixture(t) + ctrl := gomock.NewController(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + source, err := svc.GetItemResultPublishSource(ctx, f.space, f.expt, f.run, f.item) + require.NoError(t, err) + target := svcmocks.NewMockIEvalTargetService(ctrl) + evaluators := svcmocks.NewMockEvaluatorRecordService(ctrl) + target.EXPECT().GetRecordByID(gomock.Any(), f.space, f.base+41).DoAndReturn(func(context.Context, int64, int64) (*entity.EvalTargetRecord, error) { + cancel() + return &entity.EvalTargetRecord{ID: f.base + 41, SpaceID: f.space, ExperimentRunID: f.run, ItemID: f.item, TurnID: 0, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil + }) + evaluators.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + svc.evalTargetService = target + svc.evaluatorRecordService = evaluators + before := f.snapshot() + _, err = svc.PrepareItemResultPublication(ctx, source, expt) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, before, f.snapshot()) +} diff --git a/backend/modules/evaluation/domain/service/expt_result_publish_unit_test.go b/backend/modules/evaluation/domain/service/expt_result_publish_unit_test.go new file mode 100644 index 0000000000..aeba9ca45c --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_result_publish_unit_test.go @@ -0,0 +1,1112 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + entitymocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity/mocks" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func publishUnitTurn(turnID int64) *entity.ExptTurnResult { + return &entity.ExptTurnResult{ID: 100, SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: turnID} +} + +func publishUnitLog(turnID int64) *entity.ExptTurnResultRunLog { + return &entity.ExptTurnResultRunLog{ID: 101, SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: turnID, Status: entity.TurnRunState_Success} +} + +func publishUnitSource() *entity.ExptItemResultPublishSource { + turn := publishUnitTurn(0) + log := publishUnitLog(0) + source := &entity.ExptCurrentItemRunSource{ + Current: entity.ExptItemCurrentRequest{ + SpaceID: 3, ExptID: 1, RunID: 2, ItemID: 9, ItemVersionID: 7, + RequestID: "req", Mode: entity.EvaluationModeFailRetry, Status: entity.ItemRunState_Success, + }, + CanonicalTurns: []*entity.ExptTurnResult{turn}, + Turns: map[int64]*entity.ExptTurnResultRunLog{0: log}, + } + return &entity.ExptItemResultPublishSource{ + Source: source, + Item: &entity.ExptItemResult{ID: 1, SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, Status: entity.ItemRunState_Success}, + ItemLog: &entity.ExptItemResultRunLog{ItemID: 9, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, + } +} + +func publishUnitExperiment() *entity.Experiment { + return &entity.Experiment{ID: 1, SpaceID: 3} +} + +type publishUnitDeps struct { + ctrl *gomock.Controller + repo *repomocks.MockIExptResultPublishRepo + locker *lockmocks.MockILocker + targets *svcmocks.MockIEvalTargetService + records *svcmocks.MockEvaluatorRecordService + idgen *idmocks.MockIIDGenerator + svc *ExptResultServiceImpl +} + +func newPublishUnitDeps(t *testing.T) *publishUnitDeps { + t.Helper() + ctrl := gomock.NewController(t) + d := &publishUnitDeps{ + ctrl: ctrl, + repo: repomocks.NewMockIExptResultPublishRepo(ctrl), + locker: lockmocks.NewMockILocker(ctrl), + targets: svcmocks.NewMockIEvalTargetService(ctrl), + records: svcmocks.NewMockEvaluatorRecordService(ctrl), + idgen: idmocks.NewMockIIDGenerator(ctrl), + } + d.svc = &ExptResultServiceImpl{ + resultPublishRepo: d.repo, + resultPublishLocker: func() lock.ILocker { return d.locker }, + evalTargetService: d.targets, + evaluatorRecordService: d.records, + idgen: d.idgen, + } + return d +} + +func TestPublicationReadyUnit(t *testing.T) { + require.False(t, publicationReady(nil)) + require.False(t, publicationReady(&entity.ExptItemResultPublishSource{})) + ready := publishUnitSource() + require.True(t, publicationReady(ready)) + + notLogged := publishUnitSource() + notLogged.ItemLog.ResultState = int32(entity.ExptItemResultStateResulted) + require.False(t, publicationReady(notLogged)) + + unfinished := publishUnitSource() + unfinished.ItemLog.Status = int32(entity.ItemRunState_Processing) + require.False(t, publicationReady(unfinished)) +} + +func TestGetItemResultPublishSourceUnit(t *testing.T) { + t.Run("cancelled", func(t *testing.T) { + svc := &ExptResultServiceImpl{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := svc.GetItemResultPublishSource(ctx, 3, 1, 2, 9) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("repo_missing", func(t *testing.T) { + svc := &ExptResultServiceImpl{} + _, err := svc.GetItemResultPublishSource(context.Background(), 3, 1, 2, 9) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("repo_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.repo.EXPECT().GetItemResultPublishSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, errors.New("read")) + _, err := d.svc.GetItemResultPublishSource(context.Background(), 3, 1, 2, 9) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("success", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + d.repo.EXPECT().GetItemResultPublishSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(source, nil) + got, err := d.svc.GetItemResultPublishSource(context.Background(), 3, 1, 2, 9) + require.NoError(t, err) + require.Same(t, source, got) + }) +} + +func TestPrepareItemResultPublicationUnit(t *testing.T) { + t.Run("cancelled", func(t *testing.T) { + d := newPublishUnitDeps(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := d.svc.PrepareItemResultPublication(ctx, publishUnitSource(), publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("not_ready", func(t *testing.T) { + d := newPublishUnitDeps(t) + publication, err := d.svc.PrepareItemResultPublication(context.Background(), &entity.ExptItemResultPublishSource{}, publishUnitExperiment()) + require.NoError(t, err) + require.Nil(t, publication) + }) + t.Run("source_incomplete", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Item = nil + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + + source = publishUnitSource() + source.Source.CanonicalTurns = nil + _, err = d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("experiment_mismatch", func(t *testing.T) { + d := newPublishUnitDeps(t) + _, err := d.svc.PrepareItemResultPublication(context.Background(), publishUnitSource(), nil) + require.True(t, errno.NeedMQRetry(err)) + + _, err = d.svc.PrepareItemResultPublication(context.Background(), publishUnitSource(), &entity.Experiment{ID: 9, SpaceID: 3}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("canonical_turn_missing", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.CanonicalTurns = []*entity.ExptTurnResult{nil} + source.Source.Turns = map[int64]*entity.ExptTurnResultRunLog{0: publishUnitLog(0)} + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("frozen_turn_missing", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns = map[int64]*entity.ExptTurnResultRunLog{} + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("multiset_config_missing", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].TargetResultID = 88 + expt := publishUnitExperiment() + expt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, expt) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_reader_missing", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.svc.evalTargetService = nil + source := publishUnitSource() + source.Source.Turns[0].TargetResultID = 88 + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_read_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].TargetResultID = 88 + d.targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(88)).Return(nil, errors.New("read")) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_record_foreign", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].TargetResultID = 88 + d.targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(88)).Return(&entity.EvalTargetRecord{ID: 88, SpaceID: 3, ItemID: 99, TurnID: 0, ExperimentRunID: 2, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_record_valid", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].TargetResultID = 88 + d.targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(88)).Return(&entity.EvalTargetRecord{ID: 88, SpaceID: 3, ItemID: 9, TurnID: 0, ExperimentRunID: 2, ItemVersionID: 7, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil) + publication, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.NoError(t, err) + require.Len(t, publication.Turns, 1) + require.Equal(t, int64(88), publication.Turns[0].TargetResultID) + }) + t.Run("registered_ref_valid", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{61}, false, false).Return([]*entity.EvaluatorRecord{{ + ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, + EvaluatorVersionID: 51, Alias: "a", SourceType: entity.EvaluatorRecordSourceTypeBuiltin, Status: entity.EvaluatorRunStatusSuccess, + EvaluatorOutputData: &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(0.5)}}, + }}, nil) + d.idgen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{900}, nil) + publication, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.NoError(t, err) + require.Len(t, publication.Refs, 1) + require.Equal(t, int64(900), publication.Refs[0].ID) + require.NotNil(t, publication.Turns[0].WeightedScore) + }) + t.Run("evaluator_reader_missing", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.svc.evaluatorRecordService = nil + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_read_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("read")) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_record_missing", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_record_duplicated", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + record := &entity.EvaluatorRecord{ID: 61} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.EvaluatorRecord{record, record}, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("inline_source_mismatch", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "k", RecordID: 62}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{62}, false, false).Return([]*entity.EvaluatorRecord{{ + ID: 62, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, + SourceType: entity.EvaluatorRecordSourceTypeInline, InlineKey: "other", Status: entity.EvaluatorRunStatusSuccess, + }}, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("registered_source_mismatch", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.EvaluatorRecord{{ + ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, + EvaluatorVersionID: 52, Alias: "a", SourceType: entity.EvaluatorRecordSourceTypeBuiltin, Status: entity.EvaluatorRunStatusSuccess, + }}, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("existing_ref_reused", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + source.Source.CurrentRefs = []*entity.ExptTurnEvaluatorResultRef{{ID: 700, ExptTurnResultID: 100, EvaluatorVersionID: 51, EvaluatorResultID: 61, Alias: "a", SourceType: int32(entity.EvaluatorRecordSourceTypeBuiltin)}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.EvaluatorRecord{{ + ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, + EvaluatorVersionID: 51, Alias: "a", SourceType: entity.EvaluatorRecordSourceTypeBuiltin, Status: entity.EvaluatorRunStatusSuccess, + }}, nil) + publication, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.NoError(t, err) + require.Len(t, publication.Refs, 1) + require.Equal(t, int64(700), publication.Refs[0].ID) + }) + t.Run("idgen_missing", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.svc.idgen = nil + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.EvaluatorRecord{{ + ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, + EvaluatorVersionID: 51, Alias: "a", SourceType: entity.EvaluatorRecordSourceTypeBuiltin, Status: entity.EvaluatorRunStatusSuccess, + }}, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("idgen_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.EvaluatorRecord{{ + ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, + EvaluatorVersionID: 51, Alias: "a", SourceType: entity.EvaluatorRecordSourceTypeBuiltin, Status: entity.EvaluatorRunStatusSuccess, + }}, nil) + d.idgen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return(nil, errors.New("idgen")) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("idgen_incomplete", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.EvaluatorRecord{{ + ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, + EvaluatorVersionID: 51, Alias: "a", SourceType: entity.EvaluatorRecordSourceTypeBuiltin, Status: entity.EvaluatorRunStatusSuccess, + }}, nil) + d.idgen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{}, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, publishUnitExperiment()) + require.True(t, errno.NeedMQRetry(err)) + }) +} + +func TestPublishPreparedItemResultUnit(t *testing.T) { + t.Run("cancelled", func(t *testing.T) { + d := newPublishUnitDeps(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := d.svc.PublishPreparedItemResult(ctx, &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("nil_publication", func(t *testing.T) { + d := newPublishUnitDeps(t) + applied, err := d.svc.PublishPreparedItemResult(context.Background(), nil, nil) + require.NoError(t, err) + require.False(t, applied) + }) + t.Run("invalid_source", func(t *testing.T) { + d := newPublishUnitDeps(t) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: &entity.ExptItemResultPublishSource{}}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("deps_missing", func(t *testing.T) { + svc := &ExptResultServiceImpl{} + _, err := svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("locker_nil", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.svc.resultPublishLocker = func() lock.ILocker { return nil } + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("lock_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("lock_busy", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("source_check_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.repo.EXPECT().IsItemResultPublishSourceCurrent(gomock.Any(), gomock.Any()).Return(false, errors.New("check")) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("not_current", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(false, errors.New("unlock")) + d.repo.EXPECT().IsItemResultPublishSourceCurrent(gomock.Any(), gomock.Any()).Return(false, nil) + applied, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.NoError(t, err) + require.False(t, applied) + }) + t.Run("before_commit_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.repo.EXPECT().IsItemResultPublishSourceCurrent(gomock.Any(), gomock.Any()).Return(true, nil) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, func(context.Context, *entity.ExptItemResultPublishSource) error { + return errors.New("before") + }) + require.Error(t, err) + }) + t.Run("commit_error", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.repo.EXPECT().IsItemResultPublishSourceCurrent(gomock.Any(), gomock.Any()).Return(true, nil) + d.repo.EXPECT().CommitItemResult(gomock.Any(), gomock.Any()).Return(false, errors.New("commit")) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("success", func(t *testing.T) { + d := newPublishUnitDeps(t) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.repo.EXPECT().IsItemResultPublishSourceCurrent(gomock.Any(), gomock.Any()).Return(true, nil) + d.repo.EXPECT().CommitItemResult(gomock.Any(), gomock.Any()).Return(true, nil) + applied, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, func(context.Context, *entity.ExptItemResultPublishSource) error { return nil }) + require.NoError(t, err) + require.True(t, applied) + }) +} + +func TestRecordCurrentItemResultsUnit(t *testing.T) { + event := func() *entity.ExptScheduleEvent { + return &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + } + expt := publishUnitExperiment() + readyItem := func() *entity.ExptEvalItem { + source := publishUnitSource() + return &entity.ExptEvalItem{ItemID: 9, RequestID: "req", Publication: source, State: entity.ItemRunState_Success} + } + newDeps := func(t *testing.T) (*publishUnitDeps, *entitymocks.MockExptSchedulerMode, *ExptSchedulerImpl) { + d := newPublishUnitDeps(t) + mode := entitymocks.NewMockExptSchedulerMode(d.ctrl) + scheduler := &ExptSchedulerImpl{} + return d, mode, scheduler + } + + t.Run("cancelled", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, errno.NeedMQRetry(scheduler.recordCurrentItemResults(ctx, event(), nil, mode, expt, nil))) + }) + t.Run("nil_item", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + require.True(t, errno.NeedMQRetry(scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{nil}, mode, expt, nil))) + }) + t.Run("observed_without_source", func(t *testing.T) { + d, mode, scheduler := newDeps(t) + _ = d + item := &entity.ExptEvalItem{ItemID: 9, PublicationObserved: true} + require.NoError(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, nil)) + }) + t.Run("legacy_source_error", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(gomock.NewController(t)) + svc.EXPECT().GetItemResultPublishSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, errors.New("read")) + item := &entity.ExptEvalItem{ItemID: 9, RequestID: "req", State: entity.ItemRunState_Success} + require.Error(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) + t.Run("legacy_source_mismatch", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(gomock.NewController(t)) + source := publishUnitSource() + source.Source.Current.RequestID = "other" + svc.EXPECT().GetItemResultPublishSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + item := &entity.ExptEvalItem{ItemID: 9, RequestID: "req", State: entity.ItemRunState_Success} + require.NoError(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) + t.Run("not_ready", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(gomock.NewController(t)) + source := publishUnitSource() + source.ItemLog.Status = int32(entity.ItemRunState_Processing) + item := &entity.ExptEvalItem{ItemID: 9, RequestID: "req", State: entity.ItemRunState_Processing, Publication: source} + require.NoError(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) + t.Run("scope_mismatch", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(gomock.NewController(t)) + source := publishUnitSource() + source.Source.Current.SpaceID = 99 + item := &entity.ExptEvalItem{ItemID: 9, RequestID: "req", State: entity.ItemRunState_Success, Publication: source} + require.True(t, errno.NeedMQRetry(scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc))) + }) + t.Run("prepare_error", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(gomock.NewController(t)) + item := readyItem() + svc.EXPECT().PrepareItemResultPublication(gomock.Any(), item.Publication, expt).Return(nil, errors.New("prepare")) + require.Error(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) + t.Run("not_applied", func(t *testing.T) { + _, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(gomock.NewController(t)) + item := readyItem() + svc.EXPECT().PrepareItemResultPublication(gomock.Any(), item.Publication, expt).Return(&entity.ExptItemResultPublication{Source: item.Publication}, nil) + svc.EXPECT().PublishPreparedItemResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil) + require.NoError(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) + t.Run("applied_publish_result_error_is_logged", func(t *testing.T) { + d, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(d.ctrl) + results := svcmocks.NewMockExptResultService(d.ctrl) + publisher := eventmocks.NewMockExptEventPublisher(d.ctrl) + scheduler.ResultSvc = results + scheduler.Publisher = publisher + item := readyItem() + publication := &entity.ExptItemResultPublication{Source: item.Publication} + svc.EXPECT().PrepareItemResultPublication(gomock.Any(), item.Publication, expt).Return(publication, nil) + svc.EXPECT().PublishPreparedItemResult(gomock.Any(), publication, gomock.Any()).Return(true, nil) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("publish")) + results.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), []int64{9}).Return(errors.New("upsert")) + publisher.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + require.NoError(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) + t.Run("applied_publisher_error", func(t *testing.T) { + d, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(d.ctrl) + publisher := eventmocks.NewMockExptEventPublisher(d.ctrl) + results := svcmocks.NewMockExptResultService(d.ctrl) + scheduler.ResultSvc = results + scheduler.Publisher = publisher + item := readyItem() + publication := &entity.ExptItemResultPublication{Source: item.Publication} + svc.EXPECT().PrepareItemResultPublication(gomock.Any(), item.Publication, expt).Return(publication, nil) + svc.EXPECT().PublishPreparedItemResult(gomock.Any(), publication, gomock.Any()).Return(true, nil) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + results.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), []int64{9}).Return(nil) + publisher.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("publish")) + require.Error(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) + t.Run("applied_without_publisher", func(t *testing.T) { + d, mode, scheduler := newDeps(t) + svc := svcmocks.NewMockExptResultPublicationService(d.ctrl) + results := svcmocks.NewMockExptResultService(d.ctrl) + scheduler.ResultSvc = results + item := readyItem() + publication := &entity.ExptItemResultPublication{Source: item.Publication} + svc.EXPECT().PrepareItemResultPublication(gomock.Any(), item.Publication, expt).Return(publication, nil) + svc.EXPECT().PublishPreparedItemResult(gomock.Any(), publication, gomock.Any()).Return(true, nil) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + results.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), []int64{9}).Return(nil) + require.NoError(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, mode, expt, svc)) + }) +} + +var _ = time.Now + +type filterManualRepo struct { + *repomocks.MockIExptItemResultRepo + *repomocks.MockIExptManualRetryFilterRepo +} + +type materializedFilterRepo struct { + *repomocks.MockIExptTurnResultFilterRepo + *repomocks.MockIExptMaterializedFilterRepo +} + +func (m *materializedFilterRepo) PersistsMaterializedItemVersion() bool { return true } + +func TestFilterPureHelpersUnit(t *testing.T) { + require.Equal(t, []int64{1, 2, 3}, sortedFilterItems([]int64{3, 1, 2, 1})) + zero := 0.0 + require.True(t, filterWeightedScoreEqual(nil, nil)) + require.True(t, filterWeightedScoreEqual(&zero, nil)) + require.False(t, filterWeightedScoreEqual(nil, &zero)) + one := 1.0 + require.True(t, filterWeightedScoreEqual(&one, &one)) +} + +func TestWithFilterItemLockUnit(t *testing.T) { + ctx := context.Background() + t.Run("cancelled", func(t *testing.T) { + svc := &ExptResultServiceImpl{} + cancelled, cancel := context.WithCancel(ctx) + cancel() + require.True(t, errno.NeedMQRetry(svc.withFilterItemLock(cancelled, 1, 9, func(context.Context) error { return nil }))) + }) + t.Run("current_repo_without_locker", func(t *testing.T) { + ctrl := gomock.NewController(t) + svc := &ExptResultServiceImpl{ExptItemResultRepo: ¤tUnitRepo{MockIExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl), MockIExptCurrentResultRepo: repomocks.NewMockIExptCurrentResultRepo(ctrl)}} + require.True(t, errno.NeedMQRetry(svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { return nil }))) + }) + t.Run("legacy_repo_runs_without_lock", func(t *testing.T) { + ctrl := gomock.NewController(t) + svc := &ExptResultServiceImpl{ExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl)} + called := false + require.NoError(t, svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { called = true; return nil })) + require.True(t, called) + }) + t.Run("nil_locker", func(t *testing.T) { + svc := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return nil }} + require.True(t, errno.NeedMQRetry(svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { return nil }))) + }) + t.Run("lock_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:1:9", gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + svc := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { return nil }))) + }) + t.Run("busy", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, ctx, func() {}, nil) + svc := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { return nil }))) + }) + t.Run("lock_ctx_cancelled", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + cancelled, cancel := context.WithCancel(ctx) + cancel() + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, cancelled, func() {}, nil) + locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + svc := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { return nil }))) + }) + t.Run("fn_error_and_success", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, ctx, func() {}, nil).Times(2) + locker.EXPECT().Unlock(gomock.Any()).Return(true, nil).Times(2) + svc := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { return errors.New("fn") }))) + require.NoError(t, svc.withFilterItemLock(ctx, 1, 9, func(context.Context) error { return nil })) + }) +} + +func TestFilterCandidateItemsUnit(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + svc := ExptResultServiceImpl{ExptItemResultRepo: items} + got, err := svc.filterCandidateItems(context.Background(), 3, 1, []int64{9, 9, 4}) + require.NoError(t, err) + require.Equal(t, []int64{4, 9}, got) + + items.EXPECT().ListItemResultsByExptID(gomock.Any(), int64(1), int64(3), gomock.Any(), false).Return(nil, int64(0), errors.New("list")) + _, err = svc.filterCandidateItems(context.Background(), 3, 1, nil) + require.Error(t, err) + + items.EXPECT().ListItemResultsByExptID(gomock.Any(), int64(1), int64(3), gomock.Any(), false).Return([]*entity.ExptItemResult{{ItemID: 9}, nil, {ItemID: 4}}, int64(0), nil) + got, err = svc.filterCandidateItems(context.Background(), 3, 1, nil) + require.NoError(t, err) + require.Equal(t, []int64{4, 9}, got) +} + +func TestUpsertExptTurnResultFilterUnit(t *testing.T) { + t.Run("invalid_scope", func(t *testing.T) { + svc := ExptResultServiceImpl{} + require.Error(t, svc.UpsertExptTurnResultFilter(context.Background(), 0, 1, nil)) + require.Error(t, svc.UpsertExptTurnResultFilter(context.Background(), 3, 0, nil)) + }) + t.Run("cancelled", func(t *testing.T) { + svc := ExptResultServiceImpl{} + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, errno.NeedMQRetry(svc.UpsertExptTurnResultFilter(cancelled, 3, 1, nil))) + }) + t.Run("legacy_build_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{9}, gomock.Any(), false).Return(nil, int64(0), errors.New("list")) + svc := ExptResultServiceImpl{ExptItemResultRepo: items, ExptTurnResultRepo: turns} + require.Error(t, svc.UpsertExptTurnResultFilter(context.Background(), 3, 1, []int64{9})) + }) + t.Run("locked_build_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil, int64(0), errors.New("list")) + svc := ExptResultServiceImpl{ExptItemResultRepo: items, ExptTurnResultRepo: turns, resultPublishLocker: func() lock.ILocker { return locker }} + require.Error(t, svc.UpsertExptTurnResultFilter(context.Background(), 3, 1, []int64{9})) + }) + t.Run("cancel_during_item", func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + locker := lockmocks.NewMockILocker(ctrl) + ctx, cancel := context.WithCancel(context.Background()) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ string, _ any, _ any) (bool, context.Context, func(), error) { + cancel() + return true, ctx, func() {}, nil + }) + locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + svc := ExptResultServiceImpl{ExptItemResultRepo: items, resultPublishLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(svc.UpsertExptTurnResultFilter(ctx, 3, 1, []int64{9}))) + }) +} + +func TestCompareExptTurnResultFiltersUnit(t *testing.T) { + t.Run("cancelled", func(t *testing.T) { + svc := ExptResultServiceImpl{} + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, errno.NeedMQRetry(svc.CompareExptTurnResultFilters(cancelled, 3, 1, nil, 0))) + }) + t.Run("candidate_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + items.EXPECT().ListItemResultsByExptID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil, int64(0), errors.New("list")) + locker := lockmocks.NewMockILocker(ctrl) + svc := ExptResultServiceImpl{ExptItemResultRepo: items, resultPublishLocker: func() lock.ILocker { return locker }} + require.Error(t, svc.CompareExptTurnResultFilters(context.Background(), 3, 1, nil, 0)) + }) + t.Run("locked_build_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil, int64(0), errors.New("list")) + svc := ExptResultServiceImpl{ExptItemResultRepo: items, ExptTurnResultRepo: turns, resultPublishLocker: func() lock.ILocker { return locker }} + require.Error(t, svc.CompareExptTurnResultFilters(context.Background(), 3, 1, []int64{9}, 0)) + }) +} + +func TestReconcileManualRetryFilterUnit(t *testing.T) { + validManual := `{"run_id":2,"request_id":"req","mode":2}` + newSvc := func(t *testing.T) (*ExptResultServiceImpl, *filterManualRepo, *repomocks.MockIExptResultPublishRepo, *repomocks.MockIExptTurnResultFilterRepo, *repomocks.MockIExptTurnResultRepo) { + ctrl := gomock.NewController(t) + return &ExptResultServiceImpl{}, &filterManualRepo{MockIExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl), MockIExptManualRetryFilterRepo: repomocks.NewMockIExptManualRetryFilterRepo(ctrl)}, repomocks.NewMockIExptResultPublishRepo(ctrl), repomocks.NewMockIExptTurnResultFilterRepo(ctrl), repomocks.NewMockIExptTurnResultRepo(ctrl) + } + t.Run("repo_not_scanner", func(t *testing.T) { + ctrl := gomock.NewController(t) + svc := ExptResultServiceImpl{ExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl)} + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("scan_error", func(t *testing.T) { + svc, items, _, _, _ := newSvc(t) + svc.ExptItemResultRepo = items + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), int64(3), int64(1), int64(2), int64(0), 200).Return(nil, int64(0), errors.New("scan")) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("cursor_stalled", func(t *testing.T) { + svc, items, _, _, _ := newSvc(t) + svc.ExptItemResultRepo = items + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(0), 200).Return([]int64{9}, int64(5), nil) + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(5), 200).Return([]int64{9}, int64(5), nil) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("deps_missing", func(t *testing.T) { + svc, items, _, _, _ := newSvc(t) + svc.ExptItemResultRepo = items + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), 200).Return([]int64{9}, int64(0), nil) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("source_read_error", func(t *testing.T) { + svc, items, pub, filters, _ := newSvc(t) + svc.ExptItemResultRepo = items + svc.resultPublishRepo = pub + svc.exptTurnResultFilterRepo = filters + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), 200).Return([]int64{9}, int64(0), nil) + pub.EXPECT().GetItemResultPublishSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, errors.New("read")) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("source_missing", func(t *testing.T) { + svc, items, pub, filters, _ := newSvc(t) + svc.ExptItemResultRepo = items + svc.resultPublishRepo = pub + svc.exptTurnResultFilterRepo = filters + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), 200).Return([]int64{9}, int64(0), nil) + pub.EXPECT().GetItemResultPublishSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("parse_error", func(t *testing.T) { + svc, items, pub, filters, _ := newSvc(t) + svc.ExptItemResultRepo = items + svc.resultPublishRepo = pub + svc.exptTurnResultFilterRepo = filters + source := publishUnitSource() + source.Item.Ext = map[string]string{entity.ManualRetryExtKey: "{bad"} + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), 200).Return([]int64{9}, int64(0), nil) + pub.EXPECT().GetItemResultPublishSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("request_changed", func(t *testing.T) { + svc, items, pub, filters, _ := newSvc(t) + svc.ExptItemResultRepo = items + svc.resultPublishRepo = pub + svc.exptTurnResultFilterRepo = filters + source := publishUnitSource() + source.Item.Ext = map[string]string{entity.ManualRetryExtKey: validManual} + source.Source.Current.RequestID = "other" + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), 200).Return([]int64{9}, int64(0), nil) + pub.EXPECT().GetItemResultPublishSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("publication_pending", func(t *testing.T) { + svc, items, pub, filters, _ := newSvc(t) + svc.ExptItemResultRepo = items + svc.resultPublishRepo = pub + svc.exptTurnResultFilterRepo = filters + source := publishUnitSource() + source.Item.Ext = map[string]string{entity.ManualRetryExtKey: validManual} + source.Source.Current.RequestID = "req" + source.Item.Status = entity.ItemRunState_Processing + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), 200).Return([]int64{9}, int64(0), nil) + pub.EXPECT().GetItemResultPublishSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("build_error_then_empty", func(t *testing.T) { + for _, tc := range []struct { + name string + setup func(*repomocks.MockIExptTurnResultRepo) + }{ + {"error", func(turns *repomocks.MockIExptTurnResultRepo) { + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil, int64(0), errors.New("list")) + }}, + {"empty", func(turns *repomocks.MockIExptTurnResultRepo) { + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil, int64(0), nil) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + svc, items, pub, filters, turns := newSvc(t) + svc.ExptItemResultRepo = items + svc.resultPublishRepo = pub + svc.exptTurnResultFilterRepo = filters + svc.ExptTurnResultRepo = turns + source := publishUnitSource() + source.Item.Ext = map[string]string{entity.ManualRetryExtKey: validManual} + source.Source.Current.RequestID = "req" + source.ItemLog.ResultState = int32(entity.ExptItemResultStateResulted) + items.MockIExptManualRetryFilterRepo.EXPECT().ScanManualRetryFilterItems(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), 200).Return([]int64{9}, int64(0), nil) + pub.EXPECT().GetItemResultPublishSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(source, nil) + tc.setup(turns) + _, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.True(t, errno.NeedMQRetry(err)) + }) + } + }) +} + +func TestMaterializedFilterCompareUnit(t *testing.T) { + row := func(mutate func(*entity.ExptTurnResultFilterEntity)) *entity.ExptTurnResultFilterEntity { + r := &entity.ExptTurnResultFilterEntity{SpaceID: 3, ExptID: 1, ItemID: 9, ItemVersionID: 7, ItemIdx: 1, TurnID: 0, Status: entity.ItemRunState_Success, EvalTargetData: map[string]string{"actual_output": "out"}, EvaluatorScore: map[string]float64{"a": 1}} + if mutate != nil { + mutate(r) + } + return r + } + expected := func() []*entity.ExptTurnResultFilterEntity { return []*entity.ExptTurnResultFilterEntity{row(nil)} } + ctx := context.Background() + + t.Run("empty_expected", func(t *testing.T) { + var svc ExptResultServiceImpl + ok, err := svc.compareMaterializedFilterLocked(ctx, 3, 1, 9, nil) + require.NoError(t, err) + require.False(t, ok) + }) + t.Run("read_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + filters := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + filters.EXPECT().GetByExptIDItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("read")) + svc := ExptResultServiceImpl{exptTurnResultFilterRepo: filters} + _, err := svc.compareMaterializedFilterLocked(ctx, 3, 1, 9, expected()) + require.Error(t, err) + }) + t.Run("len_mismatch", func(t *testing.T) { + ctrl := gomock.NewController(t) + filters := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + filters.EXPECT().GetByExptIDItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + svc := ExptResultServiceImpl{exptTurnResultFilterRepo: filters} + ok, err := svc.compareMaterializedFilterLocked(ctx, 3, 1, 9, expected()) + require.NoError(t, err) + require.False(t, ok) + }) + t.Run("legacy_partial_read_is_not_equal", func(t *testing.T) { + ctrl := gomock.NewController(t) + filters := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + filters.EXPECT().GetByExptIDItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultFilterEntity{row(nil)}, nil) + svc := ExptResultServiceImpl{exptTurnResultFilterRepo: filters} + ok, err := svc.compareMaterializedFilterLocked(ctx, 3, 1, 9, expected()) + require.NoError(t, err) + require.False(t, ok) + }) + t.Run("full_materialized_match", func(t *testing.T) { + ctrl := gomock.NewController(t) + filters := &materializedFilterRepo{MockIExptTurnResultFilterRepo: repomocks.NewMockIExptTurnResultFilterRepo(ctrl), MockIExptMaterializedFilterRepo: repomocks.NewMockIExptMaterializedFilterRepo(ctrl)} + filters.MockIExptMaterializedFilterRepo.EXPECT().GetMaterializedByExptIDItemIDs(gomock.Any(), "3", "1", gomock.Any(), []string{"9"}).Return([]*entity.ExptTurnResultFilterEntity{row(nil)}, nil) + svc := ExptResultServiceImpl{exptTurnResultFilterRepo: filters} + ok, err := svc.compareMaterializedFilterLocked(ctx, 3, 1, 9, expected()) + require.NoError(t, err) + require.True(t, ok) + }) + t.Run("full_materialized_field_mismatch", func(t *testing.T) { + ctrl := gomock.NewController(t) + filters := &materializedFilterRepo{MockIExptTurnResultFilterRepo: repomocks.NewMockIExptTurnResultFilterRepo(ctrl), MockIExptMaterializedFilterRepo: repomocks.NewMockIExptMaterializedFilterRepo(ctrl)} + filters.MockIExptMaterializedFilterRepo.EXPECT().GetMaterializedByExptIDItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultFilterEntity{row(func(r *entity.ExptTurnResultFilterEntity) { r.ItemIdx = 2 })}, nil) + svc := ExptResultServiceImpl{exptTurnResultFilterRepo: filters} + ok, err := svc.compareMaterializedFilterLocked(ctx, 3, 1, 9, expected()) + require.NoError(t, err) + require.False(t, ok) + }) + t.Run("read_cancelled", func(t *testing.T) { + ctrl := gomock.NewController(t) + svc := ExptResultServiceImpl{exptTurnResultFilterRepo: repomocks.NewMockIExptTurnResultFilterRepo(ctrl)} + cancelled, cancel := context.WithCancel(ctx) + cancel() + _, _, err := svc.readMaterializedItemFilterLocked(cancelled, 3, 1, 9, time.Now()) + require.Error(t, err) + }) + t.Run("materialized_filter_matches_build_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil, int64(0), errors.New("list")) + svc := ExptResultServiceImpl{ExptTurnResultRepo: turns} + _, err := svc.materializedFilterMatchesLocked(ctx, 3, 1, 9) + require.Error(t, err) + }) +} + +func TestReconcileManualRetryFilterWrapperUnit(t *testing.T) { + t.Run("panic_recovered", func(t *testing.T) { + ctrl := gomock.NewController(t) + svc := svcmocks.NewMockExptManualRetryFilterService(ctrl) + svc.EXPECT().ReconcileManualRetryFilterBeforeFinish(gomock.Any(), int64(3), int64(1), int64(2)).DoAndReturn(func(context.Context, int64, int64, int64) (bool, error) { + panic("boom") + }) + ready, err := reconcileManualRetryFilterBeforeFinish(context.Background(), svc, 3, 1, 2) + require.False(t, ready) + require.Error(t, err) + }) + t.Run("success_and_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + svc := svcmocks.NewMockExptManualRetryFilterService(ctrl) + svc.EXPECT().ReconcileManualRetryFilterBeforeFinish(gomock.Any(), int64(3), int64(1), int64(2)).Return(true, nil) + svc.EXPECT().ReconcileManualRetryFilterBeforeFinish(gomock.Any(), int64(3), int64(1), int64(2)).Return(false, errors.New("f")) + ready, err := reconcileManualRetryFilterBeforeFinish(context.Background(), svc, 3, 1, 2) + require.True(t, ready) + require.NoError(t, err) + ready, err = reconcileManualRetryFilterBeforeFinish(context.Background(), svc, 3, 1, 2) + require.False(t, ready) + require.Error(t, err) + }) +} + +func TestPreparePublicationRemainingBranches(t *testing.T) { + expt := func() *entity.Experiment { return publishUnitExperiment() } + + t.Run("multiset_target_space", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].TargetResultID = 88 + source.ItemRef = &entity.ExptItemRef{ItemConfig: &entity.ExptItemConfig{TargetSourceSpaceID: 4}} + e := expt() + e.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + d.targets.EXPECT().GetRecordByID(gomock.Any(), int64(4), int64(88)).Return(&entity.EvalTargetRecord{ID: 88, SpaceID: 4, ItemID: 9, TurnID: 0, ExperimentRunID: 2, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil) + publication, err := d.svc.PrepareItemResultPublication(context.Background(), source, e) + require.NoError(t, err) + require.Len(t, publication.Turns, 1) + }) + t.Run("frozen_turn_missing_same_len", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns = map[int64]*entity.ExptTurnResultRunLog{1: publishUnitLog(1)} + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, expt()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("evaluator_ref_incomplete", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 0}}} + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, expt()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("batch_get_cancelled", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + ctx, cancel := context.WithCancel(context.Background()) + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{61}, false, false).DoAndReturn(func(context.Context, []int64, bool, bool) ([]*entity.EvaluatorRecord, error) { + cancel() + return []*entity.EvaluatorRecord{{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, EvaluatorVersionID: 51, Alias: "a", Status: entity.EvaluatorRunStatusSuccess}}, nil + }) + _, err := d.svc.PrepareItemResultPublication(ctx, source, expt()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("idgen_invalid", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.EvaluatorRecord{{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, EvaluatorVersionID: 51, Alias: "a", Status: entity.EvaluatorRunStatusSuccess}}, nil) + d.idgen.EXPECT().GenMultiIDs(gomock.Any(), 1).Return([]int64{0}, nil) + _, err := d.svc.PrepareItemResultPublication(context.Background(), source, expt()) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("sort_variants", func(t *testing.T) { + d := newPublishUnitDeps(t) + source := publishUnitSource() + source.Source.Turns[0].EvaluatorResultIds = &entity.EvaluatorResults{ + Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "b", RecordID: 61}, {VersionID: 51, Alias: "a", RecordID: 62}}, + Inline: []*entity.InlineEvalResult{{InlineKey: "z", RecordID: 63}, {InlineKey: "a", RecordID: 64}}, + } + d.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{61, 62, 63, 64}, false, false).Return([]*entity.EvaluatorRecord{ + {ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, EvaluatorVersionID: 51, Alias: "b", Status: entity.EvaluatorRunStatusSuccess}, + {ID: 62, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, EvaluatorVersionID: 51, Alias: "a", Status: entity.EvaluatorRunStatusSuccess}, + {ID: 63, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, SourceType: entity.EvaluatorRecordSourceTypeInline, InlineKey: "z", Status: entity.EvaluatorRunStatusSuccess}, + {ID: 64, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 9, TurnID: 0, SourceType: entity.EvaluatorRecordSourceTypeInline, InlineKey: "a", Status: entity.EvaluatorRunStatusSuccess}, + }, nil) + d.idgen.EXPECT().GenMultiIDs(gomock.Any(), 4).Return([]int64{901, 902, 903, 904}, nil) + publication, err := d.svc.PrepareItemResultPublication(context.Background(), source, expt()) + require.NoError(t, err) + require.Len(t, publication.Refs, 4) + }) +} + +func TestPublishPreparedRemainingBranches(t *testing.T) { + t.Run("source_check_cancels", func(t *testing.T) { + d := newPublishUnitDeps(t) + lockCtx, cancel := context.WithCancel(context.Background()) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, lockCtx, func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.repo.EXPECT().IsItemResultPublishSourceCurrent(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.ExptItemResultPublishSource) (bool, error) { + cancel() + return true, nil + }) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("before_commit_cancels", func(t *testing.T) { + d := newPublishUnitDeps(t) + lockCtx, cancel := context.WithCancel(context.Background()) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, lockCtx, func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.repo.EXPECT().IsItemResultPublishSourceCurrent(gomock.Any(), gomock.Any()).Return(true, nil) + _, err := d.svc.PublishPreparedItemResult(context.Background(), &entity.ExptItemResultPublication{Source: publishUnitSource()}, func(context.Context, *entity.ExptItemResultPublishSource) error { + cancel() + return nil + }) + require.True(t, errno.NeedMQRetry(err)) + }) +} + +func TestRecordCurrentItemResultsRemainingBranches(t *testing.T) { + event := func() *entity.ExptScheduleEvent { + return &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + } + expt := publishUnitExperiment() + t.Run("legacy_source_identity_mismatch", func(t *testing.T) { + ctrl := gomock.NewController(t) + scheduler := &ExptSchedulerImpl{} + svc := svcmocks.NewMockExptResultPublicationService(ctrl) + source := publishUnitSource() + source.Source.Current.RequestID = "req" + svc.EXPECT().GetItemResultPublishSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(source, nil) + scanned := currentUnitSource() + scanned.Current.RequestID = "other" + item := &entity.ExptEvalItem{ItemID: 9, RequestID: "req", State: entity.ItemRunState_Success, Source: scanned} + require.NoError(t, scheduler.recordCurrentItemResults(context.Background(), event(), []*entity.ExptEvalItem{item}, entitymocks.NewMockExptSchedulerMode(ctrl), expt, svc)) + }) + t.Run("cancel_after_applied", func(t *testing.T) { + ctrl := gomock.NewController(t) + d := newPublishUnitDeps(t) + _ = d + scheduler := &ExptSchedulerImpl{} + svc := svcmocks.NewMockExptResultPublicationService(ctrl) + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + source := publishUnitSource() + item := &entity.ExptEvalItem{ItemID: 9, RequestID: "req", State: entity.ItemRunState_Success, Publication: source} + ctx, cancel := context.WithCancel(context.Background()) + svc.EXPECT().PrepareItemResultPublication(gomock.Any(), source, expt).Return(&entity.ExptItemResultPublication{Source: source}, nil) + svc.EXPECT().PublishPreparedItemResult(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.ExptItemResultPublication, func(context.Context, *entity.ExptItemResultPublishSource) error) (bool, error) { + cancel() + return true, nil + }) + require.True(t, errno.NeedMQRetry(scheduler.recordCurrentItemResults(ctx, event(), []*entity.ExptEvalItem{item}, mode, expt, svc))) + }) + t.Run("cancel_during_publish_result", func(t *testing.T) { + ctrl := gomock.NewController(t) + scheduler := &ExptSchedulerImpl{} + svc := svcmocks.NewMockExptResultPublicationService(ctrl) + mode := entitymocks.NewMockExptSchedulerMode(ctrl) + source := publishUnitSource() + item := &entity.ExptEvalItem{ItemID: 9, RequestID: "req", State: entity.ItemRunState_Success, Publication: source} + ctx, cancel := context.WithCancel(context.Background()) + svc.EXPECT().PrepareItemResultPublication(gomock.Any(), source, expt).Return(&entity.ExptItemResultPublication{Source: source}, nil) + svc.EXPECT().PublishPreparedItemResult(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + mode.EXPECT().PublishResult(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, []*entity.ExptTurnEvaluatorResultRef, *entity.ExptScheduleEvent) error { + cancel() + return nil + }) + require.True(t, errno.NeedMQRetry(scheduler.recordCurrentItemResults(ctx, event(), []*entity.ExptEvalItem{item}, mode, expt, svc))) + }) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_admission_cleanup_test.go b/backend/modules/evaluation/domain/service/expt_run_admission_cleanup_test.go index bcbfd76f74..0632767135 100644 --- a/backend/modules/evaluation/domain/service/expt_run_admission_cleanup_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_admission_cleanup_test.go @@ -91,7 +91,7 @@ func TestRunAdmissionCleanup(t *testing.T) { locked = true return true, "", nil }).Times(logCalls) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Save(gomock.Any(), gomock.Any()). + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Create(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, got *entity.ExptRunLog) error { runLog = got; return nil }).Times(logCalls) } else { mgr.mutex.(*lockMocks.MockILocker).EXPECT().LockBackoff(ctx, lockKey, gomock.Any(), gomock.Any()). diff --git a/backend/modules/evaluation/domain/service/expt_run_finish.go b/backend/modules/evaluation/domain/service/expt_run_finish.go new file mode 100644 index 0000000000..d85cd96248 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_finish.go @@ -0,0 +1,226 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +// IExptNaturalRunFinisher keeps the broad manager and old test doubles compatible. +// +//go:generate mockgen -destination=mocks/expt_run_finish.go -package=mocks . IExptNaturalRunFinisher +type IExptNaturalRunFinisher interface { + FinishCurrentRun(context.Context, *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) + FinishCurrentRunWithHeldTickLock(context.Context, *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) +} + +func (e *ExptMangerImpl) FinishCurrentRun(ctx context.Context, event *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + if event == nil || e.manualRetryLocker == nil { + return nil, errno.NewMQRetryErr("finish tick locker missing") + } + locker := e.manualRetryLocker() + if locker == nil { + return nil, errno.NewMQRetryErr("finish tick locker missing") + } + key := fmt.Sprintf("expt_run_exec_lock:%d:%d", event.ExptID, event.ExptRunID) + locked, lockCtx, cancel, err := locker.LockWithRenew(ctx, key, 5*time.Second, 5*time.Minute) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if !locked { + return nil, errno.NewMQRetryErr("finish tick lock busy") + } + defer cancel() + if err := lockCtx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if e.exptRepo == nil { + return nil, errno.NewMQRetryErr("finish experiment repository missing") + } + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(lockCtx), event.ExptID, event.SpaceID) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if current == nil { + return nil, errno.NewMQRetryErr("finish experiment missing") + } + if current.LatestRunID != event.ExptRunID || entity.IsExptFinished(current.Status) { + return e.FinishCurrentRunWithHeldTickLock(lockCtx, event) + } + if calibration, ok := e.exptResultService.(ExptManualRetryFilterService); ok { + ready, err := reconcileManualRetryFilterBeforeFinish(lockCtx, calibration, event.SpaceID, event.ExptID, event.ExptRunID) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if !ready { + return &entity.ExptRunFinishResult{Disposition: entity.ExptRunFinishPending}, nil + } + } else { + return nil, errno.NewMQRetryErr("finish calibration service missing") + } + return e.FinishCurrentRunWithHeldTickLock(lockCtx, event) +} + +// The scheduler calls this only after calibration under its existing tick lease. +func (e *ExptMangerImpl) FinishCurrentRunWithHeldTickLock(ctx context.Context, event *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + if event == nil || e.runFinishRepo == nil { + return nil, errno.NewMQRetryErr("current run finish repository missing") + } + if err := ctx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + result, err := e.runFinishRepo.FinishCurrentRun(ctx, &entity.ExptRunFinishRequest{SpaceID: event.SpaceID, ExptID: event.ExptID, RunID: event.ExptRunID}) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if result == nil { + return nil, errno.NewMQRetryErr("finish returned no disposition") + } + switch result.Disposition { + case entity.ExptRunFinishPending, entity.ExptRunFinishNonCurrent: + return result, nil + case entity.ExptRunFinishFinished, entity.ExptRunFinishAlreadyFinished: + if err := e.cleanupNaturallyFinishedRun(ctx, event, result.Experiment); err != nil { + return result, errno.WrapMQRetryErr(err) + } + return result, nil + default: + return nil, errno.NewMQRetryErr("invalid finish disposition") + } +} + +func (e *ExptMangerImpl) cleanupNaturallyFinishedRun(ctx context.Context, event *entity.ExptScheduleEvent, finished *entity.Experiment) error { + if finished == nil { + return fmt.Errorf("finished experiment missing") + } + if e.manualRetryLocker == nil { + return fmt.Errorf("finish lifecycle locker missing") + } + locker := e.manualRetryLocker() + if locker == nil { + return fmt.Errorf("finish lifecycle locker missing") + } + key := e.makeExptMutexLockKey(event.ExptID) + // Remove only this run's original value, then acquire the same lifecycle key. + // If R2 wins the gap we do no cleanup. A legacy random holder is never forced. + if _, err := locker.UnlockWithValue(ctx, key, strconv.FormatInt(event.ExptRunID, 10)); err != nil { + return err + } + locked, cleanupCtx, cancel, err := locker.LockWithRenew(ctx, key, 5*time.Second, 5*time.Minute) + if err != nil { + return err + } + if !locked { + return errno.NewMQRetryErr("finish lifecycle owner unavailable; legacy random holders require expiry") + } + defer cancel() + if err := cleanupCtx.Err(); err != nil { + return err + } + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(cleanupCtx), event.ExptID, event.SpaceID) + if err != nil { + return err + } + if current == nil { + return fmt.Errorf("finished experiment disappeared") + } + if current.LatestRunID != event.ExptRunID { + return nil + } + if !entity.IsExptFinished(current.Status) || current.EndAt == nil { + return fmt.Errorf("finish cleanup observed active experiment") + } + // Every external action remains under this lifecycle lease, which also blocks + // new LogRun/LogRetryItemsRun. The SQL finish transaction has already committed. + if err := NewQuotaService(e.quotaRepo, e.configer).ReleaseExptRun(cleanupCtx, event.ExptID, event.SpaceID, event.Session); err != nil { + return err + } + if err := cleanupCtx.Err(); err != nil { + return err + } + if e.exptResultService != nil { + incomplete, err := e.exptResultService.GetIncompleteTurns(cleanupCtx, event.ExptID, event.SpaceID, event.Session) + if err != nil { + return err + } + ids := make([]int64, 0, len(incomplete)) + seen := map[int64]bool{} + for _, turn := range incomplete { + if !seen[turn.ItemID] { + ids = append(ids, turn.ItemID) + seen[turn.ItemID] = true + } + } + if len(ids) > 0 { + _ = e.terminateSandboxExecutesForCancelledItems(cleanupCtx, event.SpaceID, current.TargetSpaceID, event.ExptID, &event.ExptRunID, ids) + } + } + if err := cleanupCtx.Err(); err != nil { + return err + } + + // Resources above remain replayable. Only the completed notification segment + // is deduplicated, under the same lifecycle lease that serializes retries. + if e.idem == nil { + return errno.NewMQRetryErr("finish notification idempotency service missing") + } + notificationKey := fmt.Sprintf("finish-cleanup:%d:%d", event.ExptID, event.ExptRunID) + seen, err := e.idem.Exist(cleanupCtx, notificationKey) + if err != nil { + return err + } + if seen { + return cleanupCtx.Err() + } + + if current.ExptTemplateMeta != nil && current.ExptTemplateMeta.ID > 0 && e.templateManager != nil { + if err := e.templateManager.UpdateExptInfo(cleanupCtx, current.ExptTemplateMeta.ID, event.SpaceID, event.ExptID, current.Status, 0, nil); err != nil { + return err + } + } + if e.exptAggrResultService != nil { + if err := e.exptAggrResultService.PublishExptAggrResultEvent(cleanupCtx, &entity.AggrCalculateEvent{ExperimentID: event.ExptID, SpaceID: event.SpaceID, CalculateMode: entity.CreateAllFields}, gptr.Of(3*time.Second)); err != nil { + return err + } + } + if err := cleanupCtx.Err(); err != nil { + return err + } + if e.publisher != nil { + // Stable Processing -> final key survives cleanup retries without DB markers. + lifecycle := &entity.ExptLifecycleEvent{ExptID: event.ExptID, ExptRunID: &event.ExptRunID, SpaceID: event.SpaceID, FromStatus: entity.ExptStatus_Processing, ToStatus: current.Status, ExptType: current.ExptType, SourceType: current.SourceType} + lifecycle.IdempotentKey = fmt.Sprintf("expt_%d_%d_%d_%d", event.ExptID, event.ExptRunID, lifecycle.FromStatus, lifecycle.ToStatus) + if err := e.publisher.PublishExptLifecycleEvent(cleanupCtx, lifecycle, gptr.Of(3*time.Second), lifecycle.IdempotentKey); err != nil { + return err + } + } + if err := cleanupCtx.Err(); err != nil { + return err + } + if current.SourceType == entity.SourceType_Workflow { + e.notifyWorkflowPipelineOnExptFinished(cleanupCtx, current, event.SpaceID, current.Status) + } + + if e.mtr != nil { + e.mtr.EmitExptExecResult(event.SpaceID, int64(current.ExptType), int64(current.Status), gptr.Indirect(current.StartAt)) + } + e.emitSandboxAgentExperimentFinished(cleanupCtx, current, current.Status, gptr.Indirect(current.EndAt)) + if err := cleanupCtx.Err(); err != nil { + return err + } + duration := 3 * time.Minute + if e.configer != nil { + duration = time.Duration(e.configer.GetExptExecConf(cleanupCtx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 + } + // A crash or Set failure after RPC success can replay this segment; the key + // bounds normal duplicates, it does not provide distributed exactly-once. + return e.idem.Set(cleanupCtx, notificationKey, duration) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_finish_compatibility_test.go b/backend/modules/evaluation/domain/service/expt_run_finish_compatibility_test.go new file mode 100644 index 0000000000..84a702b58f --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_finish_compatibility_test.go @@ -0,0 +1,292 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + confmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + rpcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestRunFinishCheckDisposition(t *testing.T) { + for _, kind := range []string{"pending", "nil_result", "already", "noncurrent", "finished", "error"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + manager.EXPECT().GetRunLog(gomock.Any(), int64(1), int64(2), int64(3), gomock.Any()).Return(&entity.ExptRunLog{Status: int64(entity.ExptStatus_Success)}, nil) + probe := &checkFinishProbe{IExptManager: manager, result: &entity.ExptRunFinishResult{Disposition: entity.ExptRunFinishAlreadyFinished}} + switch kind { + case "pending": + probe.result.Disposition = entity.ExptRunFinishPending + case "nil_result": + probe.result = nil + case "noncurrent": + probe.result.Disposition = entity.ExptRunFinishNonCurrent + case "finished": + probe.result.Disposition = entity.ExptRunFinishFinished + case "error": + probe.err = errno.NewMQRetryErr("finish dependency failed") + } + scheduler := &ExptSchedulerImpl{Manager: probe} + err := scheduler.HandleEventCheck(func(context.Context, *entity.ExptScheduleEvent) error { + t.Fatal("finished event reached execution") + return nil + })(context.Background(), &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, ExptType: entity.ExptType_Offline, ExptRunMode: entity.EvaluationModeSubmit}) + if kind == "pending" || kind == "nil_result" || kind == "error" { + require.True(t, errno.NeedMQRetry(err)) + } else { + require.NoError(t, err) + } + }) + } +} + +type checkFinishProbe struct { + IExptManager + result *entity.ExptRunFinishResult + err error +} + +func (p *checkFinishProbe) FinishCurrentRun(context.Context, *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + return p.result, p.err +} + +func (p *checkFinishProbe) FinishCurrentRunWithHeldTickLock(context.Context, *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + return p.result, p.err +} + +type cleanupTestIdem struct { + seen bool + readErr, setErr error + keys []string + ttl time.Duration +} + +func (i *cleanupTestIdem) Exist(_ context.Context, key string) (bool, error) { + i.keys = append(i.keys, key) + return i.seen, i.readErr +} + +func (i *cleanupTestIdem) Set(_ context.Context, key string, ttl time.Duration) error { + i.keys = append(i.keys, key) + i.ttl = ttl + if i.setErr != nil { + return i.setErr + } + i.seen = true + return nil +} + +func (i *cleanupTestIdem) SetNX(ctx context.Context, key string, ttl time.Duration) (bool, error) { + if i.seen { + return false, nil + } + return true, i.Set(ctx, key, ttl) +} +func (i *cleanupTestIdem) Del(context.Context, string) error { i.seen = false; return nil } + +func TestRunFinishCleanupNotificationGateAndFailureOrder(t *testing.T) { + for _, fault := range []string{"none", "template", "aggr", "lifecycle", "idem_read"} { + t.Run(fault, func(t *testing.T) { + ctrl := gomock.NewController(t) + factory, _ := finishRealLockerFactory(t) + expt := &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Success, EndAt: gptr.Of(time.Now()), SourceType: entity.SourceType_Workflow, ExptTemplateMeta: &entity.ExptTemplateMeta{ID: 4}} + expts := repomocks.NewMockIExperimentRepo(ctrl) + expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt, nil).AnyTimes() + quota := repomocks.NewMockQuotaRepo(ctrl) + resources := 0 + quota.EXPECT().CreateOrUpdate(gomock.Any(), int64(3), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, int64, func(*entity.QuotaSpaceExpt) (*entity.QuotaSpaceExpt, bool, error), *entity.Session) error { + resources++ + return nil + }).AnyTimes() + template := svcmocks.NewMockIExptTemplateManager(ctrl) + aggr := svcmocks.NewMockExptAggrResultService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + pipeline := rpcmocks.NewMockIPipelineListAdapter(ctrl) + metric := metricsmocks.NewMockExptMetric(ctrl) + calls := map[string]int{} + shouldFail := func(step string) bool { calls[step]++; return fault == step && calls[step] == 1 } + template.EXPECT().UpdateExptInfo(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, int64, int64, int64, entity.ExptStatus, int64, *int64) error { + if shouldFail("template") { + return errors.New("template fault") + } + return nil + }).AnyTimes() + aggr.EXPECT().PublishExptAggrResultEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.AggrCalculateEvent, *time.Duration) error { + if shouldFail("aggr") { + return errors.New("aggr fault") + } + return nil + }).AnyTimes() + publisher.EXPECT().PublishExptLifecycleEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptLifecycleEvent, _ *time.Duration, key string) error { + require.Equal(t, "expt_1_2_3_11", key) + require.Equal(t, entity.ExptStatus_Processing, event.FromStatus) + if shouldFail("lifecycle") { + return errors.New("lifecycle fault") + } + return nil + }).AnyTimes() + pipeline.EXPECT().PipelineNodeFinishCallback(gomock.Any(), int64(1), int64(3)).DoAndReturn(func(context.Context, int64, int64) error { calls["workflow"]++; return nil }).AnyTimes() + metric.EXPECT().EmitExptExecResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(func(int64, int64, int64, time.Time) { calls["metric"]++ }).AnyTimes() + sandbox := metricsmocks.NewMockSandboxAgentMetrics(ctrl) + expt.Target = &entity.EvalTarget{EvalTargetVersion: &entity.EvalTargetVersion{EvalTargetType: entity.EvalTargetTypeSandboxAgent}} + sandbox.EXPECT().EmitExperimentFinished(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(func(metrics.SandboxAgentExperimentTags, error, time.Time, time.Time) { calls["sandbox_metric"]++ }).AnyTimes() + idem := &cleanupTestIdem{} + if fault == "idem_read" { + idem.readErr = errors.New("idem unavailable") + } + config := confmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}).AnyTimes() + manager := &ExptMangerImpl{sandboxAgentMetrics: sandbox, configer: config, exptRepo: expts, quotaRepo: quota, manualRetryLocker: factory, idem: idem, templateManager: template, exptAggrResultService: aggr, publisher: publisher, pipelineListAdapter: pipeline, mtr: metric} + event := &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} + err := manager.cleanupNaturallyFinishedRun(context.Background(), event, expt) + if fault != "none" { + require.Error(t, err) + require.Zero(t, calls["workflow"], "failure before completed notification must not emit workflow") + require.Zero(t, calls["metric"]) + require.Zero(t, calls["sandbox_metric"]) + idem.readErr = nil + require.NoError(t, manager.cleanupNaturallyFinishedRun(context.Background(), event, expt)) + } else { + require.NoError(t, err) + } + before := fmt.Sprint(calls) + require.NoError(t, manager.cleanupNaturallyFinishedRun(context.Background(), event, expt)) + require.Equal(t, before, fmt.Sprint(calls), "normal duplicate repeats notifications") + require.Equal(t, 1, calls["workflow"]) + require.Equal(t, 1, calls["metric"]) + require.Equal(t, 1, calls["sandbox_metric"]) + require.True(t, idem.seen) + require.GreaterOrEqual(t, resources, 2, "resource retries must not be skipped by notification marker") + require.Greater(t, idem.ttl, time.Hour) + }) + } +} + +func TestRunFinishCleanupSetFailureHasExplicitReplayWindow(t *testing.T) { + ctrl := gomock.NewController(t) + factory, _ := finishRealLockerFactory(t) + expt := &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Success, EndAt: gptr.Of(time.Now())} + expts := repomocks.NewMockIExperimentRepo(ctrl) + expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt, nil).AnyTimes() + quota := repomocks.NewMockQuotaRepo(ctrl) + quota.EXPECT().CreateOrUpdate(gomock.Any(), int64(3), gomock.Any(), gomock.Any()).Return(nil).Times(2) + metric := metricsmocks.NewMockExptMetric(ctrl) + count := 0 + metric.EXPECT().EmitExptExecResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(func(int64, int64, int64, time.Time) { count++ }).Times(2) + idem := &cleanupTestIdem{setErr: errors.New("marker unavailable")} + manager := &ExptMangerImpl{exptRepo: expts, quotaRepo: quota, manualRetryLocker: factory, idem: idem, mtr: metric} + event := &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} + require.Error(t, manager.cleanupNaturallyFinishedRun(context.Background(), event, expt)) + require.False(t, idem.seen) + require.Equal(t, 1, count) + idem.setErr = nil + require.NoError(t, manager.cleanupNaturallyFinishedRun(context.Background(), event, expt)) + require.True(t, idem.seen) + require.Equal(t, 2, count, "marker failure after external success is a documented replay window") +} + +func TestRunFinishCleanupMissingIdemFailsClosed(t *testing.T) { + ctrl := gomock.NewController(t) + factory, _ := finishRealLockerFactory(t) + expt := &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Success, EndAt: gptr.Of(time.Now())} + expts := repomocks.NewMockIExperimentRepo(ctrl) + expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt, nil) + quota := repomocks.NewMockQuotaRepo(ctrl) + quota.EXPECT().CreateOrUpdate(gomock.Any(), int64(3), gomock.Any(), gomock.Any()).Return(nil) + manager := &ExptMangerImpl{exptRepo: expts, quotaRepo: quota, manualRetryLocker: factory, publisher: eventmocks.NewMockExptEventPublisher(ctrl), mtr: metricsmocks.NewMockExptMetric(ctrl)} + require.True(t, errno.NeedMQRetry(manager.cleanupNaturallyFinishedRun(context.Background(), &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3}, expt))) +} + +func TestRunFinishCleanupMarkerDoesNotSkipResourceFailure(t *testing.T) { + ctrl := gomock.NewController(t) + factory, _ := finishRealLockerFactory(t) + expt := &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Success, EndAt: gptr.Of(time.Now())} + expts := repomocks.NewMockIExperimentRepo(ctrl) + expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt, nil).Times(2) + quota := repomocks.NewMockQuotaRepo(ctrl) + quota.EXPECT().CreateOrUpdate(gomock.Any(), int64(3), gomock.Any(), gomock.Any()).Return(errors.New("resource release unavailable")) + quota.EXPECT().CreateOrUpdate(gomock.Any(), int64(3), gomock.Any(), gomock.Any()).Return(nil) + manager := &ExptMangerImpl{exptRepo: expts, quotaRepo: quota, manualRetryLocker: factory, idem: &cleanupTestIdem{seen: true}, publisher: eventmocks.NewMockExptEventPublisher(ctrl), mtr: metricsmocks.NewMockExptMetric(ctrl)} + event := &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} + require.Error(t, manager.cleanupNaturallyFinishedRun(context.Background(), event, expt)) + require.NoError(t, manager.cleanupNaturallyFinishedRun(context.Background(), event, expt)) +} + +func TestRunFinishRealCommitNotificationFailureRetry(t *testing.T) { + for _, fault := range []string{"template", "aggr", "lifecycle"} { + t.Run(fault, func(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + p.publishSuccess(t, false) + f := p.f + ctx := context.Background() + ctrl := gomock.NewController(t) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"source_type": int32(entity.SourceType_Workflow), "expt_template_id": f.base + 700}).Error) + template := svcmocks.NewMockIExptTemplateManager(ctrl) + aggr := svcmocks.NewMockExptAggrResultService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + pipeline := rpcmocks.NewMockIPipelineListAdapter(ctrl) + counts := map[string]int{} + fail := func(step string) bool { counts[step]++; return fault == step && counts[step] == 1 } + template.EXPECT().UpdateExptInfo(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, int64, int64, int64, entity.ExptStatus, int64, *int64) error { + if fail("template") { + return errors.New("template commit cleanup fault") + } + return nil + }).AnyTimes() + aggr.EXPECT().PublishExptAggrResultEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.AggrCalculateEvent, *time.Duration) error { + if fail("aggr") { + return errors.New("aggr commit cleanup fault") + } + return nil + }).AnyTimes() + publisher.EXPECT().PublishExptLifecycleEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.ExptLifecycleEvent, *time.Duration, string) error { + if fail("lifecycle") { + return errors.New("lifecycle commit cleanup fault") + } + return nil + }).AnyTimes() + pipeline.EXPECT().PipelineNodeFinishCallback(gomock.Any(), f.expt, f.space).DoAndReturn(func(context.Context, int64, int64) error { counts["workflow"]++; return nil }).AnyTimes() + p.manager.templateManager = template + p.manager.exptAggrResultService = aggr + p.manager.publisher = publisher + p.manager.pipelineListAdapter = pipeline + result, err := p.manager.FinishCurrentRun(ctx, p.event) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, entity.ExptRunFinishFinished, result.Disposition) + require.Zero(t, counts["workflow"]) + require.Zero(t, p.notifications.Load()) + snapshot := f.snapshot() + result, err = p.manager.FinishCurrentRun(ctx, p.event) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishAlreadyFinished, result.Disposition) + require.Equal(t, snapshot, f.snapshot()) + require.Equal(t, 1, counts["workflow"]) + require.Equal(t, int32(1), p.notifications.Load()) + calls := fmt.Sprint(counts) + result, err = p.manager.FinishCurrentRun(ctx, p.event) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishAlreadyFinished, result.Disposition) + require.Equal(t, calls, fmt.Sprint(counts)) + require.Equal(t, int32(1), p.notifications.Load()) + require.Equal(t, int32(3), p.resources.Load()) + require.Equal(t, snapshot, f.snapshot()) + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_run_finish_pending_test.go b/backend/modules/evaluation/domain/service/expt_run_finish_pending_test.go new file mode 100644 index 0000000000..0e3197b526 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_finish_pending_test.go @@ -0,0 +1,308 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "encoding/json" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + idmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + infraredis "github.com/coze-dev/coze-loop/backend/infra/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + confmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + exptrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + exptmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + idemrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/idem" + idemredis "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/idem/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +type pendingRunFixture struct { + f *lateWriteFixture + manager *ExptMangerImpl + start *ExptSubmitExec + event *entity.ExptScheduleEvent + expt *entity.Experiment + result *ExptResultServiceImpl + factory ManualRetryLockerFactory + records *svcmocks.MockEvaluatorRecordService + resources, notifications *atomic.Int32 +} + +// Runs the actual two initializers and actual Submit.ExptStart; no fixture +// assigns Pending to the run. Item/turn SQL writes also use production ports. +func newPendingRunFixture(t *testing.T, initMode entity.ExptRunMode) *pendingRunFixture { + t.Helper() + f := newLateWriteFixture(t, "") + ctx := context.Background() + ctrl := gomock.NewController(t) + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_run_log"} { + require.NoError(t, f.conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + } + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.ExptStatus_Pending), "latest_run_id": 0, "eval_set_id": f.base + 80, "eval_set_version_id": f.base + 81}).Error) + require.NoError(t, f.conn.Model(&model.ExptStats{}).Where("space_id=?", f.space).Updates(map[string]any{"pending_cnt": 0, "processing_cnt": 0, "success_cnt": 0}).Error) + factory, server := finishRealLockerFactory(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { _ = raw.Close() }) + idem := idemrepo.NewIdempotentService(idemredis.NewIdemDAO(client)) + config := confmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), f.space).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}).AnyTimes() + ids := idmocks.NewMockIIDGenerator(ctrl) + ids.EXPECT().GenID(gomock.Any()).Return(f.run, nil).AnyTimes() + var seq atomic.Int64 + ids.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, n int) ([]int64, error) { + out := make([]int64, n) + for i := range out { + out[i] = f.base + 9000 + seq.Add(1) + } + return out, nil + }).AnyTimes() + experiments := exptrepo.NewExptRepo(exptmysql.NewExptDAO(f.provider), exptmysql.NewExptEvaluatorRefDAO(f.provider), ids) + runs := exptrepo.NewExptRunLogRepo(exptmysql.NewExptRunLogDAO(f.provider)) + stats := exptrepo.NewExptStatsRepo(exptmysql.NewExptStatsDAO(f.provider)) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + result := &ExptResultServiceImpl{ExptItemResultRepo: f.itemRepo(), ExptTurnResultRepo: f.turnRepo(), ExptStatsRepo: stats, idgen: ids, evaluatorRecordService: records, scoreCalculator: NewEvaluatorScoreCalculator(nil, nil), resultPublishRepo: exptrepo.NewExptResultPublishRepo(f.provider), resultPublishLocker: factory} + resources, notifications := &atomic.Int32{}, &atomic.Int32{} + quota := repomocks.NewMockQuotaRepo(ctrl) + quota.EXPECT().CreateOrUpdate(gomock.Any(), f.space, gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ int64, _ func(*entity.QuotaSpaceExpt) (*entity.QuotaSpaceExpt, bool, error), _ *entity.Session) error { + resources.Add(1) + var row model.Experiment + require.NoError(t, f.conn.WithContext(ctx).Where("space_id=?", f.space).Take(&row).Error) + require.True(t, entity.IsExptFinished(entity.ExptStatus(row.Status))) + return nil + }).AnyTimes() + metric := metricsmocks.NewMockExptMetric(ctrl) + metric.EXPECT().EmitExptExecRun(f.space, int64(initMode)) + metric.EXPECT().EmitExptExecResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(func(int64, int64, int64, time.Time) { notifications.Add(1) }).AnyTimes() + cleanupResult := &finishCalibrationTestResult{ExptResultService: svcmocks.NewMockExptResultService(ctrl), ready: true} + cleanupResult.ExptResultService.(*svcmocks.MockExptResultService).EXPECT().GetIncompleteTurns(gomock.Any(), f.expt, f.space, gomock.Any()).Return(nil, nil).AnyTimes() + cleanupResult.ExptResultService.(*svcmocks.MockExptResultService).EXPECT().UpsertExptTurnResultFilter(gomock.Any(), f.space, f.expt, gomock.Any()).DoAndReturn(func(ctx context.Context, _, _ int64, ids []int64) error { + var rows []model.ExptItemResult + require.NoError(t, f.conn.WithContext(ctx).Where("space_id=? AND item_id IN ?", f.space, ids).Find(&rows).Error) + for _, row := range rows { + require.Equal(t, int32(entity.ItemRunState_Terminal), row.Status) + } + return nil + }).AnyTimes() + manager := &ExptMangerImpl{configer: config, mutex: factory(), manualRetryLocker: factory, runLogRepo: runs, exptRepo: experiments, itemResultRepo: f.itemRepo(), itemRefRepo: exptrepo.NewExptItemRefRepo(exptmysql.NewExptItemRefDAO(f.provider)), turnResultRepo: f.turnRepo(), statsRepo: stats, runFinishRepo: exptrepo.NewExptRunFinishRepo(f.provider), manualRetryRepo: exptrepo.NewExptManualRetryRepo(f.provider, ids), mtr: metric, idgenerator: ids, idem: idem, centralGuard: component.NewNoopCentralReservationGuard(), currentRunWaker: component.NewNoopCurrentRunSchedulerWaker(), quotaRepo: quota, exptResultService: cleanupResult} + session := &entity.Session{UserID: "pending-run-test"} + if initMode == entity.EvaluationModeRetryItems { + run, joined, err := manager.LogRetryItemsRun(ctx, f.expt, initMode, f.space, []int64{f.item}, session) + require.NoError(t, err) + require.False(t, joined) + require.Equal(t, f.run, run) + } else { + require.NoError(t, manager.LogRun(ctx, f.expt, f.run, initMode, f.space, []int64{f.item}, session)) + } + itemSvc := svcmocks.NewMockEvaluationSetItemService(ctrl) + itemSvc.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{{ItemID: f.item, ItemVersionID: &f.version, Turns: []*entity.Turn{{ID: 0}}}}, gptr.Of(int64(1)), nil, nil, nil) + startResult := svcmocks.NewMockExptResultService(ctrl) + startResult.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), f.space, f.expt, gomock.Any()).Return(nil) + event := &entity.ExptScheduleEvent{ExptID: f.expt, ExptRunID: f.run, SpaceID: f.space, ExptRunMode: initMode, ExptType: entity.ExptType_Offline, Session: session} + expt := &entity.Experiment{ID: f.expt, SpaceID: f.space, LatestRunID: f.run, ExptType: entity.ExptType_Offline, EvalSet: &entity.EvaluationSet{ID: f.base + 80, EvaluationSetVersion: &entity.EvaluationSetVersion{ID: f.base + 81}}} + start := &ExptSubmitExec{manager: manager, exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), exptStatsRepo: stats, exptRepo: experiments, idem: idem, configer: config, idgenerator: ids, evaluationSetItemService: itemSvc, resultSvc: startResult} + require.NoError(t, start.ExptStart(ctx, event, expt)) + run, err := runs.Get(ctx, f.expt, f.run) + require.NoError(t, err) + require.Equal(t, int64(entity.ExptStatus_Pending), run.Status) + current, err := experiments.GetByID(ctx, f.expt, f.space) + require.NoError(t, err) + require.Equal(t, entity.ExptStatus_Processing, current.Status) + require.NoError(t, f.turnRepo().BatchCreateNXRunLog(ctx, []*entity.ExptTurnResultRunLog{{ID: f.base + 9900, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: 0, Status: entity.TurnRunState_Queueing}})) + return &pendingRunFixture{f: f, manager: manager, start: start, event: event, expt: current, result: result, factory: factory, records: records, resources: resources, notifications: notifications} +} + +func (p *pendingRunFixture) publishSuccess(t *testing.T, withRecord bool) { + t.Helper() + f := p.f + ctx := context.Background() + current, err := p.manager.itemResultRepo.GetCurrentRequest(ctx, f.space, f.expt, f.run, f.item) + require.NoError(t, err) + require.NotNil(t, current) + dispatch := exptrepo.NewExptItemDispatchRepo(exptmysql.NewExptItemDispatchDAO(f.provider)) + runtime, err := dispatch.LoadDispatchRuntime(ctx, f.space, f.expt, f.run, 1) + require.NoError(t, err) + require.Equal(t, []int64{f.item}, runtime.CandidateItemIDs) + require.Equal(t, current.RequestID, runtime.ItemRequestIDs[f.item]) + if current.RequestID != "" { + for _, stale := range []string{"", "obsolete-request"} { + before := f.snapshot() + claimed, err := dispatch.ClaimQuotaReserved(ctx, f.space, f.expt, f.run, []int64{f.item}, map[int64]string{f.item: stale}) + require.NoError(t, err) + require.Empty(t, claimed) + require.Equal(t, before, f.snapshot(), "stale grant cannot mutate the admitted request") + } + } + claimed, err := dispatch.ClaimQuotaReserved(ctx, f.space, f.expt, f.run, []int64{f.item}, runtime.ItemRequestIDs) + require.NoError(t, err) + require.Len(t, claimed, 1) + payload := claimed[0] + require.Equal(t, f.item, payload.ItemID) + require.Equal(t, current.RequestID, payload.RequestID) + require.Equal(t, current.Mode, payload.Mode) + require.Equal(t, current.RetryTimes, payload.RetryTimes) + assertPending := func(stage string) { + before := f.snapshot() + finish, err := p.manager.runFinishRepo.FinishCurrentRun(ctx, &entity.ExptRunFinishRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run}) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishPending, finish.Disposition, stage) + require.Equal(t, before, f.snapshot(), stage) + } + assertPending("reserved grant") + started, err := dispatch.StartReservedItem(ctx, f.space, f.expt, f.run, payload.ItemID, payload.RequestID) + require.NoError(t, err) + require.True(t, started.CurrentRequest) + require.True(t, started.Started) + before := f.snapshot() + duplicate, err := dispatch.StartReservedItem(ctx, f.space, f.expt, f.run, payload.ItemID, payload.RequestID) + require.NoError(t, err) + require.True(t, duplicate.CurrentRequest) + require.False(t, duplicate.Started) + require.Equal(t, before, f.snapshot(), "duplicate Start cannot apply a second stats delta") + assertPending("processing source") + consumer := &ExptItemEventEvalServiceImpl{exptItemResultRepo: f.itemRepo()} + invoked := false + require.NoError(t, consumer.HandleCurrentSource(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + invoked = true + source := executionSource(ctx).Source + require.Equal(t, payload.RequestID, source.Current.RequestID) + require.Equal(t, payload.Mode, source.Current.Mode) + require.Equal(t, int(payload.RetryTimes), event.RetryTimes) + return nil + })(ctx, &entity.ExptItemEvalEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, EvalSetItemID: payload.ItemID, RequestID: payload.RequestID, ExptRunMode: payload.Mode})) + require.True(t, invoked) + source := f.source() + turn := *source.Turns[0] + turn.Status = entity.TurnRunState_Success + turn.TargetResultID = 0 + turn.EvaluatorResultIds = &entity.EvaluatorResults{} + if withRecord { + turn.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{f.base + 51: f.base + 61}} + p.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{f.base + 61}, false, false).Return([]*entity.EvaluatorRecord{{ID: f.base + 61, SpaceID: f.space, ExperimentID: f.expt, ExperimentRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: 0, EvaluatorVersionID: f.base + 51, Status: entity.EvaluatorRunStatusSuccess}}, nil) + } + applied, err := f.port.StoreCurrentTurnRunResult(ctx, source, &turn) + require.NoError(t, err) + require.True(t, applied) + source = f.source() + applied, err = f.port.CompleteCurrentItemRun(ctx, source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Success, RetryTimes: source.Current.RetryTimes}) + require.NoError(t, err) + require.True(t, applied) + assertPending("logged result awaiting publication") + _, err = p.result.RecordItemRunLogs(ctx, f.expt, f.run, f.item, f.space, p.expt) + require.NoError(t, err) + log, err := p.manager.itemResultRepo.GetItemRunLog(ctx, f.expt, f.run, f.item, f.space) + require.NoError(t, err) + require.Equal(t, int32(entity.ExptItemResultStateResulted), log.ResultState) + t.Logf("REAL_MYSQL_PENDING_DISPATCH_SOURCE_PUBLICATION request=%q mode=%d duplicate_start_stats_once=true reserved_processing_logged_block_finish=true", payload.RequestID, payload.Mode) +} + +func TestRunFinishPendingRealInitializers(t *testing.T) { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeSubmit, entity.EvaluationModeRetryItems} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + p := newPendingRunFixture(t, mode) + before := p.f.snapshot() + result, err := exptrepo.NewExptRunFinishRepo(p.f.provider).FinishCurrentRun(context.Background(), &entity.ExptRunFinishRequest{SpaceID: p.f.space, ExptID: p.f.expt, RunID: p.f.run}) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishPending, result.Disposition) + require.Equal(t, before, p.f.snapshot()) + }) + } +} + +func TestRunFinishPendingRealAdmissionPublicationAndFourModes(t *testing.T) { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeSubmit, entity.EvaluationModeFailRetry, entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + initMode := mode + if mode != entity.EvaluationModeRetryItems { + initMode = entity.EvaluationModeSubmit + } + p := newPendingRunFixture(t, initMode) + f := p.f + p.publishSuccess(t, false) + req := &entity.RetryInCurrentRunRequest{SpaceID: f.space, ExptID: f.expt, ExpectedRunID: f.run, RequestID: "72cc8921-40f4-4a43-b772-aaf2b5dcdf05", Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{f.item}, Session: p.event.Session} + receipt, err := p.manager.RetryInCurrentRun(context.Background(), req) + require.NoError(t, err) + require.Equal(t, int32(1), receipt.AcceptedCount) + run, err := p.manager.runLogRepo.Get(context.Background(), f.expt, f.run) + require.NoError(t, err) + require.Equal(t, int64(entity.ExptStatus_Pending), run.Status) + finish, err := exptrepo.NewExptRunFinishRepo(f.provider).FinishCurrentRun(context.Background(), &entity.ExptRunFinishRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run}) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishPending, finish.Disposition) + p.publishSuccess(t, true) + event := *p.event + event.ExptRunMode = mode + event.ExecEvalSetItemIDs = []int64{f.item} + base := p.manager.exptResultService + var end func(context.Context, *entity.ExptScheduleEvent, *entity.Experiment, int, int) (bool, error) + switch mode { + case entity.EvaluationModeSubmit: + end = (&ExptSubmitExec{manager: p.manager, idem: p.manager.idem, configer: p.manager.configer, exptItemResultRepo: f.itemRepo(), resultSvc: base}).ExptEnd + case entity.EvaluationModeFailRetry: + end = (&ExptFailRetryExec{manager: p.manager, idem: p.manager.idem, configer: p.manager.configer, exptItemResultRepo: f.itemRepo(), resultSvc: base}).ExptEnd + case entity.EvaluationModeRetryAll: + end = (&ExptRetryAllExec{manager: p.manager, idem: p.manager.idem, configer: p.manager.configer, exptItemResultRepo: f.itemRepo(), resultSvc: base}).ExptEnd + case entity.EvaluationModeRetryItems: + end = (&ExptRetryItemsExec{manager: p.manager, idem: p.manager.idem, configer: p.manager.configer, exptItemResultRepo: f.itemRepo(), exptRunLogRepo: p.manager.runLogRepo, resultSvc: base}).ExptEnd + } + scheduler := &ExptSchedulerImpl{Manager: p.manager, Mutex: p.factory()} + require.NoError(t, scheduler.HandleEventLock(func(ctx context.Context, e *entity.ExptScheduleEvent) error { + next, err := end(ctx, e, p.expt, 0, 0) + require.False(t, next) + return err + })(context.Background(), &event)) + after := f.snapshot() + require.Equal(t, int32(1), p.notifications.Load()) + require.NoError(t, scheduler.HandleEventCheck(func(context.Context, *entity.ExptScheduleEvent) error { + t.Fatal("duplicate finished event reentered execution") + return nil + })(context.Background(), &event)) + require.Equal(t, int32(1), p.notifications.Load(), "real idem must suppress normal AlreadyFinished duplicates") + require.Equal(t, int32(2), p.resources.Load(), "resource cleanup still replays after notification marker") + require.Equal(t, after, f.snapshot(), "AlreadyFinished cannot recalculate stats or endAt") + again, err := p.manager.RetryInCurrentRun(context.Background(), req) + require.NoError(t, err) + a, _ := json.Marshal(receipt) + b, _ := json.Marshal(again) + require.Equal(t, string(a), string(b)) + require.Equal(t, after, f.snapshot()) + req.RequestID = "c551f9c8-e195-48a0-8505-f63b20802206" + _, err = p.manager.RetryInCurrentRun(context.Background(), req) + require.Error(t, err) + require.Equal(t, after, f.snapshot()) + }) + } +} + +func TestRunFinishPendingRealHalfTerminalCheckRetries(t *testing.T) { + p := newPendingRunFixture(t, entity.EvaluationModeSubmit) + p.publishSuccess(t, false) + f := p.f + // Reproduce the old split CompleteRun/CompleteExpt half-state explicitly. + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + before := f.snapshot() + scheduler := &ExptSchedulerImpl{Manager: p.manager} + err := scheduler.HandleEventCheck(func(context.Context, *entity.ExptScheduleEvent) error { + t.Fatal("half-terminal event entered execution") + return nil + })(context.Background(), p.event) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, before, f.snapshot()) + require.Zero(t, p.resources.Load()) + require.Zero(t, p.notifications.Load()) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_finish_provider.go b/backend/modules/evaluation/domain/service/expt_run_finish_provider.go new file mode 100644 index 0000000000..192ee350f0 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_finish_provider.go @@ -0,0 +1,63 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "github.com/coze-dev/coze-loop/backend/infra/external/audit" + "github.com/coze-dev/coze-loop/backend/infra/external/benefit" + "github.com/coze-dev/coze-loop/backend/infra/idgen" + "github.com/coze-dev/coze-loop/backend/infra/lock" + "github.com/coze-dev/coze-loop/backend/infra/platestwrite" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" +) + +// NewExptManagerWithRunFinish is the production provider; the old constructor +// retains existing call sites while current repositories fail closed without it. +func NewExptManagerWithRunFinish( + exptResultService ExptResultService, + exptRepo repo.IExperimentRepo, + exptRunLogRepo repo.IExptRunLogRepo, + exptStatsRepo repo.IExptStatsRepo, + exptItemResultRepo repo.IExptItemResultRepo, + exptItemRefRepo repo.IExptItemRefRepo, + exptTurnResultRepo repo.IExptTurnResultRepo, + configer component.IConfiger, + quotaRepo repo.QuotaRepo, + mutex lock.ILocker, + idem idem.IdempotentService, + publisher events.ExptEventPublisher, + audit audit.IAuditService, + idgen idgen.IIDGenerator, + metric metrics.ExptMetric, + lwt platestwrite.ILatestWriteTracker, + evaluationSetVersionService EvaluationSetVersionService, + evaluationSetService IEvaluationSetService, + evalTargetService IEvalTargetService, + evaluatorService EvaluatorService, + benefitService benefit.IBenefitService, + exptAggrResultService ExptAggrResultService, + templateRepo repo.IExptTemplateRepo, + templateManager IExptTemplateManager, + notifyRPCAdapter rpc.INotifyRPCAdapter, + userProvider rpc.IUserProvider, + pipelineListAdapter rpc.IPipelineListAdapter, + resourceAccessAuthorizer ResourceAccessAuthorizer, + sandboxAgentMetrics metrics.SandboxAgentMetrics, + centralScopeProvider component.ICentralSchedulerScopeProvider, + centralAdmissionPolicy component.ICentralAdmissionPolicy, + centralGuard component.ICentralReservationGuard, + manualRetryRepo repo.IExptManualRetryRepo, + evaluatorRecords EvaluatorRecordService, + currentRunWaker component.ICurrentRunSchedulerWaker, + manualRetryLocker ManualRetryLockerFactory, + runFinishRepo repo.IExptRunFinishRepo, +) IExptManager { + manager := NewExptManager(exptResultService, exptRepo, exptRunLogRepo, exptStatsRepo, exptItemResultRepo, exptItemRefRepo, exptTurnResultRepo, configer, quotaRepo, mutex, idem, publisher, audit, idgen, metric, lwt, evaluationSetVersionService, evaluationSetService, evalTargetService, evaluatorService, benefitService, exptAggrResultService, templateRepo, templateManager, notifyRPCAdapter, userProvider, pipelineListAdapter, resourceAccessAuthorizer, sandboxAgentMetrics, centralScopeProvider, centralAdmissionPolicy, centralGuard, manualRetryRepo, evaluatorRecords, currentRunWaker, manualRetryLocker).(*ExptMangerImpl) + manager.runFinishRepo = runFinishRepo + return manager +} diff --git a/backend/modules/evaluation/domain/service/expt_run_finish_test.go b/backend/modules/evaluation/domain/service/expt_run_finish_test.go new file mode 100644 index 0000000000..8c27e50dbc --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_finish_test.go @@ -0,0 +1,375 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/bytedance/gg/gptr" + idgenmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + infraredis "github.com/coze-dev/coze-loop/backend/infra/redis" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + mysqlmodel "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/redis/go-redis/v9" + + idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +type naturalFinishProbe struct { + IExptManager + calls int + disposition entity.ExptRunFinishDisposition + err error +} + +func (p *naturalFinishProbe) FinishCurrentRun(ctx context.Context, event *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + return p.FinishCurrentRunWithHeldTickLock(ctx, event) +} + +func (p *naturalFinishProbe) FinishCurrentRunWithHeldTickLock(context.Context, *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + p.calls++ + return &entity.ExptRunFinishResult{Disposition: p.disposition}, p.err +} + +// The real common mode endpoint must use the composite transaction after +// calibration, including the database pending disposition. +func TestRunFinishCommonEndpointUsesComposition(t *testing.T) { + for _, disposition := range []entity.ExptRunFinishDisposition{entity.ExptRunFinishPending, entity.ExptRunFinishFinished, entity.ExptRunFinishAlreadyFinished, entity.ExptRunFinishNonCurrent} { + t.Run(string(rune('0'+disposition)), func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := &naturalFinishProbe{IExptManager: svcmocks.NewMockIExptManager(ctrl), disposition: disposition} + idem := idemmocks.NewMockIdempotentService(ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{}).AnyTimes() + base := &exptBaseExec{Manager: manager, idem: idem, configer: config, resultSvc: &finishCalibrationTestResult{ready: true}} + err := base.exptEnd(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit}, &entity.Experiment{ExptType: entity.ExptType_Offline}) + if disposition == entity.ExptRunFinishPending { + require.True(t, errno.NeedMQRetry(err)) + } else { + require.NoError(t, err) + } + require.Equal(t, 1, manager.calls) + }) + } +} + +func TestRunFinishCommonCalibrationBlocksComposition(t *testing.T) { + for _, fault := range []error{nil, errors.New("calibration unavailable")} { + ctrl := gomock.NewController(t) + manager := &naturalFinishProbe{IExptManager: svcmocks.NewMockIExptManager(ctrl)} + idem := idemmocks.NewMockIdempotentService(ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + base := &exptBaseExec{Manager: manager, idem: idem, resultSvc: &finishCalibrationTestResult{err: fault}} + err := base.exptEnd(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, &entity.Experiment{ExptType: entity.ExptType_Offline}) + require.True(t, errno.NeedMQRetry(err)) + require.Zero(t, manager.calls) + } +} + +func TestRunFinishRescanIncludesLogged(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), int64(0), int64(1), int64(3)).DoAndReturn(func(context.Context, int64, int64, *entity.ExptItemRunLogFilter, int64, int64, int64) ([]*entity.ExptItemResultRunLog, int64, error) { + return nil, 0, nil + }) + items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), int64(0), int64(1), int64(3)).DoAndReturn(func(_ context.Context, _ int64, _ int64, filter *entity.ExptItemRunLogFilter, _ int64, _ int64, _ int64) ([]*entity.ExptItemResultRunLog, int64, error) { + require.NotNil(t, filter.ResultState) + require.Equal(t, entity.ExptItemResultStateLogged, *filter.ResultState) + return []*entity.ExptItemResultRunLog{{ItemID: 11}}, 1, nil + }) + pending, err := (&exptBaseExec{exptItemResultRepo: &finishCurrentItemRepo{IExptItemResultRepo: items}}).hasPendingAfterRescan(context.Background(), &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3}) + require.NoError(t, err) + require.True(t, pending) +} + +type finishRepoProbe struct { + repo.IExptRunFinishRepo + calls int + result *entity.ExptRunFinishResult + err error +} + +func (p *finishRepoProbe) FinishCurrentRun(context.Context, *entity.ExptRunFinishRequest) (*entity.ExptRunFinishResult, error) { + p.calls++ + return p.result, p.err +} + +func TestRunFinishManagerCommitFailureAndPendingHaveZeroCleanup(t *testing.T) { + for _, fault := range []error{errors.New("commit failed"), nil} { + ctrl := gomock.NewController(t) + port := &finishRepoProbe{err: fault, result: &entity.ExptRunFinishResult{Disposition: entity.ExptRunFinishPending}} + manager := &ExptMangerImpl{runFinishRepo: port, mutex: lockmocks.NewMockILocker(ctrl)} + result, err := manager.FinishCurrentRunWithHeldTickLock(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}) + if fault != nil { + require.Error(t, err) + require.Nil(t, result) + } else { + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishPending, result.Disposition) + } + require.Equal(t, 1, port.calls) + } +} + +func TestRunFinishManagerCurrentRepositoryFailsClosed(t *testing.T) { + ctrl := gomock.NewController(t) + manager := &ExptMangerImpl{itemResultRepo: &finishCurrentItemRepo{IExptItemResultRepo: repomocks.NewMockIExptItemResultRepo(ctrl)}, mutex: lockmocks.NewMockILocker(ctrl)} + _, err := manager.FinishCurrentRunWithHeldTickLock(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}) + require.Error(t, err) +} + +type finishCurrentItemRepo struct { + repo.IExptItemResultRepo + repo.IExptCurrentResultRepo +} + +func TestRunFinishManagerCleanupFailureReplaysAfterCommit(t *testing.T) { + ctrl := gomock.NewController(t) + port := &finishRepoProbe{result: &entity.ExptRunFinishResult{Disposition: entity.ExptRunFinishFinished, Experiment: &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Success, EndAt: gptr.Of(time.Now())}}} + locker := lockmocks.NewMockILocker(ctrl) + var locks atomic.Int32 + locker.EXPECT().UnlockWithValue(gomock.Any(), "expt_run_mutex_lock:1", "2").Return(false, nil).Times(2) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_run_mutex_lock:1", gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ string, _ time.Duration, _ time.Duration) (bool, context.Context, func(), error) { + if locks.Add(1) == 1 { + return false, ctx, func() {}, errors.New("Redis unavailable") + } + return true, ctx, func() {}, nil + }).Times(2) + expts := repomocks.NewMockIExperimentRepo(ctrl) + expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 99, Status: entity.ExptStatus_Processing}, nil) + manager := &ExptMangerImpl{runFinishRepo: port, mutex: locker, exptRepo: expts, manualRetryLocker: func() lock.ILocker { return locker }} + _, err := manager.FinishCurrentRunWithHeldTickLock(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, 1, port.calls) + port.result.Disposition = entity.ExptRunFinishAlreadyFinished + _, err = manager.FinishCurrentRunWithHeldTickLock(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}) + require.NoError(t, err) + require.Equal(t, 2, port.calls) +} + +func TestRunFinishFinishedSchedulerEventReplaysCleanup(t *testing.T) { + ctrl := gomock.NewController(t) + base := svcmocks.NewMockIExptManager(ctrl) + base.EXPECT().GetRunLog(gomock.Any(), int64(1), int64(2), int64(3), gomock.Any()).Return(&entity.ExptRunLog{Status: int64(entity.ExptStatus_Success)}, nil) + manager := &naturalFinishProbe{IExptManager: base, disposition: entity.ExptRunFinishAlreadyFinished} + scheduler := &ExptSchedulerImpl{Manager: manager} + err := scheduler.HandleEventCheck(func(context.Context, *entity.ExptScheduleEvent) error { + t.Fatal("finished event entered execution") + return nil + })(context.Background(), &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, ExptType: entity.ExptType_Offline, ExptRunMode: entity.EvaluationModeSubmit}) + require.NoError(t, err) + require.Equal(t, 1, manager.calls, "ACK swallowed failed postcommit cleanup retry") +} + +func TestRunFinishLogRunBindsLifecycleOwnerToRunID(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}) + locker.EXPECT().BackoffLockWithValue(gomock.Any(), "expt_run_mutex_lock:1", "2", gomock.Any(), time.Second).Return(true, "2", nil) + runs := repomocks.NewMockIExptRunLogRepo(ctrl) + runs.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) + expts := repomocks.NewMockIExperimentRepo(ctrl) + expts.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil) + metric := metricsmocks.NewMockExptMetric(ctrl) + metric.EXPECT().EmitExptExecRun(int64(3), int64(entity.EvaluationModeSubmit)) + manager := &ExptMangerImpl{runFinishRepo: &finishRepoProbe{}, mutex: locker, configer: config, runLogRepo: runs, exptRepo: expts, mtr: metric} + require.NoError(t, manager.LogRun(context.Background(), 1, 2, entity.EvaluationModeSubmit, 3, nil, &entity.Session{UserID: "test"})) +} + +func finishRealLockerFactory(t *testing.T) (ManualRetryLockerFactory, *miniredis.Miniredis) { + t.Helper() + server := miniredis.RunT(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { _ = raw.Close() }) + return func() lock.ILocker { return filterRealTestLocker(client) }, server +} + +func TestRunFinishCleanupDoesNotDeleteR2LifecycleOwner(t *testing.T) { + factory, server := finishRealLockerFactory(t) + key := "expt_run_mutex_lock:1" + require.NoError(t, server.Set(key, "99")) + manager := &ExptMangerImpl{manualRetryLocker: factory} + err := manager.cleanupNaturallyFinishedRun(context.Background(), &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3}, &entity.Experiment{ID: 1}) + require.True(t, errno.NeedMQRetry(err)) + owner, err := server.Get(key) + require.NoError(t, err) + require.Equal(t, "99", owner) + // A pre-upgrade random owner cannot be safely migrated or force deleted. + require.NoError(t, server.Set(key, "legacy-random-holder")) + err = manager.cleanupNaturallyFinishedRun(context.Background(), &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3}, &entity.Experiment{ID: 1}) + require.Error(t, err) + owner, _ = server.Get(key) + require.Equal(t, "legacy-random-holder", owner) +} + +// Exercise the real common ExptEnd under the scheduler mutex, with an actual +// SQL finisher and a postcommit failure. Retry cannot rewrite counts/endAt. +func TestRunFinishActualModesCommitThenCleanupRetry(t *testing.T) { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeSubmit, entity.EvaluationModeFailRetry, entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + ctrl := gomock.NewController(t) + require.NoError(t, f.conn.Model(&mysqlmodel.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.ExptStatus_Processing), "expt_type": int32(entity.ExptType_Offline)}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Processing)).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=?", f.space).Update("status", int32(entity.ItemRunState_Success)).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResultRunLog{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": 1}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResult{}).Where("space_id=?", f.space).Update("status", int32(entity.TurnRunState_Success)).Error) + factory, server := finishRealLockerFactory(t) + require.NoError(t, server.Set(fmt.Sprintf("expt_run_mutex_lock:%d", f.expt), fmt.Sprint(f.run))) + experiments := repomocks.NewMockIExperimentRepo(ctrl) + experiments.EXPECT().GetByID(gomock.Any(), f.expt, f.space).DoAndReturn(func(ctx context.Context, _, _ int64) (*entity.Experiment, error) { + var row mysqlmodel.Experiment + err := f.conn.WithContext(ctx).Where("space_id=? AND id=?", f.space, f.expt).Take(&row).Error + return &entity.Experiment{ID: f.expt, SpaceID: f.space, LatestRunID: row.LatestRunID, ExptType: entity.ExptType_Offline, Status: entity.ExptStatus(row.Status), EndAt: row.EndAt}, err + }).AnyTimes() + quota := repomocks.NewMockQuotaRepo(ctrl) + var releases atomic.Int32 + quota.EXPECT().CreateOrUpdate(gomock.Any(), f.space, gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ int64, mutate func(*entity.QuotaSpaceExpt) (*entity.QuotaSpaceExpt, bool, error), _ *entity.Session) error { + var row mysqlmodel.Experiment + require.NoError(t, f.conn.WithContext(ctx).Where("space_id=? AND id=?", f.space, f.expt).Take(&row).Error) + require.True(t, entity.IsExptFinished(entity.ExptStatus(row.Status)), "cleanup before finish commit") + // Cleanup's lifecycle key must reject a concurrent R2 creation. + locked, _, err := factory().BackoffLockWithValue(ctx, fmt.Sprintf("expt_run_mutex_lock:%d", f.expt), fmt.Sprint(f.run+1), time.Minute, time.Millisecond) + require.NoError(t, err) + require.False(t, locked) + if releases.Add(1) == 1 { + return errors.New("quota cleanup fault") + } + _, _, err = mutate(&entity.QuotaSpaceExpt{ExptID2RunTime: map[int64]int64{f.expt: 1}}) + return err + }).Times(2) + result := &finishCalibrationTestResult{ExptResultService: svcmocks.NewMockExptResultService(ctrl), ready: true} + result.ExptResultService.(*svcmocks.MockExptResultService).EXPECT().GetIncompleteTurns(gomock.Any(), f.expt, f.space, gomock.Any()).Return(nil, nil) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + publisher.EXPECT().PublishExptLifecycleEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + manager := &ExptMangerImpl{runFinishRepo: experimentrepo.NewExptRunFinishRepo(f.provider), manualRetryLocker: factory, exptRepo: experiments, quotaRepo: quota, exptResultService: result, publisher: publisher} + idem := idemmocks.NewMockIdempotentService(ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + manager.idem = idem + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{}).AnyTimes() + event := &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ExptRunMode: mode, ExptType: entity.ExptType_Offline} + items := f.itemRepo() + expt := &entity.Experiment{ID: f.expt, SpaceID: f.space, ExptType: entity.ExptType_Offline} + var end func(context.Context, *entity.ExptScheduleEvent, *entity.Experiment, int, int) (bool, error) + switch mode { + case entity.EvaluationModeSubmit: + end = (&ExptSubmitExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result}).ExptEnd + case entity.EvaluationModeFailRetry: + end = (&ExptFailRetryExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result}).ExptEnd + case entity.EvaluationModeRetryAll: + end = (&ExptRetryAllExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result}).ExptEnd + case entity.EvaluationModeRetryItems: + runs := repomocks.NewMockIExptRunLogRepo(ctrl) + runs.EXPECT().Get(gomock.Any(), f.expt, f.run).Return(&entity.ExptRunLog{}, nil).AnyTimes() + manager.runLogRepo = runs + manager.mutex = factory() + end = (&ExptRetryItemsExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, exptRunLogRepo: runs, resultSvc: result}).ExptEnd + } + scheduler := &ExptSchedulerImpl{Manager: manager, Mutex: factory()} + err := scheduler.HandleEventLock(func(ctx context.Context, event *entity.ExptScheduleEvent) error { + _, err := end(ctx, event, expt, 0, 0) + return err + })(context.Background(), event) + require.True(t, errno.NeedMQRetry(err)) + var first mysqlmodel.Experiment + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&first).Error) + require.NotNil(t, first.EndAt) + before := lateWriteSnapshot(f) + retryResult, retryErr := manager.FinishCurrentRun(context.Background(), event) + require.NoError(t, retryErr) + require.Equal(t, entity.ExptRunFinishAlreadyFinished, retryResult.Disposition) + require.Equal(t, before, lateWriteSnapshot(f)) + require.Equal(t, int32(2), releases.Load()) + }) + } +} + +func lateWriteSnapshot(f *lateWriteFixture) map[string]any { + result := map[string]any{} + for _, table := range []string{"experiment", "expt_run_log", "expt_stats"} { + var rows []map[string]any + require.NoError(f.t, f.conn.Table(table).Where("space_id=?", f.space).Find(&rows).Error) + result[table] = rows + } + return result +} + +func TestRunFinishDirectAcquiresTickAndLeaseLossStopsSQL(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + port := &finishRepoProbe{} + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_run_exec_lock:1:2", 5*time.Second, 5*time.Minute).DoAndReturn(func(ctx context.Context, _ string, _ time.Duration, _ time.Duration) (bool, context.Context, func(), error) { + child, cancel := context.WithCancel(ctx) + cancel() + return true, child, cancel, nil + }) + manager := &ExptMangerImpl{runFinishRepo: port, manualRetryLocker: func() lock.ILocker { return locker }} + _, err := manager.FinishCurrentRun(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}) + require.True(t, errno.NeedMQRetry(err)) + require.Zero(t, port.calls) +} + +func TestRunFinishHeldHelperCancelledLeaseStopsSQL(t *testing.T) { + port := &finishRepoProbe{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + manager := &ExptMangerImpl{runFinishRepo: port} + _, err := manager.FinishCurrentRunWithHeldTickLock(ctx, &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}) + require.True(t, errno.NeedMQRetry(err)) + require.Zero(t, port.calls) +} + +func TestRunFinishCleanupLostLeaseStopsQuotaAndNotifications(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().UnlockWithValue(gomock.Any(), "expt_run_mutex_lock:1", "2").Return(true, nil) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_run_mutex_lock:1", 5*time.Second, 5*time.Minute).DoAndReturn(func(ctx context.Context, _ string, _ time.Duration, _ time.Duration) (bool, context.Context, func(), error) { + child, cancel := context.WithCancel(ctx) + cancel() + return true, child, cancel, nil + }) + manager := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return locker }, quotaRepo: repomocks.NewMockQuotaRepo(ctrl), publisher: eventmocks.NewMockExptEventPublisher(ctrl), exptRepo: repomocks.NewMockIExperimentRepo(ctrl)} + require.Error(t, manager.cleanupNaturallyFinishedRun(context.Background(), &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3}, &entity.Experiment{ID: 1})) +} + +// Numeric ownership on Submit must not broaden legacy live RetryItems join. +func TestRunFinishNumericSubmitOwnerDoesNotAllowRetryItemsJoin(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + ids := idgenmocks.NewMockIIDGenerator(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}) + ids.EXPECT().GenID(gomock.Any()).Return(int64(99), nil) + locker.EXPECT().BackoffLockWithValue(gomock.Any(), "expt_run_mutex_lock:1", "99", gomock.Any(), time.Second).Return(false, "2", nil) + locker.EXPECT().Exists(gomock.Any(), "expt_completing_mutex_lock:1:2").Return(false, nil).AnyTimes() + runs := repomocks.NewMockIExptRunLogRepo(ctrl) + runs.EXPECT().Get(gomock.Any(), int64(1), int64(2)).Return(&entity.ExptRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, Mode: int32(entity.EvaluationModeSubmit), Status: int64(entity.ExptStatus_Processing)}, nil).AnyTimes() + manager := &ExptMangerImpl{runFinishRepo: &finishRepoProbe{}, mutex: locker, configer: config, idgenerator: ids, runLogRepo: runs} + _, _, err := manager.LogRetryItemsRun(context.Background(), 1, entity.EvaluationModeRetryItems, 3, []int64{11}, &entity.Session{UserID: "test"}) + require.Error(t, err) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go index 8d1b187bbe..cea3c12a84 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go @@ -44,6 +44,7 @@ type ExptItemEventEvalServiceImpl struct { configer component.IConfiger quotaRepo repo.QuotaRepo mutex lock.ILocker + itemLocker ManualRetryLockerFactory idem idem.IdempotentService auditClient audit.IAuditService metric metrics.ExptMetric @@ -132,6 +133,9 @@ func NewExptRecordEvalService( i.sandboxAgentNotifier = sandboxAgentNotifier[0] } + if m, ok := manager.(*ExptMangerImpl); ok { + i.itemLocker = m.manualRetryLocker + } i.endpoints = RecordEvalChain( i.HandleEventErr, i.HandleEventCheck, @@ -139,6 +143,7 @@ func NewExptRecordEvalService( // 而放在 Lock 之前可避免为一条注定要丢弃的消息（不归本进程 / 无 Scope）去抢 item 锁。 i.HandleCentralAdmission, i.HandleEventLock, + i.HandleCurrentSource, // ★ 取执行权与兑现投影必须在锁内：见 HandleCentralReservation 注释。 i.HandleCentralReservation, i.HandleEventExec, @@ -173,9 +178,9 @@ func RecordEvalChain(mws ...RecordEvalMiddleware) RecordEvalMiddleware { func (e *ExptItemEventEvalServiceImpl) HandleEventCheck(next RecordEvalEndPoint) RecordEvalEndPoint { return func(ctx context.Context, event *entity.ExptItemEvalEvent) error { - runLog, err := e.manager.GetRunLog(ctx, event.ExptID, event.ExptRunID, event.SpaceID, event.Session) + runLog, err := e.manager.GetRunLog(contexts.WithCtxWriteDB(ctx), event.ExptID, event.ExptRunID, event.SpaceID, event.Session) if err != nil { - return err + return errno.WrapMQRetryErr(err) } if status := entity.ExptStatus(runLog.Status); entity.IsExptFinished(status) || entity.IsExptFinishing(status) { @@ -183,20 +188,11 @@ func (e *ExptItemEventEvalServiceImpl) HandleEventCheck(next RecordEvalEndPoint) return nil } - // item 级闸门（实验级判定之后、正常执行之前）：该行已被用户行级终止 (TerminateItems) 则丢弃事件， - // 对应「排队中的行被终止后不再被调度」。判定失败只告警不拦截 —— 宁可多跑一次， - // 也不能因为一次查询抖动把正常行的执行掐掉（终止行即便漏拦，也有 CompleteItemRun 的 Terminal 覆盖保护兜底）。 - if e.isItemTerminated(ctx, event) { - logs.CtxInfo(ctx, "[ExptItemTerminate] drop event of terminated item, expt_id: %v, expt_run_id: %v, item_id: %v", event.ExptID, event.ExptRunID, event.EvalSetItemID) - return nil - } - return next(ctx, event) } } -// isItemTerminated 查该 item 当前 run log 是否已处于 Terminal。查询失败返回 false（不拦截），见 HandleEventCheck 注释。 -// 强制读主库（WithCtxWriteDB）：从库延迟会让刚写下的 Terminal 读不到，闸门形同虚设。 +// isItemTerminated preserves the existing retry-yield terminal check on the primary database. func (e *ExptItemEventEvalServiceImpl) isItemTerminated(ctx context.Context, event *entity.ExptItemEvalEvent) bool { if event == nil || e.exptItemResultRepo == nil { return false @@ -229,7 +225,7 @@ func (e *ExptItemEventEvalServiceImpl) HandleCentralAdmission(next RecordEvalEnd return func(ctx context.Context, event *entity.ExptItemEvalEvent) error { expt, err := e.experimentRepo.GetByID(ctx, event.ExptID, event.SpaceID) if err != nil { - return err + return errno.WrapMQRetryErr(err) } if !entity.IsCentralDispatch(expt.ExptDispatchMode) { // legacy 实验：完全不经额度闸，行为与改动前一致 @@ -260,7 +256,7 @@ func (e *ExptItemEventEvalServiceImpl) HandleCentralAdmission(next RecordEvalEnd owned, err := e.centralScopeOwner.OwnsSchedulerScope(ctx, expt.SchedulerScope) if err != nil { // 无法判定归属：返回错误重试，而不是赌一把放行。 - return err + return errno.WrapMQRetryErr(err) } if !owned { logs.CtxWarn(ctx, "[CentralReservation] scheduler scope not owned by this instance, discard event, expt_scope: %v, expt_id: %v, item_id: %v", @@ -309,7 +305,13 @@ func (e *ExptItemEventEvalServiceImpl) HandleCentralReservation(next RecordEvalE return nil } - ok, err := guard.ConfirmRunning(ctx, expt.SchedulerScope, event.ExptRunID, event.EvalSetItemID) + // Direct callers with an item repository use the same source/turn checks + // as the production HandleCurrentSource middleware before touching quota. + if execution := executionSource(ctx); e.exptItemResultRepo != nil && (execution == nil || !execution.SourceValidated) { + return e.HandleCurrentSource(e.HandleCentralReservation(next))(ctx, event) + } + event = event.Clone() + ok, err := guard.ConfirmRunning(ctx, expt.SchedulerScope, event.ExptRunID, event.EvalSetItemID, event.RequestID) if err != nil { // 账本暂时不可用：返回错误让 MQ 重试，而不是丢弃 —— item 已被预占， // 丢弃会让它停在 Queueing 直到 reservation 超时清理，白等一轮。 @@ -322,123 +324,19 @@ func (e *ExptItemEventEvalServiceImpl) HandleCentralReservation(next RecordEvalE return nil } - // 把 run log 投影从 Queueing/reserved 兑现为 Processing/none。 - // - // 为什么必须在这里而不是留给下游 handleEventExec：投影是调度器算并发占用的依据， - // 若 item 已开始执行而投影仍停在 Queueing/reserved，下一拍会把它当"已预占未消费" - // 继续计入占用（这没错），但一旦 reservation 因超时被清理，它就变成"既不 Processing - // 也无 reservation"的孤儿，对账要多绕一圈才能修。就地兑现让两侧同步收敛。 - // - // CAS 未命中（started=false）不阻断执行：可能是重复投递（已 Processing）或 - // 投影已被 repair 修正。此时 reservation 校验已通过，说明额度是真的，继续执行是安全的。 - // 现在本层在 item 锁内，同一 item 不会有并发的第二个执行者，未命中只剩"已 Processing - // 的重复投递"与"被 repair 修正"两种，两者继续执行都由 item 锁 + 幂等写兜住。 + // The DAO commits canonical/turn/log/stats together. Only a current + // nonterminal source can continue into the existing recovery path. if e.dispatchRepo != nil { - started, err := e.dispatchRepo.StartReservedItem(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID) + started, err := e.dispatchRepo.StartReservedItem(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.RequestID) if err != nil { - // 投影写失败：返回错误让 MQ 重试。额度已预占且 reservation 已转 Running， - // 丢弃消息会让这份额度占着直到超时清理。 return err } - if !started { - logs.CtxInfo(ctx, "[CentralReservation] run log projection not claimed (duplicate delivery or repaired), continue, expt_run_id: %v, item_id: %v", - event.ExptRunID, event.EvalSetItemID) - } - - // 主表 expt_item_result 也要跟着推进到 Processing。 - // - // run log 是执行真值，但**用户看到的是主表** —— MGetExperimentResult 走 - // expt_item_result 构造 run_state。只写 run log 会让实验详情把正在执行的 item - // 一直显示成 Queueing（实测：5 个 item 已 Processing 14 小时，results 接口仍全部 - // 报 queueing），现象是"实验看着没动"，而它其实跑得很正常。 - // - // legacy 的 handleToSubmits 一直是成对写这两张表的 - // （expt_run_scheduler_event_impl.go 的 UpdateItemRunLog + UpdateItemsResult）， - // 中心调度这条新派发路径漏了后者 —— 平行实现漏字段的又一例。 - // - // 失败只告警不阻断：主表是展示投影，写不进去不影响执行与额度正确性， - // 而这里返回错误会让已经拿到执行权的 item 被 MQ 重投一遍。 - // - // ★ 记账与主表推进必须**同一个判据**，见下方 expt_stats 那段的论证。 - // 所以这里先读主表当前状态，只有真的发生「非 Processing → Processing」才动两处。 - prevState, prevKnown := entity.ItemRunState_Unknown, false - if e.exptItemResultRepo != nil { - if got, gerr := e.exptItemResultRepo.BatchGet(ctx, event.SpaceID, event.ExptID, []int64{event.EvalSetItemID}); gerr != nil { - logs.CtxWarn(ctx, "[CentralReservation] read main table state failed, skip advancing (display only), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, gerr) - } else if len(got) > 0 && got[0] != nil { - prevState, prevKnown = got[0].Status, true - } - } - - advanced := prevKnown && prevState != entity.ItemRunState_Processing - if advanced { - if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, - []int64{event.EvalSetItemID}, map[string]any{"status": int32(entity.ItemRunState_Processing)}); err != nil { - logs.CtxWarn(ctx, "[CentralReservation] advance main table to Processing failed (display only, execution unaffected), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, err) - advanced = false - } - } - - // expt_stats 的 Queueing→Processing 也要在此记账，否则计数单向下溢。 - // - // 完成侧（expt_result_impl.go 的 statsCntOp）做的是「从 item 原状态减 1、往新状态加 1」， - // 它减的是 Processing 桶。legacy 在 handleToSubmits 里派发时就把 item 计入了 Processing， - // 两边配对；中心调度的派发只把 run log CAS 成 Queueing/reserved（**status 仍是 Queueing**）， - // 从未有人往 Processing 桶加过 —— 于是完成时减一个从未加过的计数， - // 表现为 processing_turn_count 走负、pending_turn_count 永不下降。 - // 实测：一个 14 题 enforce 实验 fail 累到 4 时 processing = -4，pending 恒 14。 - // - // ★ 判据是**主表真的发生了状态迁移**，不是 run log 的 CAS。 - // - // 原先绑在 started（run log CAS）上，而主表推进是无条件的 —— 两个判据一分叉， - // 计数行就与主表对不上。而完成侧 statsCntOp 恰恰是**按主表状态**做「-1」的 - // （`statsCntOp[itemResult.Status] -= 1`，见 expt_result_impl.go）， - // 于是它会去减一个计数行里从来没加过的桶。 - // 实测代价：PPE 一个 900 题 enforce 实验，pending 虚高 281、success 少 249， - // 而同泳道的 legacy 实验分毫不差 —— legacy 的派发是四张表无条件一起写的。 - // - // 用 prevState 而不是写死 Queueing：中心调度派发时主表可能停在 Queueing， - // 也可能因为 repair/重投停在别的状态，减错桶同样会让计数行歪掉。 - // - // 与主表同为展示投影，失败只告警不阻断：返回错误会让已取得执行权的 item 被 MQ 重投， - // 而重投一次 Agent 执行的代价远大于一次计数偏差。**残留的偏差由调度器每拍的对账收敛** - // （ExptSchedulerImpl.reconcileExptStats）。 - if advanced && e.exptStatsRepo != nil { - if err := e.exptStatsRepo.ArithOperateCount(ctx, event.ExptID, event.SpaceID, &entity.StatsCntArithOp{ - OpStatusCnt: map[entity.ItemRunState]int{ - entity.ItemRunState_Processing: 1, - prevState: -1, - }, - }); err != nil { - logs.CtxWarn(ctx, "[CentralReservation] advance expt stats to Processing failed (display only, execution unaffected), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, err) - } - } - - // turn 主表与 CK 加速表同为展示投影，legacy 的 handleToSubmits 派发时是与 - // run log / 主表 / stats 一起五写的，中心调度这条路径此前只写了前三项。 - // - // 漏 CK 的后果最隐蔽：`item_run_state` 筛选打的是 etrf.status，且开启加速器时 - // 结果里的 run_state 也从 CK 读（QueryItemIDStates），于是执行期间按「运行中」筛 - // 恒为空、按「排队中」筛反而捞出正在跑的 item 并显示成排队中。完成时那次 upsert - // 会把它纠回来，所以事后对着终态数据看不出问题。 - // 实测：PPE 一个 enforce 实验，主表已 status=1，CK 仍为 0。 - // - // 必须在主表推进之后调：CK 行的 status 是 BuildTurnResultFilter 按主表状态回填的。 - // 与上面两项同样只告警不阻断（返回错误会让已取得执行权的 item 被 MQ 重投）。 - if advanced && e.exptTurnResultRepo != nil { - if err := e.exptTurnResultRepo.UpdateTurnResultsWithItemIDs(ctx, event.ExptID, []int64{event.EvalSetItemID}, event.SpaceID, - map[string]any{"status": int32(entity.TurnRunState_Processing)}); err != nil { - logs.CtxWarn(ctx, "[CentralReservation] advance turn results to Processing failed (display only, execution unaffected), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, err) - } + if !started.CurrentRequest { + return nil } - if advanced && e.resultSvc != nil { - if err := e.resultSvc.UpsertExptTurnResultFilter(ctx, event.SpaceID, event.ExptID, []int64{event.EvalSetItemID}); err != nil { - logs.CtxWarn(ctx, "[CentralReservation] upsert turn result filter failed (display only, execution unaffected), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, err) + if started.Started && e.resultSvc != nil { + if err := refreshFilterWithHeldItemLock(ctx, e.resultSvc, event.SpaceID, event.ExptID, event.EvalSetItemID); err != nil { + logs.CtxWarn(ctx, "[CentralReservation] refresh committed Processing filter failed, expt_id: %v, item_id: %v: %v", event.ExptID, event.EvalSetItemID, err) } } } @@ -463,8 +361,21 @@ func (e *ExptItemEventEvalServiceImpl) HandleCentralReservation(next RecordEvalE // 而不必把 Scope / guard 这些中心调度概念泄进 HandleEventErr 的签名。 event.WithCtxCentralQuotaHeld(ctx, expt.SchedulerScope) + releaseEvent := event.Clone() execErr := next(ctx, event) - e.releaseQuotaIfItemTerminal(ctx, event, expt.SchedulerScope, guard, execErr) + if errno.NeedMQRetry(execErr) || errors.Is(execErr, errExptNonCurrentSource) { + if execution := executionSource(ctx); execution != nil && execution.Completed && ctx.Err() != nil { + e.releaseQuotaIfItemTerminal(ctx, releaseEvent, expt.SchedulerScope, guard, execErr) + } + return execErr + } + if err := ctx.Err(); err != nil { + if execution := executionSource(ctx); execution != nil && execution.Completed { + e.releaseQuotaIfItemTerminal(ctx, releaseEvent, expt.SchedulerScope, guard, execErr) + } + return errno.WrapMQRetryErr(err) + } + e.releaseQuotaIfItemTerminal(ctx, releaseEvent, expt.SchedulerScope, guard, execErr) return execErr } } @@ -510,7 +421,7 @@ func (e *ExptItemEventEvalServiceImpl) requeueOrphanedItemOnReservationAbsent(ct } if state == entity.ItemRunState_Processing { - requeued, rerr := e.dispatchRepo.RequeueProcessingItem(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID) + requeued, rerr := e.dispatchRepo.RequeueProcessingItem(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.RequestID) if rerr != nil { logs.CtxError(ctx, "[CentralReservation] requeue orphaned processing item failed, item will occupy a concurrency slot until zombie timeout, expt_id: %v, expt_run_id: %v, item_id: %v, err: %v", event.ExptID, event.ExptRunID, event.EvalSetItemID, rerr) @@ -519,50 +430,8 @@ func (e *ExptItemEventEvalServiceImpl) requeueOrphanedItemOnReservationAbsent(ct logs.CtxWarn(ctx, "[CentralReservation] orphaned item (Processing without reservation) requeued to Queueing, expt_id: %v, expt_run_id: %v, item_id: %v, requeued: %v", event.ExptID, event.ExptRunID, event.EvalSetItemID, requeued) - // stats 的 Processing 桶必须同步退回，否则 processing_turn_count 只增不减。 - // 只在 CAS 真的命中时记账 —— 与 StartReservedItem 处 started 的用法同理， - // CAS 是这里唯一的"恰好一次"信号。 - if requeued && e.exptStatsRepo != nil { - if serr := e.exptStatsRepo.ArithOperateCount(ctx, event.ExptID, event.SpaceID, &entity.StatsCntArithOp{ - OpStatusCnt: map[entity.ItemRunState]int{ - entity.ItemRunState_Processing: -1, - entity.ItemRunState_Queueing: 1, - }, - }); serr != nil { - logs.CtxWarn(ctx, "[CentralReservation] rollback expt stats to Queueing failed (display only), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, serr) - } - } - - // 主表同为展示投影，跟着退回，避免详情页把已回队列的 item 一直显示成执行中。 - // - // ★ 必须与 stats 一样绑定 requeued：主表 status 不是纯展示字段，而是 stats 的锚点 —— - // 完成侧 statsCntOp 读 items_result.Status 做「-1」（expt_result_impl.go）， - // 若 CAS 未命中（并发 zombie / sandbox sweep 抢先落终态）却把主表改成 Queueing， - // stats 上那笔 Processing 就再也减不掉，processing_turn_count 永远归不了零。 - // 同仓 expt_run_scheduler_event_impl.go 的 zombie 路径已就此写过警示、 - // c4a6a953 也已就此修过一次，这里不能再踩。 - if requeued && e.exptItemResultRepo != nil { - if uerr := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, - []int64{event.EvalSetItemID}, map[string]any{"status": int32(entity.ItemRunState_Queueing)}); uerr != nil { - logs.CtxWarn(ctx, "[CentralReservation] rollback main table to Queueing failed (display only), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, uerr) - } - } - - // turn 主表与 CK 加速表也要跟着退回：派发侧推进的是这五项，回退只做前三项就会留下 - // 「item 已回队列、turn 与 CK 仍显示执行中」的错位。它不会自愈 —— 实验若以 Failed - // 收口，CompleteExpt 的 default 分支只销毁沙箱、不动 turn 状态。 - // 同样绑定 requeued（判据与 stats、主表一致），失败只告警。 - if requeued && e.exptTurnResultRepo != nil { - if uerr := e.exptTurnResultRepo.UpdateTurnResultsWithItemIDs(ctx, event.ExptID, []int64{event.EvalSetItemID}, event.SpaceID, - map[string]any{"status": int32(entity.TurnRunState_Queueing)}); uerr != nil { - logs.CtxWarn(ctx, "[CentralReservation] rollback turn results to Queueing failed (display only), expt_id: %v, item_id: %v: %v", - event.ExptID, event.EvalSetItemID, uerr) - } - } if requeued && e.resultSvc != nil { - if uerr := e.resultSvc.UpsertExptTurnResultFilter(ctx, event.SpaceID, event.ExptID, []int64{event.EvalSetItemID}); uerr != nil { + if uerr := refreshFilterWithHeldItemLock(ctx, e.resultSvc, event.SpaceID, event.ExptID, event.EvalSetItemID); uerr != nil { logs.CtxWarn(ctx, "[CentralReservation] rollback turn result filter failed (display only), expt_id: %v, item_id: %v: %v", event.ExptID, event.EvalSetItemID, uerr) } @@ -572,7 +441,7 @@ func (e *ExptItemEventEvalServiceImpl) requeueOrphanedItemOnReservationAbsent(ct // 剩下只有 Queueing：reserved 的清回 none 让它重新可授予；none 的本就在候选里，无需动作。 if obs[0].QuotaReservationState.IsQuotaReserved() { - reset, rerr := e.dispatchRepo.ResetQuotaReserved(ctx, event.SpaceID, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}) + reset, rerr := e.dispatchRepo.ResetQuotaReserved(ctx, event.SpaceID, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}, map[int64]string{event.EvalSetItemID: event.RequestID}) if rerr != nil { logs.CtxError(ctx, "[CentralReservation] reset stale reserved projection failed, expt_id: %v, expt_run_id: %v, item_id: %v, err: %v", event.ExptID, event.ExptRunID, event.EvalSetItemID, rerr) @@ -610,12 +479,16 @@ func (e *ExptItemEventEvalServiceImpl) releaseQuotaIfItemTerminal( guard component.ICentralReservationGuard, execErr error, ) { + if execution := executionSource(ctx); execution != nil && execution.QuotaReleased { + return + } if guard == nil || e.dispatchRepo == nil { return } + event = quotaReleaseSourceEvent(ctx, event) // 用独立超时的 ctx：主 ctx 可能已因执行失败被取消，而释放必须尽力完成。 - relCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + relCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) defer cancel() obs, err := e.dispatchRepo.MGetDispatchObservations(relCtx, event.SpaceID, event.ExptID, event.ExptRunID, @@ -625,11 +498,14 @@ func (e *ExptItemEventEvalServiceImpl) releaseQuotaIfItemTerminal( event.ExptRunID, event.EvalSetItemID, err) return } - if len(obs) == 0 { + if len(obs) != 1 || obs[0] == nil { // run log 查不到：item 记录已被清理（重跑清表等）。此时 reservation 也该走， // 交由对账处理 —— 这里贸然释放可能释放的是新一轮 run 的额度。 return } + if obs[0].ItemID != event.EvalSetItemID || obs[0].RequestID != event.RequestID { + return + } if !entity.IsItemRunFinished(entity.ItemRunState(obs[0].Status)) { // 仍在执行 / 等待重投：保留 reservation。 @@ -637,11 +513,14 @@ func (e *ExptItemEventEvalServiceImpl) releaseQuotaIfItemTerminal( } reason := terminalReleaseReason(entity.ItemRunState(obs[0].Status), execErr) - if err := guard.Release(relCtx, schedulerScope, event.ExptRunID, event.EvalSetItemID, reason); err != nil { + if err := guard.Release(relCtx, schedulerScope, event.ExptRunID, event.EvalSetItemID, event.RequestID, reason); err != nil { logs.CtxWarn(relCtx, "[CentralReservation] release quota fail, expt_run_id: %v, item_id: %v, err: %v", event.ExptRunID, event.EvalSetItemID, err) return } + if execution := executionSource(ctx); execution != nil { + execution.QuotaReleased = true + } logs.CtxInfo(relCtx, "[CentralReservation] quota released on item terminal, scope: %v, expt_run_id: %v, item_id: %v, reason: %v", schedulerScope, event.ExptRunID, event.EvalSetItemID, reason) } @@ -685,11 +564,37 @@ func terminalReleaseReason(status entity.ItemRunState, execErr error) string { func (e *ExptItemEventEvalServiceImpl) HandleEventErr(next RecordEvalEndPoint) RecordEvalEndPoint { return func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + execution := ¤tItemExecution{} + ctx = context.WithValue(ctx, currentItemSourceKey{}, execution) nextErr := func(ctx context.Context, event *entity.ExptItemEvalEvent) (err error) { defer goroutine.Recover(ctx, &err) return next(ctx, event) }(ctx, event) + if event.ExptRunMode == entity.EvaluationModeFailRetry && errors.Is(nextErr, errRetryStartDependencyFailure) { + return errno.WrapMQRetryErr(nextErr) + } + if errno.NeedMQRetry(nextErr) { + return nextErr + } + if errors.Is(nextErr, errExptNonCurrentSource) { + return nil + } + if execution.LockLost { + return errno.NewMQRetryErr("item execution lock lost") + } + if port, ok := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + if nextErr == nil { + if e.metric != nil { + e.metric.EmitItemExecResult(event.SpaceID, int64(event.ExptRunMode), false, false, false, 0, event.CreateAt) + } + return nil + } + return e.handleCurrentEventErr(ctx, event, nextErr, port, execution) + } retryConf := e.configer.GetErrRetryConf(ctx, event.SpaceID, nextErr) needRetry := event.RetryTimes < retryConf.GetRetryTimes() if event.MaxRetryTimes > 0 { @@ -795,6 +700,9 @@ func (e *ExptItemEventEvalServiceImpl) HandleEventErr(next RecordEvalEndPoint) R // // best-effort：释放失败只告警。让"额度归还失败"阻断终态收口会把额度泄漏升级成实验不收敛。 func (e *ExptItemEventEvalServiceImpl) releaseCentralQuotaOutsideGate(ctx context.Context, event *entity.ExptItemEvalEvent, evalErr error, reasonPrefix string) { + if execution := executionSource(ctx); execution != nil && execution.QuotaReleased { + return + } if event == nil || evalErr == nil || e.centralGuard == nil { return } @@ -804,21 +712,35 @@ func (e *ExptItemEventEvalServiceImpl) releaseCentralQuotaOutsideGate(ctx contex // 没持有预占：legacy 实验，或消息在取得执行权之前就被丢弃/失败了。 return } + event = quotaReleaseSourceEvent(ctx, event) // 独立超时的 ctx：主 ctx 可能已因执行失败被取消，而释放必须尽力完成。 - relCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + relCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) defer cancel() reason := reasonPrefix + ": " + evalErr.Error() - if err := e.centralGuard.Release(relCtx, schedulerScope, event.ExptRunID, event.EvalSetItemID, reason); err != nil { + if err := e.centralGuard.Release(relCtx, schedulerScope, event.ExptRunID, event.EvalSetItemID, event.RequestID, reason); err != nil { logs.CtxWarn(relCtx, "[CentralReservation] release quota outside gate fail, scope: %v, expt_run_id: %v, item_id: %v, err: %v", schedulerScope, event.ExptRunID, event.EvalSetItemID, err) return } + if execution := executionSource(ctx); execution != nil { + execution.QuotaReleased = true + } logs.CtxInfo(relCtx, "[CentralReservation] quota released outside gate, scope: %v, expt_run_id: %v, item_id: %v, reason: %v", schedulerScope, event.ExptRunID, event.EvalSetItemID, reason) } +func quotaReleaseSourceEvent(ctx context.Context, event *entity.ExptItemEvalEvent) *entity.ExptItemEvalEvent { + event = event.Clone() + if execution := executionSource(ctx); execution != nil && execution.Source != nil { + current := execution.Source.Current + event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID = current.SpaceID, current.ExptID, current.RunID, current.ItemID + event.RequestID = current.RequestID + } + return event +} + // completeItemRunOnUnretriableErr 将 item 落为 Fail 并写入错误信息。 // 仅在"失败且不可重试"时调用; 写库失败只告警不影响主流程(僵尸清理仍是最后防线)。 // @@ -833,7 +755,22 @@ func (e *ExptItemEventEvalServiceImpl) completeItemRunOnUnretriableErr(ctx conte return } - persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + if err := ctx.Err(); err != nil { + return + } + if port, ok := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + source, err := sourceForExecution(ctx, port, event) + if err != nil { + logs.CtxWarn(ctx, "unretriable error source unavailable: %v", err) + return + } + if _, err := port.CompleteCurrentItemRun(ctx, source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, ErrMsg: []byte(errno.SerializeErr(evalErr)), RetryTimes: int32(event.RetryTimes), FailPendingTurns: true}); err != nil { + logs.CtxWarn(ctx, "unretriable error storage: %v", err) + } + return + } + + persistCtx, cancel := context.WithTimeout(ctx, exptRunLogPersistTimeout) defer cancel() // Terminal 是吸收态：该行已被用户行级终止时，在途的失败结果 MUST NOT 把 status / err_msg 回写成 Fail。 @@ -898,7 +835,22 @@ func (e *ExptItemEventEvalServiceImpl) yieldItemRunForRetry(ctx context.Context, return } - persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + if err := ctx.Err(); err != nil { + return + } + if port, ok := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + source, err := sourceForExecution(ctx, port, event) + if err != nil { + logs.CtxWarn(ctx, "yield source unavailable: %v", err) + return + } + if _, err := port.CompleteCurrentItemRun(ctx, source, &entity.ExptCurrentItemRunWrite{Yield: true, ErrMsg: []byte(errno.SerializeErr(evalErr)), RetryTimes: int32(event.RetryTimes) + 1}); err != nil { + logs.CtxWarn(ctx, "yield storage: %v", err) + } + return + } + + persistCtx, cancel := context.WithTimeout(ctx, exptRunLogPersistTimeout) defer cancel() itemIDs := []int64{event.EvalSetItemID} @@ -955,25 +907,35 @@ func (e *ExptItemEventEvalServiceImpl) yieldItemRunForRetry(ctx context.Context, func (e *ExptItemEventEvalServiceImpl) HandleEventLock(next RecordEvalEndPoint) RecordEvalEndPoint { return func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + mutex := e.mutex + if e.itemLocker != nil { + mutex = e.itemLocker() + } + if mutex == nil { + return errno.NewMQRetryErr("item execution locker missing") + } lockKey := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", event.ExptID, event.EvalSetItemID) - locked, ctx, cancel, err := e.mutex.LockWithRenew(ctx, lockKey, time.Second*5, time.Second*60*60) + locked, lockCtx, cancel, err := mutex.LockWithRenew(ctx, lockKey, 5*time.Second, time.Hour) if err != nil { - return err + return errno.WrapMQRetryErr(err) } - if !locked { - logs.CtxWarn(ctx, "ExptRecordEvalConsumer.HandleEventLock found locked item eval event: %v. Abort event, err: %v", json.Jsonify(event), err) - return nil + return errno.NewMQRetryErr("item execution lock busy") } - defer func() { cancel() - if _, err := e.mutex.Unlock(lockKey); err != nil { - logs.CtxWarn(ctx, "failed to unlock key: %v, err: %v", lockKey, err) + if _, err := mutex.Unlock(lockKey); err != nil { + logs.CtxWarn(ctx, "unlock item execution: %v", err) } }() - - return next(ctx, event) + err = next(lockCtx, event) + if lockCtx.Err() != nil { + if execution := executionSource(ctx); execution != nil { + execution.LockLost = true + } + return errno.WrapMQRetryErr(lockCtx.Err()) + } + return err } } @@ -992,7 +954,16 @@ func (e *ExptItemEventEvalServiceImpl) eval(ctx context.Context, event *entity.E return err } + if err := validateManualFrozenTurns(eiec); err != nil { + return err + } + if inflight, err := e.ordinaryInvocationInFlight(ctx, eiec); err != nil || inflight { + return err + } ctx = e.WithCtx(ctx, eiec) + if execution := executionSource(ctx); execution != nil { + execution.LogCtx = ctx + } mode, err := NewRecordEvalMode( eiec.Event, @@ -1013,6 +984,11 @@ func (e *ExptItemEventEvalServiceImpl) eval(ctx context.Context, event *entity.E if err := mode.PreEval(ctx, eiec); err != nil { return err } + if execution := executionSource(ctx); execution != nil && execution.Source != nil && len(execution.Source.Turns) == 0 && event.RequestID == "" { + for id, log := range eiec.GetExistTurnResultLogs() { + execution.Source.Turns[id] = log + } + } if err := NewExptItemEvaluation(e.exptTurnResultRepo, e.exptItemResultRepo, e.configer, e.metric, e.evaTargetService, e.evaluatorRecordService, e.evaluatorService, e.benefitService, e.evalAsyncRepo, e.evaluationSetItemService, e.itemCompletePublisher, e.sandboxAgentMetrics, e.sandboxAgentNotifier). Eval(ctx, eiec); err != nil { @@ -1033,7 +1009,7 @@ func (e *ExptItemEventEvalServiceImpl) WithCtx(ctx context.Context, eiec *entity func (e *ExptItemEventEvalServiceImpl) BuildExptRecordEvalCtx(ctx context.Context, event *entity.ExptItemEvalEvent) (*entity.ExptItemEvalCtx, error) { exptDetail, err := e.manager.GetDetail(ctx, event.ExptID, event.SpaceID, event.Session) if err != nil { - return nil, err + return nil, errno.WrapMQRetryErr(err) } // 默认用实验级主集 (老实验 / SingleSet); MultiSetConfig 下面按 item 归属集覆盖。 @@ -1055,18 +1031,17 @@ func (e *ExptItemEventEvalServiceImpl) BuildExptRecordEvalCtx(ctx context.Contex var itemConfig *entity.ExptItemConfig var itemVersionID *int64 // 新数据集 item 级版本; nil=老数据集(无 item 版本) if exptDetail.EvalSetSourceType == entity.ExptEvalSetSourceType_MultiSetConfig && e.exptItemRefRepo != nil { - ref, refErr := e.exptItemRefRepo.GetByExptIDAndItemID(ctx, event.SpaceID, event.ExptID, event.EvalSetItemID) + ref, refErr := e.exptItemRefRepo.GetByExptIDAndItemID(contexts.WithCtxWriteDB(ctx), event.SpaceID, event.ExptID, event.EvalSetItemID) if refErr != nil { logs.CtxError(ctx, "BuildExptRecordEvalCtx GetByExptIDAndItemID fail, expt_id: %v, item_id: %v, err: %v", event.ExptID, event.EvalSetItemID, refErr) - return nil, errorx.Wrapf(refErr, "get expt_item_ref fail, expt_id: %v, item_id: %v", event.ExptID, event.EvalSetItemID) + return nil, errno.WrapMQRetryErr(refErr) } if ref == nil || ref.ItemConfig == nil { // 正常调度必已写非 nil ItemConfig; 走到这里是 ref 未写全或 item_config 读空 —— 报错重试, 不静默漏评。 logs.CtxError(ctx, "BuildExptRecordEvalCtx expt_item_ref missing item_config, expt_id: %v, item_id: %v, ref_nil: %v", event.ExptID, event.EvalSetItemID, ref == nil) - return nil, errorx.NewByCode(errno.CommonInternalErrorCode, - errorx.WithExtraMsg(fmt.Sprintf("expt_item_ref missing item_config, expt_id: %v, item_id: %v", event.ExptID, event.EvalSetItemID))) + return nil, errno.NewMQRetryErr("expt_item_ref missing frozen item_config") } itemConfig = ref.ItemConfig if ref.EvalSetID > 0 { @@ -1084,10 +1059,13 @@ func (e *ExptItemEventEvalServiceImpl) BuildExptRecordEvalCtx(ctx context.Contex // item 版本在 ExptStart 已落进 expt_item_result_run_log, 这里读回 (供版本评测集按 item 版本取数); // 无版本评测集 run_log.ItemVersionID==0 → 保持 nil, 按集版本定位, 行为不变。 if itemVersionID == nil { - if runLog, rlErr := e.exptItemResultRepo.GetItemRunLog(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID); rlErr != nil { + if runLog, rlErr := e.exptItemResultRepo.GetItemRunLog(contexts.WithCtxWriteDB(ctx), event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID); rlErr != nil { logs.CtxWarn(ctx, "BuildExptRecordEvalCtx GetItemRunLog for item version fail, expt_id: %v, item_id: %v, err: %v", event.ExptID, event.EvalSetItemID, rlErr) - } else if runLog != nil && runLog.ItemVersionID > 0 { + return nil, errno.WrapMQRetryErr(rlErr) + } else if runLog == nil { + return nil, errno.NewMQRetryErr("item source run log missing") + } else if runLog.ItemVersionID > 0 { itemVersionID = gptr.Of(runLog.ItemVersionID) } } @@ -1120,7 +1098,7 @@ func (e *ExptItemEventEvalServiceImpl) BuildExptRecordEvalCtx(ctx context.Contex } items, err := e.evaluationSetItemService.BatchGetEvaluationSetItems(ctx, batchGetEvaluationSetItemsParam) if err != nil { - return nil, err + return nil, errno.WrapMQRetryErr(err) } if len(items) != 1 { @@ -1138,7 +1116,7 @@ func (e *ExptItemEventEvalServiceImpl) BuildExptRecordEvalCtx(ctx context.Contex existResult, err := e.GetExistExptRecordEvalResult(ctx, event) if err != nil { - return nil, err + return nil, errno.WrapMQRetryErr(err) } return &entity.ExptItemEvalCtx{ @@ -1152,14 +1130,19 @@ func (e *ExptItemEventEvalServiceImpl) BuildExptRecordEvalCtx(ctx context.Contex } func (e *ExptItemEventEvalServiceImpl) GetExistExptRecordEvalResult(ctx context.Context, event *entity.ExptItemEvalEvent) (*entity.ExptItemEvalResult, error) { - if event.ExptRunMode == entity.EvaluationModeFailRetry { - ctx = contexts.WithCtxWriteDB(ctx) - } - turnRunLogs, err := e.exptTurnResultRepo.GetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID) - if err != nil { - return nil, err + ctx = contexts.WithCtxWriteDB(ctx) + var turnRunLogs []*entity.ExptTurnResultRunLog + if execution := executionSource(ctx); execution != nil && execution.Source != nil { + for _, log := range execution.Source.Turns { + turnRunLogs = append(turnRunLogs, log) + } + } else { + var err error + turnRunLogs, err = e.exptTurnResultRepo.GetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID) + if err != nil { + return nil, err + } } - turnRunResultMap := make(map[int64]*entity.ExptTurnResultRunLog, len(turnRunLogs)) for _, result := range turnRunLogs { turnRunResultMap[result.TurnID] = result @@ -1268,14 +1251,15 @@ func (e *ExptRecordEvalModeSubmit) PreEval(ctx context.Context, eiec *entity.Exp turnRunResults := make([]*entity.ExptTurnResultRunLog, 0, len(absentRunLogTurnIDs)) for idx, turnID := range absentRunLogTurnIDs { turnRunResults = append(turnRunResults, &entity.ExptTurnResultRunLog{ - ID: ids[idx], - SpaceID: event.SpaceID, - ExptID: event.ExptID, - ExptRunID: event.ExptRunID, - ItemID: event.EvalSetItemID, - TurnID: turnID, - Status: entity.TurnRunState_Processing, - LogID: logID, + ID: ids[idx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: event.EvalSetItemID, + ItemVersionID: gptr.Indirect(eiec.EvalSetItem.ItemVersionID), + TurnID: turnID, + Status: entity.TurnRunState_Processing, + LogID: logID, }) } @@ -1347,21 +1331,9 @@ func (s *retryEvaluatorSourceLogs) matches(ctx context.Context, tr *entity.ExptT record.ItemID != s.event.EvalSetItemID || tr.ItemID != s.event.EvalSetItemID || record.TurnID != tr.TurnID { return false, nil } - byTurn, loaded := s.logsByRun[record.ExperimentRunID] - if !loaded { - // Shared evaluator records can live in a resource space; run logs belong to the experiment's space. - runLogs, err := s.repo.MGetItemTurnRunLogs(ctx, s.event.ExptID, record.ExperimentRunID, []int64{s.event.EvalSetItemID}, s.event.SpaceID) - if err != nil { - return false, err - } - byTurn = make(map[int64]*entity.ExptTurnResultRunLog, len(runLogs)) - for _, runLog := range runLogs { - if runLog != nil && runLog.SpaceID == s.event.SpaceID && runLog.ExptID == s.event.ExptID && - runLog.ExptRunID == record.ExperimentRunID && runLog.ItemID == s.event.EvalSetItemID { - byTurn[runLog.TurnID] = runLog - } - } - s.logsByRun[record.ExperimentRunID] = byTurn + byTurn, err := s.loadRun(ctx, record.ExperimentRunID) + if err != nil { + return false, err } origin := byTurn[tr.TurnID] if origin == nil || origin.TargetResultID != tr.TargetResultID || origin.EvaluatorResultIds == nil { @@ -1381,6 +1353,26 @@ func (s *retryEvaluatorSourceLogs) matches(ctx context.Context, tr *entity.ExptT return record.SourceType != entity.EvaluatorRecordSourceTypeInline && record.Alias == "" && refs.EvalVerIDToResID[record.EvaluatorVersionID] == record.ID, nil } +func (s *retryEvaluatorSourceLogs) loadRun(ctx context.Context, runID int64) (map[int64]*entity.ExptTurnResultRunLog, error) { + byTurn, loaded := s.logsByRun[runID] + if !loaded { + // Shared evaluator records can live in a resource space; run logs belong to the experiment's space. + runLogs, err := s.repo.MGetItemTurnRunLogs(ctx, s.event.ExptID, runID, []int64{s.event.EvalSetItemID}, s.event.SpaceID) + if err != nil { + return nil, err + } + byTurn = make(map[int64]*entity.ExptTurnResultRunLog, len(runLogs)) + for _, runLog := range runLogs { + if runLog != nil && runLog.SpaceID == s.event.SpaceID && runLog.ExptID == s.event.ExptID && + runLog.ExptRunID == runID && runLog.ItemID == s.event.EvalSetItemID { + byTurn[runLog.TurnID] = runLog + } + } + s.logsByRun[runID] = byTurn + } + return byTurn, nil +} + func retryEvaluatorResultsFromRefs(turnResultID int64, refs []*entity.ExptTurnEvaluatorResultRef) *entity.EvaluatorResults { results := &entity.EvaluatorResults{} for _, ref := range refs { @@ -1528,6 +1520,9 @@ type ExptRecordEvalModeFailRetry struct { } func (e *ExptRecordEvalModeFailRetry) PreEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + if eiec.Event != nil && eiec.Event.RequestID != "" { + return validateManualFrozenTurns(eiec) + } if eiec.GetExistItemResultLog() != nil && len(eiec.GetExistTurnResultLogs()) > 0 { return nil } @@ -1608,6 +1603,9 @@ type ExptRecordEvalModeRetryIgnoreResult struct { } func (e *ExptRecordEvalModeRetryIgnoreResult) PreEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + if eiec.Event != nil && eiec.Event.RequestID != "" { + return validateManualFrozenTurns(eiec) + } if eiec.GetExistItemResultLog() != nil && len(eiec.GetExistTurnResultLogs()) > 0 { return nil } @@ -1623,14 +1621,15 @@ func (e *ExptRecordEvalModeRetryIgnoreResult) PreEval(ctx context.Context, eiec turnRunLogs := make([]*entity.ExptTurnResultRunLog, 0, len(eiec.EvalSetItem.Turns)) for idx, turn := range eiec.EvalSetItem.Turns { turnRunLogs = append(turnRunLogs, &entity.ExptTurnResultRunLog{ - ID: ids[idx], - SpaceID: event.SpaceID, - ExptID: event.ExptID, - ExptRunID: event.ExptRunID, - ItemID: event.EvalSetItemID, - TurnID: turn.ID, - Status: entity.TurnRunState_Processing, - LogID: logID, + ID: ids[idx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: event.EvalSetItemID, + ItemVersionID: gptr.Indirect(eiec.EvalSetItem.ItemVersionID), + TurnID: turn.ID, + Status: entity.TurnRunState_Processing, + LogID: logID, }) } @@ -1648,3 +1647,166 @@ func (e *ExptRecordEvalModeRetryIgnoreResult) PreEval(ctx context.Context, eiec func (e *ExptRecordEvalModeRetryIgnoreResult) PostEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { return nil } + +func (e *ExptItemEventEvalServiceImpl) HandleCurrentSource(next RecordEvalEndPoint) RecordEvalEndPoint { + return func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + source, err := e.exptItemResultRepo.GetCurrentRequest(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !source.Matches(event.ExptRunID, event.RequestID) { + return nil + } + if event.ExptRunMode != source.Mode { + return errno.NewMQRetryErr("event/current item mode mismatch") + } + if executionSource(ctx) == nil { + ctx = context.WithValue(ctx, currentItemSourceKey{}, ¤tItemExecution{}) + } + if port, ok := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + frozen, err := port.GetCurrentItemRunSource(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if frozen == nil || !frozen.Current.Matches(event.ExptRunID, event.RequestID) { + return nil + } + if execution := executionSource(ctx); execution != nil { + execution.Source = frozen + } + source = &frozen.Current + } + valid, sourceErr := e.validateResumeSource(ctx, event, source) + if sourceErr != nil { + return sourceErr + } + if !valid { + return nil + } + if event.RetryTimes < 0 { + return errno.NewMQRetryErr("invalid event retry count") + } + if int(source.RetryTimes) > event.RetryTimes { + event.RetryTimes = int(source.RetryTimes) + } + if source.Status == entity.ItemRunState_Queueing { + if _, central := event.CtxCentralAdmittedExpt(ctx); !central { + advanced, err := e.exptItemResultRepo.MarkProcessingIfCurrent(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.RequestID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !advanced { + current, err := e.exptItemResultRepo.GetCurrentRequest(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.EvalSetItemID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !current.Matches(event.ExptRunID, event.RequestID) { + return nil + } + if current.Status != entity.ItemRunState_Processing { + return errno.NewMQRetryErr("current item remains Queueing") + } + } + } + } + if port, ok := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + frozen, err := sourceForExecution(ctx, port, event) + if err != nil { + return err + } + applied, err := port.CompleteCurrentItemRun(ctx, frozen, &entity.ExptCurrentItemRunWrite{RetryOnly: true, RetryTimes: int32(event.RetryTimes)}) + if err := checkedCurrentWrite(ctx, applied, err); err != nil { + return err + } + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + executionSource(ctx).SourceValidated = true + return next(ctx, event) + } +} + +func validateManualFrozenTurns(eiec *entity.ExptItemEvalCtx) error { + if eiec == nil || eiec.Event == nil || eiec.Event.RequestID == "" { + return nil + } + if eiec.GetExistItemResultLog() == nil || eiec.EvalSetItem == nil || len(eiec.EvalSetItem.Turns) == 0 { + return errno.NewMQRetryErr("manual frozen item or turns missing") + } + for _, turn := range eiec.EvalSetItem.Turns { + if turn == nil { + return errno.NewMQRetryErr("manual frozen turn missing") + } + log := eiec.GetExistTurnResultRunLog(turn.ID) + if log == nil || log.ExptID != eiec.Event.ExptID || log.ExptRunID != eiec.Event.ExptRunID || log.ItemID != eiec.Event.EvalSetItemID || log.SpaceID != eiec.Event.SpaceID { + return errno.NewMQRetryErr("manual frozen turn log missing or foreign") + } + if log.Status < entity.TurnRunState_Queueing || log.Status > entity.TurnRunState_Terminal { + return errno.NewMQRetryErr("unknown frozen turn status") + } + } + return nil +} + +func (e *ExptItemEventEvalServiceImpl) ordinaryInvocationInFlight(ctx context.Context, eiec *entity.ExptItemEvalCtx) (bool, error) { + if eiec.Event.AsyncReportTrigger || eiec.Event.AsyncEvaluatorReportTrigger { + return false, nil + } + for _, log := range eiec.GetExistTurnResultLogs() { + if log == nil { + return false, errno.NewMQRetryErr("nil turn source") + } + if log.Status < entity.TurnRunState_Queueing || log.Status > entity.TurnRunState_Terminal { + return false, errno.NewMQRetryErr("unknown source turn state") + } + if log.Status != entity.TurnRunState_Processing { + continue + } + if log.TargetResultID > 0 { + record, err := e.evaTargetService.GetRecordByID(ctx, resolveLoadSpaceID(eiec.Event.SpaceID, eiec.TargetSourceSpaceID()), log.TargetResultID) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if record == nil { + return false, errno.NewMQRetryErr("target source record missing") + } + if gptr.Indirect(record.Status) == entity.EvalTargetRunStatusAsyncInvoking { + return true, nil + } + } + refs := log.EvaluatorResultIds + if refs == nil { + continue + } + ids := make(map[int64]struct{}) + for _, id := range refs.EvalVerIDToResID { + if id > 0 { + ids[id] = struct{}{} + } + } + for _, ref := range refs.Registered { + if ref != nil && ref.RecordID > 0 { + ids[ref.RecordID] = struct{}{} + } + } + for _, ref := range refs.Inline { + if ref != nil && ref.RecordID > 0 { + ids[ref.RecordID] = struct{}{} + } + } + for id := range ids { + record, err := e.evaluatorRecordService.GetEvaluatorRecord(ctx, id, false) + if err != nil { + return false, errno.WrapMQRetryErr(err) + } + if record == nil { + return false, errno.NewMQRetryErr("evaluator source record missing") + } + if record.Status == entity.EvaluatorRunStatusAsyncInvoking { + return true, nil + } + } + } + return false, nil +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go index 80f2324244..aac6d088fb 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go @@ -377,7 +377,7 @@ func TestExptItemEventEvalServiceImpl_HandleEventLock(t *testing.T) { Return(false, nil, nil, nil) }, event: &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, EvalSetItemID: 3}, - wantErr: false, + wantErr: true, }, { name: "Acquire lock failed - error", diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index 973f29c205..e94d56b7bb 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -5,6 +5,7 @@ package service import ( "context" + "errors" "fmt" "strconv" "strings" @@ -15,6 +16,7 @@ import ( "github.com/jinzhu/copier" "github.com/coze-dev/coze-loop/backend/infra/external/benefit" + evalconsts "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" @@ -109,7 +111,10 @@ func (e *ExptItemEvalCtxExecutor) EvalTurns(ctx context.Context, eiec *entity.Ex return false, fmt.Errorf("EvalTurns with invalid empty eval_set_item") } - for _, turn := range eiec.EvalSetItem.Turns { + for turnIndex, turn := range eiec.EvalSetItem.Turns { + if err := ctx.Err(); err != nil { + return false, errno.WrapMQRetryErr(err) + } // 行级终止的「不再继续」语义：每轮开始前查一次该行是否已被终止，是则 break， // 当前已跑完的轮次结果保留、后续轮次不再开始（对应 spec「执行中的行在当轮结束后停止」）。 // 放在循环体首部（而非尾部）是为了同时覆盖「排队中被终止、事件已在途」的情形。 @@ -123,6 +128,19 @@ func (e *ExptItemEvalCtxExecutor) EvalTurns(ctx context.Context, eiec *entity.Ex if err != nil { return false, err } + if existing := etec.GetExistTurnResultRunLog(turn.ID); existing != nil && existing.Status == entity.TurnRunState_Success && !etec.Event.IgnoreExistedTargetResult() { + target := etec.ExptTurnRunResult.GetTargetResult() + if existing.TargetResultID > 0 && target == nil { + return false, fmt.Errorf("success turn target record %d missing during history resume", existing.TargetResultID) + } + if turnIndex+1 < len(eiec.EvalSetItem.Turns) { + if err := e.loadTargetHistoryOutput(ctx, target); err != nil { + return false, err + } + } + history = append(history, buildHistoryMessage(ctx, etec.ExptTurnRunResult)...) + continue + } ctx = context.WithValue(ctx, consts.CtxKeyLogID, etec.GetTurnEvalLogID(ctx, turn.ID)) //nolint:staticcheck @@ -152,6 +170,11 @@ func (e *ExptItemEvalCtxExecutor) EvalTurns(ctx context.Context, eiec *entity.Ex } e.emitSandboxAgentE2EFinishedIfTerminal(ctx, etec, eiec.Event, turnRunRes, nil) + if turnIndex+1 < len(eiec.EvalSetItem.Turns) { + if err := e.loadTargetHistoryOutput(ctx, turnRunRes.GetTargetResult()); err != nil { + return false, err + } + } history = append(history, buildHistoryMessage(ctx, turnRunRes)...) } @@ -179,10 +202,13 @@ func (e *ExptItemEvalCtxExecutor) isItemTerminated(ctx context.Context, eiec *en } func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec *entity.ExptTurnEvalCtx, result *entity.ExptTurnRunResult) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } if result == nil { return fmt.Errorf("StoreTurnRunResult with nil result") } - persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + persistCtx, cancel := context.WithTimeout(ctx, exptRunLogPersistTimeout) defer cancel() turn := etec.Turn @@ -218,11 +244,15 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * logs.CtxWarn(ctx, "[ExptTurnEval] nil evaluator record, evaluator_version_id: %v", evID) continue } - clone.EvaluatorResultIds.Registered = append(clone.EvaluatorResultIds.Registered, &entity.RegisteredEvalResult{ - VersionID: er.EvaluatorVersionID, - Alias: er.Alias, - RecordID: er.ID, - }) + if er.SourceType == entity.EvaluatorRecordSourceTypeInline { + clone.EvaluatorResultIds.Inline = append(clone.EvaluatorResultIds.Inline, &entity.InlineEvalResult{InlineKey: er.InlineKey, RecordID: er.ID}) + } else { + clone.EvaluatorResultIds.Registered = append(clone.EvaluatorResultIds.Registered, &entity.RegisteredEvalResult{ + VersionID: er.EvaluatorVersionID, + Alias: er.Alias, + RecordID: er.ID, + }) + } if er.EvaluatorOutputData != nil && er.EvaluatorOutputData.EvaluatorRunError != nil && er.EvaluatorOutputData.EvaluatorRunError.Code > 0 { evalErr = errno.NewEvaluatorResultErr(er.EvaluatorOutputData.EvaluatorRunError.Message) } @@ -271,10 +301,25 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * result.SetEvalErr(evalErr) - if err := e.TurnResultRepo.SaveTurnRunLogs(persistCtx, []*entity.ExptTurnResultRunLog{clone}); err != nil { + if port, ok := e.ItemResultRepo.(repo.IExptCurrentResultRepo); ok { + source, err := sourceForExecution(ctx, port, etec.Event) + if err != nil { + return err + } + applied, err := port.StoreCurrentTurnRunResult(persistCtx, source, clone) + if err := checkedCurrentWrite(ctx, applied, err); err != nil { + return err + } + // Advance only the reference that this execution just stored. Never read latest + // and replace the expected identity after an RPC or admission. + source.Turns[turn.ID] = clone + } else if err := e.TurnResultRepo.SaveTurnRunLogs(persistCtx, []*entity.ExptTurnResultRunLog{clone}); err != nil { return err } - resumeCtx, resumeCancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + resumeCtx, resumeCancel := context.WithTimeout(ctx, exptRunLogPersistTimeout) defer resumeCancel() var resumeWG sync.WaitGroup for _, record := range result.EvaluatorResults { @@ -390,14 +435,16 @@ func (e *ExptItemEvalCtxExecutor) SetItemRunProcessing(ctx context.Context, expt } func (e *ExptItemEvalCtxExecutor) buildExptTurnEvalCtx(ctx context.Context, turn *entity.Turn, eiec *entity.ExptItemEvalCtx, history []*entity.Message) (*entity.ExptTurnEvalCtx, error) { + turnItemCtx := *eiec + turnItemCtx.Event = eiec.Event.CloneForTurn(turn.ID) var ( spaceID = eiec.Event.SpaceID existTurnRunResult = eiec.GetExistTurnResultRunLog(turn.ID) etec = &entity.ExptTurnEvalCtx{ - ExptItemEvalCtx: eiec, + ExptItemEvalCtx: &turnItemCtx, Turn: turn, ExptTurnRunResult: &entity.ExptTurnRunResult{}, - // History: history, + History: history, } ) etec.Ext = make(map[string]string) @@ -487,6 +534,19 @@ func (e *ExptItemEvalCtxExecutor) buildExptTurnEvalCtx(ctx context.Context, turn return nil, err } etec.ExptTurnRunResult.EvaluatorResults = evaluatorRecords + if isTurnAsyncResume(etec, false) || isTurnAsyncResume(etec, true) || existTurnRunResult.Status == entity.TurnRunState_Success { + loaded := make(map[int64]bool, len(evaluatorRecords)) + for _, record := range evaluatorRecords { + if record != nil { + loaded[record.ID] = true + } + } + for _, id := range recordIDs { + if !loaded[id] { + return nil, fmt.Errorf("turn evaluator record %d missing during resume", id) + } + } + } } } @@ -494,8 +554,14 @@ func (e *ExptItemEvalCtxExecutor) buildExptTurnEvalCtx(ctx context.Context, turn } func (e *ExptItemEvalCtxExecutor) CompleteItemRun(ctx context.Context, eiec *entity.ExptItemEvalCtx, evalErr error) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if errno.NeedMQRetry(evalErr) || errors.Is(evalErr, errExptNonCurrentSource) { + return evalErr + } event := eiec.Event - persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + persistCtx, cancel := context.WithTimeout(ctx, exptRunLogPersistTimeout) defer cancel() if evalErr != nil { @@ -516,26 +582,31 @@ func (e *ExptItemEvalCtxExecutor) CompleteItemRun(ctx context.Context, eiec *ent ufields["status"] = int32(entity.ItemRunState_Success) } - // Terminal 是吸收态：该行已被用户行级终止时，在途执行的成功/失败结果 MUST NOT 回写 status / err_msg - // （对应 spec「终止后到达的执行结果不覆盖终止状态」）—— 用户要看到的是"被主动终止"，而不是终止前 - // 那次执行的报错。 - // - // ⚠️ 这里刻意**不**用「先 SELECT 判 Terminal 再 UPDATE」：普通 SELECT 会被路由到只读从库 - // （商业化侧 db 以 WithReadReplicas 构建），主从延迟窗口内会读回终止前的旧状态并把 Terminal - // 覆盖成 Success —— 该行还会被重新扫成成功行、发 item-complete MQ、记进 success_cnt， - // 用户的终止操作被静默吞掉。即便读主库，读-判-写之间仍有 TOCTOU 窗口。 - // 故拆成两条写，用 DB 的 WHERE 做原子判定： - // (1) 条件写：status/err_msg/result_state，仅对 status <> Terminal 的行生效； - // (2) 无条件写：result_state=Logged —— 对已 Terminal 的行也必须落，否则调度侧收不了口。 - // 该字段与终止语义无冲突（TerminateItems 自己也写 Logged），重复写幂等。 - if err := e.ItemResultRepo.UpdateItemRunLogIfNotTerminal(persistCtx, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}, ufields, event.SpaceID); err != nil { - return err - } - - if err := e.ItemResultRepo.UpdateItemRunLog(persistCtx, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}, map[string]any{ - "result_state": entity.ExptItemResultStateLogged, - }, event.SpaceID); err != nil { - return err + if port, ok := e.ItemResultRepo.(repo.IExptCurrentResultRepo); ok { + source, err := sourceForExecution(ctx, port, event) + if err != nil { + return err + } + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Success, RetryTimes: int32(event.RetryTimes)} + if evalErr != nil { + write.Status = entity.ItemRunState_Fail + write.ErrMsg = []byte(errno.SerializeErr(evalErr)) + write.FailPendingTurns = true + } + applied, err := port.CompleteCurrentItemRun(persistCtx, source, write) + if err := checkedCurrentWrite(ctx, applied, err); err != nil { + return err + } + if execution := executionSource(ctx); execution != nil { + execution.Completed = true + } + } else { + if err := e.ItemResultRepo.UpdateItemRunLogIfNotTerminal(persistCtx, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}, ufields, event.SpaceID); err != nil { + return err + } + if err := e.ItemResultRepo.UpdateItemRunLog(persistCtx, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}, map[string]any{"result_state": entity.ExptItemResultStateLogged}, event.SpaceID); err != nil { + return err + } } // 沙箱 agent 实验:单行走到终态失败,立即发一张飞书卡。Notifier 内部做 sandbox agent + Enable 判定, @@ -708,7 +779,34 @@ func (e *ExptItemEvalCtxExecutor) evalErrNeedTerminateExpt(ctx context.Context, return conf.IsInDebt } +// Both DB reads and fresh persisted target records can contain only a preview. +func (e *ExptItemEvalCtxExecutor) loadTargetHistoryOutput(ctx context.Context, target *entity.EvalTargetRecord) error { + if target == nil || target.EvalTargetOutputData == nil { + return nil + } + output := target.EvalTargetOutputData.OutputFields[evalconsts.EvalTargetOutputFieldKeyActualOutput] + if output == nil || !output.IsContentOmitted() { + return nil + } + if err := e.evalTargetService.LoadRecordOutputFields(ctx, target, []string{evalconsts.EvalTargetOutputFieldKeyActualOutput}); err != nil { + return fmt.Errorf("load target history record %d: %w", target.ID, err) + } + if target.EvalTargetOutputData == nil { + return fmt.Errorf("target history record %d missing output after load", target.ID) + } + output = target.EvalTargetOutputData.OutputFields[evalconsts.EvalTargetOutputFieldKeyActualOutput] + if output == nil || output.IsContentOmitted() { + return fmt.Errorf("target history record %d incomplete after load", target.ID) + } + return nil +} + func buildHistoryMessage(ctx context.Context, turnRunResult *entity.ExptTurnRunResult) []*entity.Message { + if target := turnRunResult.GetTargetResult(); target != nil && target.EvalTargetOutputData != nil { + if output := target.EvalTargetOutputData.OutputFields[evalconsts.EvalTargetOutputFieldKeyActualOutput]; output != nil { + return []*entity.Message{{Role: entity.RoleAssistant, Content: output}} + } + } return nil } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index eb69862d76..6f336fe6f7 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -405,22 +405,15 @@ func Test_ExptItemEvalCtxExecutor_CompleteSetItemRun(t *testing.T) { assert.Contains(t, err.Error(), "mock updateitemrunlog error") }) - t.Run("ctx取消后仍落item失败状态", func(t *testing.T) { + t.Run("ctx取消后不落item失败状态", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - - mockConfiger.EXPECT().GetErrRetryConf(gomock.Any(), int64(4), gomock.Any()).AnyTimes().Return(&entity.RetryConf{IsInDebt: false}) - mockItemResultRepo.EXPECT().UpdateItemRunLogIfNotTerminal(gomock.Any(), int64(1), int64(2), []int64{3}, gomock.Any(), int64(4)). - DoAndReturn(func(ctx context.Context, _, _ int64, _ []int64, ufields map[string]any, _ int64) error { - require.NoError(t, ctx.Err()) - assert.Equal(t, int32(entity.ItemRunState_Fail), ufields["status"]) - return nil - }) - mockItemResultRepo.EXPECT().UpdateItemRunLog(gomock.Any(), int64(1), int64(2), []int64{3}, gomock.Any(), int64(4)).Return(nil) - + mockItemResultRepo.EXPECT().UpdateItemRunLogIfNotTerminal(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + mockItemResultRepo.EXPECT().UpdateItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) eiec := &entity.ExptItemEvalCtx{Event: &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, EvalSetItemID: 3, SpaceID: 4, RetryTimes: 1}} err := executor.CompleteItemRun(ctx, eiec, errors.New("target timeout")) - assert.NoError(t, err) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) }) } @@ -862,35 +855,13 @@ func Test_ExptItemEvalCtxExecutor_storeTurnRunResult(t *testing.T) { assert.NoError(t, err) }) - t.Run("ctx取消后仍落turn失败状态", func(t *testing.T) { + t.Run("ctx取消后不落turn失败状态", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - - turnResultLog := &entity.ExptTurnResultRunLog{ID: 1, TurnID: 1} - etec := &entity.ExptTurnEvalCtx{ - Turn: &entity.Turn{ID: 1}, - ExptItemEvalCtx: &entity.ExptItemEvalCtx{ - Expt: &entity.Experiment{ID: 1, SourceID: "src", SpaceID: 2}, - Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, - EvalSetItem: &entity.EvaluationSetItem{ItemID: 2}, - ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{1: turnResultLog}}, - }, - } - result := &entity.ExptTurnRunResult{EvalErr: errors.New("target timeout")} - - mockConfiger.EXPECT().GetErrCtrl(gomock.Any()).DoAndReturn(func(ctx context.Context) *entity.ExptErrCtrl { - require.NoError(t, ctx.Err()) - return entity.DefaultExptErrCtrl() - }) - mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, logs []*entity.ExptTurnResultRunLog) error { - require.NoError(t, ctx.Err()) - require.Len(t, logs, 1) - assert.Equal(t, entity.TurnRunState_Fail, logs[0].Status) - return nil - }) - - err := executor.storeTurnRunResult(ctx, etec, result) - assert.NoError(t, err) + mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Times(0) + err := executor.storeTurnRunResult(ctx, &entity.ExptTurnEvalCtx{}, &entity.ExptTurnRunResult{EvalErr: errors.New("target timeout")}) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) }) } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index a51fff6333..4a1cbb3271 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -11,7 +11,6 @@ import ( "time" "github.com/bytedance/gg/gcond" - "github.com/bytedance/gg/gmap" "github.com/bytedance/gg/gptr" "github.com/bytedance/gg/gslice" @@ -24,6 +23,7 @@ import ( "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" "github.com/coze-dev/coze-loop/backend/pkg/lang/goroutine" + "github.com/coze-dev/coze-loop/backend/pkg/lang/mem" "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" "github.com/coze-dev/coze-loop/backend/pkg/logs" ) @@ -103,6 +103,7 @@ func (e *DefaultExptTurnEvaluationImpl) Eval(ctx context.Context, etec *entity.E } func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *entity.ExptTurnEvalCtx) (*entity.EvalTargetRecord, error) { + scopeTurnEvalEvent(etec) if e.skipTargetNode(etec.Expt) { return &entity.EvalTargetRecord{EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: make(map[string]*entity.Content)}}, nil } @@ -113,8 +114,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *en tr := etec.ExptTurnRunResult.GetTargetResult() - if etec.Event.AsyncReportTrigger || etec.Event.AsyncEvaluatorReportTrigger { - etec.Event.WithCtxTargetCalled(ctx) + if isTurnAsyncResume(etec, false) || isTurnAsyncResume(etec, true) { return tr, nil } if tr != nil && gptr.Indirect(tr.Status) == entity.EvalTargetRunStatusSuccess && !etec.Event.IgnoreExistedTargetResult() { @@ -136,13 +136,41 @@ func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *en return record, nil } +// Direct turn callers also need a local event before mode-based reuse checks. +func scopeTurnEvalEvent(etec *entity.ExptTurnEvalCtx) { + if etec == nil || etec.Event == nil || etec.Turn == nil || etec.Event.ResumeSource == nil || etec.Event.ResumeSource.TurnID == etec.Turn.ID { + return + } + // A non-source turn with cleared flags is already scoped. Keep its event + // identity so CallEvaluators sees CallTarget's marker after a new invoke. + if !etec.Event.AsyncReportTrigger && !etec.Event.AsyncEvaluatorReportTrigger { + return + } + itemCtx := *etec.ExptItemEvalCtx + itemCtx.Event = etec.Event.CloneForTurn(etec.Turn.ID) + etec.ExptItemEvalCtx = &itemCtx +} + func (e *DefaultExptTurnEvaluationImpl) validateEvalTargetCtx(etec *entity.ExptTurnEvalCtx) error { - if (etec.Event.AsyncReportTrigger || etec.Event.AsyncEvaluatorReportTrigger) && etec.ExptTurnRunResult.GetTargetResult() == nil { + if (isTurnAsyncResume(etec, false) || isTurnAsyncResume(etec, true)) && etec.ExptTurnRunResult.GetTargetResult() == nil { return errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("target result must not be nil in async reported event")) } return nil } +func isTurnAsyncResume(etec *entity.ExptTurnEvalCtx, evaluator bool) bool { + if etec == nil || etec.Event == nil { + return false + } + if source := etec.Event.ResumeSource; source != nil && (etec.Turn == nil || source.TurnID != etec.Turn.ID) { + return false + } + if evaluator { + return etec.Event.AsyncEvaluatorReportTrigger + } + return etec.Event.AsyncReportTrigger +} + // skipTargetNode Whether target is called is determined by the target info bound in expt; // ConnectorConf.TargetConf serves as the config info for executing the target, and CheckConnector completes the validity check when creating experiment. // 仅记录型（*Online）不需要执行对象，跳过 target 节点 @@ -287,7 +315,7 @@ func (e *DefaultExptTurnEvaluationImpl) callTarget(ctx context.Context, etec *en return nil, err } - ext := gmap.Clone(etec.Ext) + ext := entity.WithoutManualRetryExt(etec.Ext) if targetConf.IngressConf != nil && targetConf.IngressConf.CustomConf != nil { for _, fc := range targetConf.IngressConf.CustomConf.FieldConfs { if fc.FieldName == consts.FieldAdapterBuiltinFieldNameRuntimeParam { @@ -344,8 +372,15 @@ func (e *DefaultExptTurnEvaluationImpl) callTarget(ctx context.Context, etec *en if etec.Expt.EvalConf != nil { etc.EnableExtractTrajectory = etec.Expt.EvalConf.EnableExtractTrajectory } + // Provider storage clips Content in place; retain independent history for later turns. + var historyMessages []*entity.Message + if history != nil { + if err := mem.DeepCopy(history, &historyMessages); err != nil { + return nil, errorx.Wrapf(err, "copy target history") + } + } etid := &entity.EvalTargetInputData{ - HistoryMessages: history, + HistoryMessages: historyMessages, InputFields: inputFields, Ext: ext, } @@ -356,7 +391,7 @@ func (e *DefaultExptTurnEvaluationImpl) callTarget(ctx context.Context, etec *en ts := time.Now() makeAsyncCtx := func(recordID int64, callee string) *entity.EvalAsyncCtx { - return &entity.EvalAsyncCtx{ + return (&entity.EvalAsyncCtx{ Event: etec.Event, RecordID: recordID, AsyncUnixMS: ts.UnixMilli(), @@ -370,7 +405,7 @@ func (e *DefaultExptTurnEvaluationImpl) callTarget(ctx context.Context, etec *en DatasetKey: pickDatasetKey(etec), AgentName: pickAgentName(etec), ApplicationID: pickApplicationID(etec), - } + }).Clone() } preparedID := int64(0) etc.PrepareAsyncContext = func(ctx context.Context, recordID int64, callee string) error { @@ -465,11 +500,12 @@ func pickApplicationID(etec *entity.ExptTurnEvalCtx) string { } func (e *DefaultExptTurnEvaluationImpl) CallEvaluators(ctx context.Context, etec *entity.ExptTurnEvalCtx, targetResult *entity.EvalTargetRecord) ([]*entity.EvaluatorRecord, error) { + scopeTurnEvalEvent(etec) if e.skipEvaluatorNode(etec.Expt) { return nil, nil } - if etec.Event.AsyncEvaluatorReportTrigger { + if isTurnAsyncResume(etec, true) || (isTurnAsyncResume(etec, false) && len(etec.ExptTurnRunResult.EvaluatorResults) > 0) { logs.CtxInfo(ctx, "CallEvaluators skip re-run due to async report trigger, return existing evaluator results: %d", len(etec.ExptTurnRunResult.EvaluatorResults)) return etec.ExptTurnRunResult.EvaluatorResults, nil } @@ -490,6 +526,13 @@ func (e *DefaultExptTurnEvaluationImpl) CallEvaluators(ctx context.Context, etec expt := etec.Expt evaluatorResults := make([]*entity.EvaluatorRecord, 0, len(expt.Evaluators)) pendingEvaluatorVersionIDs := make([]int64, 0, len(expt.Evaluators)) + if !etec.Event.IgnoreExistedEvaluatorResult(ctx) { + for _, record := range etec.ExptTurnRunResult.EvaluatorResults { + if record != nil && record.SourceType == entity.EvaluatorRecordSourceTypeInline && (record.Status == entity.EvaluatorRunStatusSuccess || record.Status == entity.EvaluatorRunStatusAsyncInvoking) { + evaluatorResults = append(evaluatorResults, record) + } + } + } for _, evaluatorVersion := range expt.Evaluators { versionID := evaluatorVersion.GetEvaluatorVersionID() @@ -650,7 +693,7 @@ func (e *DefaultExptTurnEvaluationImpl) callEvaluators(ctx context.Context, exec ExperimentRunID: etec.Event.ExptRunID, ItemID: item.ItemID, TurnID: turn.ID, - Ext: etec.Ext, + Ext: entity.WithoutManualRetryExt(etec.Ext), }); interceptErr != nil { logs.CtxWarn(ctx, "[CallEvaluators] ShouldInterceptEvaluator failed, evaluator_version_id: %d, err: %v", evForCapture.GetEvaluatorVersionID(), interceptErr) } else if intercepted { @@ -667,7 +710,7 @@ func (e *DefaultExptTurnEvaluationImpl) callEvaluators(ctx context.Context, exec ExperimentRunID: etec.Event.ExptRunID, ItemID: item.ItemID, TurnID: turn.ID, - Ext: etec.Ext, + Ext: entity.WithoutManualRetryExt(etec.Ext), EvaluatorRunConf: ecForCapture.RunConf, } @@ -748,6 +791,13 @@ func (e *DefaultExptTurnEvaluationImpl) callEvaluatorsByItemConfig( } // 按 versionID 建一个 Evaluator 索引, 用于 ItemConfig 循环里复用 (type/inputSchema 等元数据)。 + if !etec.Event.IgnoreExistedEvaluatorResult(ctx) { + for _, record := range etec.ExptTurnRunResult.EvaluatorResults { + if record != nil && record.SourceType == entity.EvaluatorRecordSourceTypeInline && (record.Status == entity.EvaluatorRunStatusSuccess || record.Status == entity.EvaluatorRunStatusAsyncInvoking) { + collector.store(record) + } + } + } evByVer := make(map[int64]*entity.Evaluator, len(expt.Evaluators)) for _, ev := range expt.Evaluators { if ev != nil { @@ -933,7 +983,7 @@ func (e *DefaultExptTurnEvaluationImpl) callEvaluatorsByItemConfig( ExperimentRunID: etec.Event.ExptRunID, ItemID: item.ItemID, TurnID: turn.ID, - Ext: etec.Ext, + Ext: entity.WithoutManualRetryExt(etec.Ext), Alias: aliasForCapture, SourceType: entity.EvaluatorRecordSourceTypeBuiltin, }); interceptErr != nil { @@ -966,7 +1016,7 @@ func (e *DefaultExptTurnEvaluationImpl) callEvaluatorsByItemConfig( ExperimentRunID: etec.Event.ExptRunID, ItemID: item.ItemID, TurnID: turn.ID, - Ext: etec.Ext, + Ext: entity.WithoutManualRetryExt(etec.Ext), EvaluatorRunConf: runConfForCapture, Alias: aliasForCapture, SourceType: entity.EvaluatorRecordSourceTypeBuiltin, @@ -1008,16 +1058,16 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluatorWithAlias( ExperimentRunID: etec.Event.ExptRunID, ItemID: etec.EvalSetItem.ItemID, TurnID: etec.Turn.ID, - Ext: etec.Ext, + Ext: entity.WithoutManualRetryExt(etec.Ext), EvaluatorRunConf: runConf, Alias: alias, SourceType: entity.EvaluatorRecordSourceTypeBuiltin, - AsyncCtx: &entity.EvalAsyncCtx{ + AsyncCtx: (&entity.EvalAsyncCtx{ Event: etec.Event, Session: etec.Event.Session, EvaluatorVersionID: ev.GetEvaluatorVersionID(), ResumeReady: false, - }, + }).Clone(), }) if evaluatorRecord != nil { collector.store(evaluatorRecord) @@ -1047,14 +1097,14 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluator( ExperimentRunID: etec.Event.ExptRunID, ItemID: etec.EvalSetItem.ItemID, TurnID: etec.Turn.ID, - Ext: etec.Ext, + Ext: entity.WithoutManualRetryExt(etec.Ext), EvaluatorRunConf: ec.RunConf, - AsyncCtx: &entity.EvalAsyncCtx{ + AsyncCtx: (&entity.EvalAsyncCtx{ Event: etec.Event, Session: etec.Event.Session, EvaluatorVersionID: ev.GetEvaluatorVersionID(), ResumeReady: false, - }, + }).Clone(), } evaluatorRecord, err := e.evaluatorService.AsyncRunEvaluator(ctx, asyncReq) if evaluatorRecord != nil { @@ -1299,7 +1349,7 @@ func (e *DefaultExptTurnEvaluationImpl) getContentByJsonPath(content *entity.Con } func (e *DefaultExptTurnEvaluationImpl) buildEvaluatorInputDataExt(ext map[string]string, runConf *entity.EvaluatorRunConfig, evalConf *entity.EvaluationConfiguration) map[string]string { - builtExt := gmap.Clone(ext) + builtExt := entity.WithoutManualRetryExt(ext) if builtExt == nil { builtExt = make(map[string]string) } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 5776d38ae5..3a4e5e3fda 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -2157,14 +2157,6 @@ func TestDefaultExptTurnEvaluationImpl_callTarget_Async(t *testing.T) { evalAsyncRepo: mockEvalAsyncRepo, } - if preRegister { - service.evalAsyncRepo = &preparingAsyncRepoTest{MockIEvalAsyncRepo: mockEvalAsyncRepo, prepare: func(_ context.Context, key string, actx *entity.EvalAsyncCtx) (bool, error) { - require.Equal(t, "9999", key) - require.Equal(t, int64(777), actx.Event.ExptRunID) - require.Equal(t, "callee-service", actx.Callee) - return true, nil - }} - } spaceID := int64(42) targetID := int64(101) targetVersionID := int64(202) @@ -2226,6 +2218,16 @@ func TestDefaultExptTurnEvaluationImpl_callTarget_Async(t *testing.T) { Ext: map[string]string{"ext-key": "ext-val"}, } + if preRegister { + service.evalAsyncRepo = &preparingAsyncRepoTest{MockIEvalAsyncRepo: mockEvalAsyncRepo, prepare: func(_ context.Context, key string, actx *entity.EvalAsyncCtx) (bool, error) { + require.Equal(t, "9999", key) + require.Equal(t, int64(777), actx.Event.ExptRunID) + require.Equal(t, "callee-service", actx.Callee) + assertIndependentTargetAsyncContext(t, etec.Event, actx) + return true, nil + }} + } + mockMetric.EXPECT().EmitTurnExecTargetResult(spaceID, false) mockEvalTargetService.EXPECT().AsyncExecuteTarget(gomock.Any(), spaceID, targetID, targetVersionID, gomock.Any(), gomock.Any()).DoAndReturn( @@ -2259,6 +2261,7 @@ func TestDefaultExptTurnEvaluationImpl_callTarget_Async(t *testing.T) { if assert.NotNil(t, actx) { assert.Equal(t, "callee-service", actx.Callee) assert.Equal(t, etec.Event, actx.Event) + assertIndependentTargetAsyncContext(t, etec.Event, actx) } return nil }, @@ -5195,7 +5198,8 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluatorWithAlias_CustomRPC(t * assert.Equal(t, entity.EvaluatorRecordSourceTypeBuiltin, req.SourceType) assert.Same(t, runConf, req.EvaluatorRunConf) require.NotNil(t, req.AsyncCtx) - assert.Same(t, etec.Event, req.AsyncCtx.Event) + assert.Equal(t, etec.Event, req.AsyncCtx.Event) + assert.NotSame(t, etec.Event, req.AsyncCtx.Event) assert.False(t, req.AsyncCtx.ResumeReady) return &entity.EvaluatorRecord{ID: 7, EvaluatorVersionID: 101, Alias: "judge_a", Status: entity.EvaluatorRunStatusAsyncInvoking}, nil }, @@ -5344,7 +5348,8 @@ func TestDefaultExptTurnEvaluationImpl_AsyncTargetCallbackRunsMixedEvaluators(t require.True(t, isAgent || isCustomRPC, "async evaluator %d must be Agent or CustomRPC", req.EvaluatorVersionID) require.Equal(t, "ppe_trae_work_async_evaluator", gptr.Indirect(req.EvaluatorRunConf.Env)) require.NotNil(t, req.AsyncCtx) - require.Same(t, event, req.AsyncCtx.Event) + require.Equal(t, event, req.AsyncCtx.Event) + require.NotSame(t, event, req.AsyncCtx.Event) require.False(t, req.AsyncCtx.ResumeReady) return &entity.EvaluatorRecord{ID: 1000 + req.EvaluatorVersionID, EvaluatorVersionID: req.EvaluatorVersionID, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil }, diff --git a/backend/modules/evaluation/domain/service/expt_run_logging_cleanup_test.go b/backend/modules/evaluation/domain/service/expt_run_logging_cleanup_test.go index 32ce8da743..7716e2283d 100644 --- a/backend/modules/evaluation/domain/service/expt_run_logging_cleanup_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_logging_cleanup_test.go @@ -70,7 +70,7 @@ func TestLogRunPersistenceFailureReleasesOwnedLock(t *testing.T) { DoAndReturn(func(context.Context, string, string, time.Duration, time.Duration) (bool, string, error) { return acquire(), "2", nil }).Times(2) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Save(ctx, gomock.Any()).DoAndReturn(persist).Times(2) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Create(ctx, gomock.Any()).DoAndReturn(persist).Times(2) mgr.mtr.(*metricsMocks.MockExptMetric).EXPECT().EmitExptExecRun(spaceID, int64(entity.EvaluationModeRetryItems)) } else { mgr.mutex.(*lockMocks.MockILocker).EXPECT().LockBackoff(ctx, mgr.makeExptMutexLockKey(exptID), gomock.Any(), gomock.Any()). @@ -154,13 +154,11 @@ func TestLogRetryItemsPersistenceFailurePreservesExistingRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker).EXPECT().BackoffLockWithValue(ctx, mgr.makeExptMutexLockKey(exptID), "3", gomock.Any(), gomock.Any()).Return(false, "2", nil) mgr.mutex.(*lockMocks.MockILocker).EXPECT().Exists(ctx, gomock.Any()).Return(false, nil) stored := &entity.ExptRunLog{ExptID: exptID, ExptRunID: existingRunID, Status: int64(entity.ExptStatus_Processing)} - loaded := *stored - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Get(ctx, exptID, existingRunID).Return(&loaded, nil) writeErr := errors.New("append failed") - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Save(ctx, &loaded). - DoAndReturn(func(_ context.Context, run *entity.ExptRunLog) error { + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().AppendRunItemIDs(ctx, exptID, existingRunID, spaceID, []int64{5}). + DoAndReturn(func(_ context.Context, _, _, _ int64, ids []int64) error { if tc.committed { - *stored = *run + require.NoError(t, stored.AppendItemIDs(ids)) } return writeErr }) @@ -224,7 +222,7 @@ func TestLogRunLatestFailureDoesNotRollbackSupersededQuota(t *testing.T) { if retryItems { mgr.idgenerator.(*idgenMocks.MockIIDGenerator).EXPECT().GenID(ctx).Return(newRunID, nil) mgr.mutex.(*lockMocks.MockILocker).EXPECT().BackoffLockWithValue(ctx, gomock.Any(), "3", gomock.Any(), gomock.Any()).Return(true, "", nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Save(ctx, gomock.Any()).Return(nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Create(ctx, gomock.Any()).Return(nil) } else { mgr.mutex.(*lockMocks.MockILocker).EXPECT().LockBackoff(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Create(ctx, gomock.Any()).Return(nil) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index 4316c9b7c5..a4feda2dc7 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "maps" "strconv" "strings" "time" @@ -47,6 +48,7 @@ type ExptSchedulerImpl struct { Configer component.IConfiger QuotaRepo repo.QuotaRepo Mutex lock.ILocker + itemLocker ManualRetryLockerFactory AuditClient audit.IAuditService Metric metrics.ExptMetric Endpoints SchedulerEndPoint @@ -135,6 +137,9 @@ func NewExptSchedulerSvc( exptItemRefRepo: exptItemRefRepo, sandboxAgentMetrics: sandboxAgentMetrics, } + if manager, ok := manager.(*ExptMangerImpl); ok { + i.itemLocker = manager.manualRetryLocker + } if len(sandboxAgentNotifier) > 0 { i.sandboxAgentNotifier = sandboxAgentNotifier[0] } @@ -193,7 +198,42 @@ func (e *ExptSchedulerImpl) HandleEventCheck(next SchedulerEndPoint) SchedulerEn } if status := entity.ExptStatus(runLog.Status); entity.IsExptFinished(status) || entity.IsExptFinishing(status) { + if event.ExptType == entity.ExptType_Offline && event.ExptRunMode != entity.EvaluationModeAppend && (status == entity.ExptStatus_Terminated || status == entity.ExptStatus_Failed) { + if terminate, ok := e.Manager.(IExptCurrentRunTerminator); ok { + options := []entity.CompleteExptOptionFn{entity.WithStatus(status)} + if status == entity.ExptStatus_Terminated { + options = append(options, entity.NoAggrCalculate()) + } + _, err := terminate.TerminateCurrentRun(ctx, event.ExptID, event.ExptRunID, event.SpaceID, event.Session, nil, options...) + return err + } + } + if status == entity.ExptStatus_Terminating && event.ExptType == entity.ExptType_Offline && event.ExptRunMode != entity.EvaluationModeAppend { + if terminate, ok := e.Manager.(IExptCurrentRunTerminator); ok { + _, err := terminate.TerminateCurrentRun(ctx, event.ExptID, event.ExptRunID, event.SpaceID, event.Session, nil, entity.WithStatus(entity.ExptStatus_Terminated), entity.NoAggrCalculate()) + return err + } + } logs.CtxInfo(ctx, "ExptSchedulerConsumer consume finished expt run event, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) + if entity.IsExptFinished(status) && event.ExptType == entity.ExptType_Offline && event.ExptRunMode != entity.EvaluationModeAppend && event.ExptRunMode != entity.EvaluationModeTrialRun { + if finish, ok := e.Manager.(IExptNaturalRunFinisher); ok { + result, err := finish.FinishCurrentRun(ctx, event) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if result == nil { + return errno.NewMQRetryErr("finish returned no disposition") + } + switch result.Disposition { + case entity.ExptRunFinishFinished, entity.ExptRunFinishAlreadyFinished, entity.ExptRunFinishNonCurrent: + return nil + case entity.ExptRunFinishPending: + return errno.NewMQRetryErr("current run finish pending") + default: + return errno.NewMQRetryErr("invalid finish disposition") + } + } + } return nil } @@ -279,6 +319,9 @@ func (e *ExptSchedulerImpl) HandleEventErr(next SchedulerEndPoint) SchedulerEndP return next(ctx, event) }(ctx, event) + if errno.NeedMQRetry(nextErr) { + return nextErr + } if nextErr == nil { logs.CtxInfo(ctx, "[ExptEval] handle event success, event: %v", json.Jsonify(event)) return nil @@ -297,6 +340,17 @@ func (e *ExptSchedulerImpl) HandleEventErr(next SchedulerEndPoint) SchedulerEndP logs.CtxWarn(ctx, "[ExptEval] infra error detected, attempting reschedule (%d/%d), expt_id: %v, expt_run_id: %v, err: %v", event.InfraErrorRetryTimes+1, maxInfraRetry, event.ExptID, event.ExptRunID, nextErr) + if _, current := e.Manager.(IExptCurrentRunTerminator); current && event.ExptType == entity.ExptType_Offline && event.ExptRunMode != entity.EvaluationModeAppend { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + fresh := *event + fresh.InfraErrorRetryTimes++ + if err := e.Publisher.PublishExptScheduleEvent(ctx, &fresh, gptr.Of(5*time.Second)); err != nil { + return errno.WrapMQRetryErr(err) + } + return nil + } // 复制context保留存储内容，仅断开cancel传播 freshCtx := context.WithoutCancel(ctx) event.InfraErrorRetryTimes++ @@ -314,6 +368,10 @@ func (e *ExptSchedulerImpl) HandleEventErr(next SchedulerEndPoint) SchedulerEndP } } + if terminate, ok := e.Manager.(IExptCurrentRunTerminator); ok && event.ExptType == entity.ExptType_Offline && event.ExptRunMode != entity.EvaluationModeAppend { + _, err := terminate.TerminateCurrentRun(ctx, event.ExptID, event.ExptRunID, event.SpaceID, event.Session, nil, entity.WithStatus(entity.ExptStatus_Failed), entity.WithStatusMessage(userVisibleErrMsg(nextErr))) + return err + } // 业务逻辑错误：终止实验 completeCID := fmt.Sprintf("exptexec:onerr:%d", event.ExptRunID) @@ -460,6 +518,26 @@ func (e *ExptSchedulerImpl) schedule(ctx context.Context, event *entity.ExptSche nextTick, err := mode.ExptEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) if err != nil { + if _, current := e.Manager.(IExptNaturalRunFinisher); current && errno.NeedMQRetry(err) && + event.ExptType == entity.ExptType_Offline && exptDetail.ExptType == entity.ExptType_Offline && + exptDetail.LatestRunID == event.ExptRunID && + event.ExptRunMode != entity.EvaluationModeAppend && event.ExptRunMode != entity.EvaluationModeTrialRun { + if ctxErr := ctx.Err(); ctxErr != nil { + return errno.WrapMQRetryErr(ctxErr) + } + if e.Publisher == nil { + return err + } + // Keep publication ticking while the finish gate remains blocked. + fresh := *event + if pubErr := mode.NextTick(ctx, &fresh, true); pubErr != nil { + return errno.WrapMQRetryErr(pubErr) + } + if ctxErr := ctx.Err(); ctxErr != nil { + return errno.WrapMQRetryErr(ctxErr) + } + return nil + } return err } @@ -482,6 +560,9 @@ func (e *ExptSchedulerImpl) schedule(ctx context.Context, event *entity.ExptSche } func (e *ExptSchedulerImpl) recordEvalItemRunLogs(ctx context.Context, event *entity.ExptScheduleEvent, completeItems []*entity.ExptEvalItem, mode entity.ExptSchedulerMode, expt *entity.Experiment) error { + if publicationSvc, ok := e.ResultSvc.(ExptResultPublicationService); ok { + return e.recordCurrentItemResults(ctx, event, completeItems, mode, expt, publicationSvc) + } time.Sleep(time.Millisecond * 1000) // avoid master-slave delay caused by asynchronous and other factors // 循环外批量补齐 item-complete 事件组装所需的 per-item 归属集/版本/ItemKey(供循环内 sendItemComplete 用)。 @@ -645,12 +726,18 @@ func (e *ExptSchedulerImpl) resolveItemCompleteMeta(ctx context.Context, event * // sendItemComplete 组装并发送单行 item-complete(success) 事件。 // 发送失败返回 error, 由调用方中断本次调度、靠下次调度重扫 completeItems 补发; 元数据缺失属组装侧问题, 跳过不阻断。 func (e *ExptSchedulerImpl) sendItemComplete(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, item *entity.ExptEvalItem, evalSetItem *entity.EvaluationSetItem, evalSetVersionID int64) error { + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } // 元数据缺失(resolveItemCompleteMeta 已 CtxWarn)无法靠重试恢复, 跳过不阻断本次调度。 if evalSetItem == nil { logs.CtxWarn(ctx, "[ExptEval] item complete meta missing, skip publish, expt_id: %v, item_id: %v", event.ExptID, item.ItemID) return nil } completeEvent := BuildItemCompleteEventFromScheduler(event.SpaceID, event.ExptID, event.ExptRunID, expt, item, evalSetItem, evalSetVersionID) + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } if err := e.itemCompletePublisher.PublishItemComplete(ctx, completeEvent); err != nil { logs.CtxError(ctx, "[ExptEval] publish item complete event failed, expt_id: %v, item_id: %v, err: %v", event.ExptID, item.ItemID, err) return err @@ -659,74 +746,72 @@ func (e *ExptSchedulerImpl) sendItemComplete(ctx context.Context, event *entity. } func (e *ExptSchedulerImpl) handleToSubmits(ctx context.Context, event *entity.ExptScheduleEvent, toSubmits []*entity.ExptEvalItem) error { - if len(toSubmits) == 0 { - return nil - } - - now := time.Now().Unix() - itemIDs := make([]int64, 0, len(toSubmits)) - itemEvalEvents := make([]*entity.ExptItemEvalEvent, 0, len(toSubmits)) - for _, ts := range toSubmits { - if entity.IsItemRunFinished(ts.State) { + for _, candidate := range toSubmits { + if candidate == nil || candidate.State != entity.ItemRunState_Queueing { continue } - itemIDs = append(itemIDs, ts.ItemID) - itemEvalEvents = append(itemEvalEvents, &entity.ExptItemEvalEvent{ - SpaceID: event.SpaceID, - ExptID: event.ExptID, - ExptRunID: event.ExptRunID, - ExptRunMode: event.ExptRunMode, - EvalSetItemID: ts.ItemID, - CreateAt: now, - RetryTimes: int(ts.RetryTimes), // ★ 回填持久重试次数: 让位后由调度器新建的事件据此判收敛(需重试→让位, 达上限→落 Fail) - MaxRetryTimes: event.ItemRetryTimes, - Ext: event.Ext, - Session: event.Session, - }) + if e.itemLocker == nil { + return fmt.Errorf("item locker factory unavailable") + } + mutex := e.itemLocker() + if err := e.submitCurrentItem(ctx, event, candidate, mutex); err != nil { + return errno.WrapMQRetryErr(err) + } } + return nil +} - logs.CtxInfo(ctx, "submit item eval events: %v", json.Jsonify(itemEvalEvents)) - - interval := e.Configer.GetExptExecConf(ctx, event.SpaceID).GetExptItemEvalConf().GetInterval() - if err := e.Publisher.BatchPublishExptRecordEvalEvent(ctx, itemEvalEvents, gptr.Of(interval)); err != nil { +func (e *ExptSchedulerImpl) submitCurrentItem(ctx context.Context, event *entity.ExptScheduleEvent, candidate *entity.ExptEvalItem, mutex lock.ILocker) error { + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", event.ExptID, candidate.ItemID) + locked, lockCtx, cancel, err := mutex.LockWithRenew(ctx, key, 5*time.Second, 5*time.Minute) + if err != nil { return err } - - defer e.Metric.EmitItemExecEval(event.SpaceID, int64(event.ExptRunMode), len(toSubmits)) - - if err := e.ExptItemResultRepo.UpdateItemRunLog(ctx, event.ExptID, event.ExptRunID, itemIDs, map[string]any{"status": int32(entity.ItemRunState_Processing)}, - event.SpaceID); err != nil { - return err + if !locked { + return nil } - - if err := e.ExptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, itemIDs, map[string]any{"status": int32(entity.ItemRunState_Processing)}); err != nil { + defer func() { + cancel() + if _, err := mutex.Unlock(key); err != nil { + logs.CtxWarn(ctx, "unlock producer item: %v", err) + } + }() + source, err := e.ExptItemResultRepo.GetCurrentRequest(lockCtx, event.SpaceID, event.ExptID, event.ExptRunID, candidate.ItemID) + if err != nil { return err } - - err := e.ResultSvc.UpsertExptTurnResultFilter(ctx, event.SpaceID, event.ExptID, itemIDs) - if err != nil { - logs.CtxError(ctx, "ExptSubmitExec.ExptStart UpsertExptTurnResultFilter fail, expt_id: %v, err: %v", event.ExptID, err) + if !source.Matches(event.ExptRunID, candidate.RequestID) || source.Status != entity.ItemRunState_Queueing { + return nil } - logs.CtxInfo(ctx, "ExptSchedulerImpl handleToSubmits UpsertExptTurnResultFilter success, expt_id: %v", event.ExptID) - - if err := e.ExptTurnResultRepo.UpdateTurnResultsWithItemIDs(ctx, event.ExptID, itemIDs, event.SpaceID, map[string]any{"status": int32(entity.TurnRunState_Processing)}); err != nil { + if source.Mode == entity.EvaluationModeUnknown { + return fmt.Errorf("current item mode unavailable") + } + itemEvent := &entity.ExptItemEvalEvent{RequestID: source.RequestID, SpaceID: event.SpaceID, ExptID: event.ExptID, ExptRunID: event.ExptRunID, ExptRunMode: source.Mode, EvalSetItemID: source.ItemID, CreateAt: time.Now().Unix(), RetryTimes: int(source.RetryTimes), MaxRetryTimes: event.ItemRetryTimes, Ext: maps.Clone(event.Ext)} + if event.Session != nil { + session := *event.Session + itemEvent.Session = &session + } + interval := e.Configer.GetExptExecConf(lockCtx, event.SpaceID).GetExptItemEvalConf().GetInterval() + if err := lockCtx.Err(); err != nil { return err } - - itemResults, err := e.ExptItemResultRepo.BatchGet(ctx, event.SpaceID, event.ExptID, itemIDs) - if err != nil { + if err := e.Publisher.BatchPublishExptRecordEvalEvent(lockCtx, []*entity.ExptItemEvalEvent{itemEvent}, gptr.Of(interval)); err != nil { return err } - - if err := e.ExptStatsRepo.ArithOperateCount(ctx, event.ExptID, event.SpaceID, &entity.StatsCntArithOp{ - OpStatusCnt: map[entity.ItemRunState]int{ - entity.ItemRunState_Processing: len(itemResults), - entity.ItemRunState_Queueing: 0 - len(itemResults), - }, - }); err != nil { + changed, err := e.ExptItemResultRepo.MarkProcessingIfCurrent(lockCtx, event.SpaceID, event.ExptID, event.ExptRunID, source.ItemID, source.RequestID) + if err != nil { return err } - + if changed { + if e.Metric != nil { + e.Metric.EmitItemExecEval(event.SpaceID, int64(source.Mode), 1) + } + if e.ResultSvc != nil { + if err := refreshFilterWithHeldItemLock(lockCtx, e.ResultSvc, event.SpaceID, event.ExptID, source.ItemID); err != nil { + logs.CtxWarn(ctx, "refresh dispatched item projection: %v", err) + } + } + } return nil } @@ -736,6 +821,15 @@ func (e *ExptSchedulerImpl) handleZombies(ctx context.Context, event *entity.Exp asyncExec = expt.AsyncExec() } zombieSecond := e.Configer.GetConsumerConf(ctx).GetExptExecConf(event.SpaceID).GetExptItemEvalConf().GetItemZombieSecond(asyncExec) + if _, ok := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + return e.handleCurrentZombies(ctx, event, items, expt, zombieSecond, asyncExec) + } + for _, item := range items { + if item != nil && item.RequestID != "" && item.Source == nil { + return nil, nil, errno.NewMQRetryErr("same-scan original item source missing") + } + } + zombieRequestIDs := scannedQuotaRequestIDs(items) for _, item := range items { if item.State == entity.ItemRunState_Processing && item.UpdatedAt != nil && !gptr.Indirect(item.UpdatedAt).IsZero() { if time.Since(gptr.Indirect(item.UpdatedAt)).Seconds() > float64(zombieSecond) { @@ -790,7 +884,7 @@ func (e *ExptSchedulerImpl) handleZombies(ctx context.Context, event *entity.Exp // item 已落终态 → 释放其额度预占。放在 run log 写库之后：调度侧判占用读的是 // run log（LoadDispatchRuntime 按 status IN (Queueing, Processing) 扫）， // 先落 Fail 再放额度，才不会留下"额度已归还、run log 仍算占用"的窗口。 - e.releaseCentralQuotaForItems(ctx, expt, event.ExptRunID, zombieItemIDs, "item zombie timeout") + e.releaseCentralQuotaForItems(ctx, expt, event.ExptRunID, zombieItemIDs, zombieRequestIDs, "item zombie timeout") // 不清 run_log 的 target_result_id / evaluator_result_ids： // zombie 场景是「终态失败」，需要保留已入库的 record id， @@ -822,9 +916,17 @@ func (e *ExptSchedulerImpl) terminateZombieEvaluatorRecords(ctx context.Context, return nil } - turnRunLogs, err := e.ExptTurnResultRepo.MGetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, zombieItemIDs, event.SpaceID) - if err != nil { - return err + var turnRunLogs []*entity.ExptTurnResultRunLog + if execution := executionSource(ctx); execution != nil && execution.Source != nil { + for _, log := range execution.Source.Turns { + turnRunLogs = append(turnRunLogs, log) + } + } else { + var err error + turnRunLogs, err = e.ExptTurnResultRepo.MGetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, zombieItemIDs, event.SpaceID) + if err != nil { + return err + } } recordIDSet := make(map[int64]struct{}) @@ -903,6 +1005,14 @@ func (e *ExptSchedulerImpl) sweepTerminatedSandboxItems(ctx context.Context, eve return items, nil, nil } + if _, ok := e.ExptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + return e.sweepCurrentSandboxItems(ctx, event, items, expt) + } + for _, item := range items { + if item != nil && item.RequestID != "" && item.Source == nil { + return nil, nil, errno.NewMQRetryErr("same-scan original item source missing") + } + } processingItemIDs := make([]int64, 0, len(items)) for _, item := range items { if item == nil || item.State != entity.ItemRunState_Processing { @@ -961,6 +1071,7 @@ func (e *ExptSchedulerImpl) sweepTerminatedSandboxItems(ctx context.Context, eve firstStatus = "Terminated" } + terminatedRequestIDs := scannedQuotaRequestIDs(items) alives = make([]*entity.ExptEvalItem, 0, len(items)) terminated = make([]*entity.ExptEvalItem, 0, len(terminatedItemIDSet)) for _, item := range items { @@ -1014,7 +1125,7 @@ func (e *ExptSchedulerImpl) sweepTerminatedSandboxItems(ctx context.Context, eve // item 已落终态 → 释放额度预占。沙箱已提前终止，对应 consumer 消息不会再回来， // 不在此释放则这些 item 的额度永久泄漏。 - e.releaseCentralQuotaForItems(ctx, expt, event.ExptRunID, terminatedItemIDs, "sandbox terminated before report") + e.releaseCentralQuotaForItems(ctx, expt, event.ExptRunID, terminatedItemIDs, terminatedRequestIDs, "sandbox terminated before report") // 打点：复用沙箱 agent 评测对象的 EmitInvokeFinished，让"沙箱终态未回调 → 兜底失败" // 与正常回调路径的 invoke_finished 在同一 dashboard 上可比。err_code 用 @@ -1089,9 +1200,22 @@ func (e *ExptSchedulerImpl) terminateZombieEvalTargetRecords(ctx context.Context return nil } - turnRunLogs, err := e.ExptTurnResultRepo.MGetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, zombieItemIDs, event.SpaceID) - if err != nil { - return err + var turnRunLogs []*entity.ExptTurnResultRunLog + if execution := executionSource(ctx); execution != nil && execution.Source != nil { + for _, log := range execution.Source.Turns { + turnRunLogs = append(turnRunLogs, log) + } + for _, turn := range execution.Source.CanonicalTurns { + if execution.Source.Turns[turn.TurnID] == nil { + turnRunLogs = append(turnRunLogs, turn.ToRunLogDO()) + } + } + } else { + var err error + turnRunLogs, err = e.ExptTurnResultRepo.MGetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, zombieItemIDs, event.SpaceID) + if err != nil { + return err + } } // recordID -> 关联 itemIDs, 打点时需要按 (record, item) 展开 tag @@ -1215,7 +1339,22 @@ func sandboxAgentTargetTagsFromExpt(expt *entity.Experiment) (agentName string, // legacy 实验直接返回：它们从不预占额度，调用释放只是无谓的 Redis 往返。 // 全程 best-effort：单个 item 释放失败只告警，不影响 zombie 清理本身 —— // 清理失败会让实验永不收敛，比额度泄漏更严重，而额度有对账兜底。 -func (e *ExptSchedulerImpl) releaseCentralQuotaForItems(ctx context.Context, expt *entity.Experiment, exptRunID int64, itemIDs []int64, reason string) { +func scannedQuotaRequestIDs(items []*entity.ExptEvalItem) map[int64]string { + requests := make(map[int64]string, len(items)) + for _, item := range items { + if item == nil { + continue + } + request := item.RequestID + if item.Source != nil { + request = item.Source.Current.RequestID + } + requests[item.ItemID] = request + } + return requests +} + +func (e *ExptSchedulerImpl) releaseCentralQuotaForItems(ctx context.Context, expt *entity.Experiment, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string, reason string) { if e.centralGuard == nil || expt == nil || len(itemIDs) == 0 { return } @@ -1230,8 +1369,15 @@ func (e *ExptSchedulerImpl) releaseCentralQuotaForItems(ctx context.Context, exp return } + relCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) + defer cancel() for _, itemID := range itemIDs { - if err := e.centralGuard.Release(ctx, expt.SchedulerScope, exptRunID, itemID, reason); err != nil { + request, observed := itemRequestIDs[itemID] + if !observed { + logs.CtxWarn(relCtx, "[CentralReservation] original scanned request missing, skip release, item_id: %v", itemID) + continue + } + if err := e.centralGuard.Release(relCtx, expt.SchedulerScope, exptRunID, itemID, request, reason); err != nil { logs.CtxWarn(ctx, "[CentralReservation] release quota fail, scope: %v, expt_run_id: %v, item_id: %v, reason: %v, err: %v", expt.SchedulerScope, exptRunID, itemID, reason, err) } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go index 391a15e250..62d82aa741 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/coze-dev/coze-loop/backend/infra/lock" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" "github.com/stretchr/testify/assert" @@ -362,6 +364,7 @@ func TestExptSchedulerImpl_SubmitItemEval(t *testing.T) { publisher *eventmocks.MockExptEventPublisher metric *metricsmocks.MockExptMetric resultSvc *svcmocks.MockExptResultService + mutex *lockmocks.MockILocker } type args struct { @@ -402,9 +405,10 @@ func TestExptSchedulerImpl_SubmitItemEval(t *testing.T) { }, }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { // Modification: add ctrl parameter - f.exptItemResultRepo.EXPECT().UpdateItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - f.exptItemResultRepo.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{}, nil).AnyTimes() + f.mutex.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, args.ctx, func() {}, nil) + f.mutex.EXPECT().Unlock(gomock.Any()).Return(true, nil) + f.exptItemResultRepo.EXPECT().GetCurrentRequest(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, int64(3)).Return(&entity.ExptItemCurrentRequest{RunID: args.event.ExptRunID, ItemID: 3, Status: entity.ItemRunState_Queueing, Mode: entity.EvaluationModeSubmit}, nil) + f.exptItemResultRepo.EXPECT().MarkProcessingIfCurrent(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, int64(3), "").Return(true, nil) f.exptTurnResultRepo.EXPECT().UpdateTurnResultsWithItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() f.exptTurnResultRepo.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{}, nil).AnyTimes() f.publisher.EXPECT().BatchPublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -439,6 +443,7 @@ func TestExptSchedulerImpl_SubmitItemEval(t *testing.T) { publisher: eventmocks.NewMockExptEventPublisher(ctrl), metric: metricsmocks.NewMockExptMetric(ctrl), resultSvc: svcmocks.NewMockExptResultService(ctrl), + mutex: lockmocks.NewMockILocker(ctrl), } if tt.prepareMock != nil { @@ -453,6 +458,7 @@ func TestExptSchedulerImpl_SubmitItemEval(t *testing.T) { Publisher: f.publisher, Metric: f.metric, ResultSvc: f.resultSvc, + itemLocker: func() lock.ILocker { return f.mutex }, } err := svc.handleToSubmits(tt.args.ctx, tt.args.event, tt.args.toSubmits) @@ -2906,7 +2912,11 @@ func TestExptSchedulerImpl_handleToSubmits_BackfillRetryTimes(t *testing.T) { metric := metricsmocks.NewMockExptMetric(ctrl) resultSvc := svcmocks.NewMockExptResultService(ctrl) + mutex := lockmocks.NewMockILocker(ctrl) + mutex.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil).Times(2) + mutex.EXPECT().Unlock(gomock.Any()).Return(true, nil).Times(2) svc := &ExptSchedulerImpl{ + itemLocker: func() lock.ILocker { return mutex }, ExptItemResultRepo: itemRepo, ExptTurnResultRepo: turnRepo, ExptStatsRepo: statsRepo, @@ -2923,19 +2933,18 @@ func TestExptSchedulerImpl_handleToSubmits_BackfillRetryTimes(t *testing.T) { var captured []*entity.ExptItemEvalEvent publisher.EXPECT().BatchPublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, events []*entity.ExptItemEvalEvent, _ *time.Duration) error { - captured = events + captured = append(captured, events...) return nil - }) - itemRepo.EXPECT().UpdateItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - itemRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - itemRepo.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{}, nil) - turnRepo.EXPECT().UpdateTurnResultsWithItemIDs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - statsRepo.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - resultSvc.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - metric.EXPECT().EmitItemExecEval(gomock.Any(), gomock.Any(), gomock.Any()) + }).Times(2) + for _, source := range []*entity.ExptItemCurrentRequest{{RunID: 2, ItemID: 10, Mode: entity.EvaluationModeSubmit, Status: entity.ItemRunState_Queueing, RetryTimes: 0}, {RunID: 2, ItemID: 11, Mode: entity.EvaluationModeSubmit, Status: entity.ItemRunState_Queueing, RetryTimes: 3}} { + itemRepo.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), source.ItemID).Return(source, nil) + itemRepo.EXPECT().MarkProcessingIfCurrent(gomock.Any(), int64(3), int64(1), int64(2), source.ItemID, "").Return(true, nil) + } + resultSvc.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) + metric.EXPECT().EmitItemExecEval(gomock.Any(), gomock.Any(), gomock.Any()).Times(2) event := &entity.ExptScheduleEvent{ - ExptID: 1, ExptRunID: 2, SpaceID: 3, ItemRetryTimes: 5, + ExptID: 1, ExptRunID: 2, SpaceID: 3, ItemRetryTimes: 5, ExptRunMode: entity.EvaluationModeSubmit, Ext: map[string]string{entity.RetryYieldExtKey: "true"}, } toSubmits := []*entity.ExptEvalItem{ diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_finish_tick_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_finish_tick_test.go new file mode 100644 index 0000000000..6da88c822a --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_finish_tick_test.go @@ -0,0 +1,281 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// Only startup and scan inputs are controlled; ExptEnd, calibration and NextTick stay real. +type finishTickMode struct { + entity.ExptSchedulerMode + startErr, scheduleErr, scanErr, endErr error + beforeEnd func() +} + +func (m *finishTickMode) ExptStart(context.Context, *entity.ExptScheduleEvent, *entity.Experiment) error { + return m.startErr +} + +func (m *finishTickMode) ScheduleStart(context.Context, *entity.ExptScheduleEvent, *entity.Experiment) error { + return m.scheduleErr +} + +func (m *finishTickMode) ScanEvalItems(context.Context, *entity.ExptScheduleEvent, *entity.Experiment) ([]*entity.ExptEvalItem, []*entity.ExptEvalItem, []*entity.ExptEvalItem, error) { + return nil, nil, nil, m.scanErr +} + +func (m *finishTickMode) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (bool, error) { + if m.beforeEnd != nil { + m.beforeEnd() + } + if m.endErr != nil { + return false, m.endErr + } + return m.ExptSchedulerMode.ExptEnd(ctx, event, expt, toSubmit, incomplete) +} + +type finishTickCalibration struct { + ExptResultService + ready bool + err error + calls int +} + +func (r *finishTickCalibration) ReconcileManualRetryFilterBeforeFinish(context.Context, int64, int64, int64) (bool, error) { + r.calls++ + return r.ready, r.err +} + +type finishTickFixture struct { + scheduler *ExptSchedulerImpl + mode *finishTickMode + manager *naturalFinishProbe + result *finishTickCalibration + publisher *eventmocks.MockExptEventPublisher + event *entity.ExptScheduleEvent + expt *entity.Experiment +} + +func newFinishTickFixture(t *testing.T, runMode entity.ExptRunMode, dispatch string) *finishTickFixture { + t.Helper() + ctrl := gomock.NewController(t) + event := &entity.ExptScheduleEvent{ + SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptType: entity.ExptType_Offline, ExptRunMode: runMode, + CreatedAt: time.Now().Unix(), Ext: map[string]string{"__item_ids": "[10,11]", "retry_yield_enabled": "false", "other": "keep"}, + Session: &entity.Session{UserID: "operator", AppID: 7}, ItemRetryTimes: 4, InfraErrorRetryTimes: 6, ExecEvalSetItemIDs: []int64{10, 11}, + } + expt := &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline, ExptDispatchMode: dispatch} + oldManager := svcmocks.NewMockIExptManager(ctrl) + oldManager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), event.Session).Return(expt, nil).AnyTimes() + oldManager.EXPECT().GetRunLog(gomock.Any(), int64(1), int64(2), int64(3), event.Session).Return(&entity.ExptRunLog{Status: int64(entity.ExptStatus_Processing)}, nil).AnyTimes() + manager := &naturalFinishProbe{IExptManager: oldManager, disposition: entity.ExptRunFinishPending} + items := repomocks.NewMockIExptItemResultRepo(ctrl) + items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), int64(0), int64(1), int64(3)).Return(nil, int64(0), nil).AnyTimes() + idem := idemmocks.NewMockIdempotentService(ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetConsumerConf(gomock.Any()).Return(&entity.ExptConsumerConf{}).AnyTimes() + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 17}).AnyTimes() + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + result := &finishTickCalibration{} + var realMode entity.ExptSchedulerMode + switch runMode { + case entity.EvaluationModeSubmit: + realMode = &ExptSubmitExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result, publisher: publisher} + case entity.EvaluationModeFailRetry: + realMode = &ExptFailRetryExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result, publisher: publisher} + case entity.EvaluationModeRetryAll: + realMode = &ExptRetryAllExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result, publisher: publisher} + case entity.EvaluationModeRetryItems: + runLogs := repomocks.NewMockIExptRunLogRepo(ctrl) + runLogs.EXPECT().Get(gomock.Any(), int64(1), int64(2)).Return(&entity.ExptRunLog{}, nil).AnyTimes() + oldManager.EXPECT().LockCompletingRun(gomock.Any(), int64(1), int64(2), int64(3), event.Session).Return(nil).AnyTimes() + oldManager.EXPECT().UnlockCompletingRun(gomock.Any(), int64(1), int64(2), int64(3), event.Session).Return(nil).AnyTimes() + realMode = &ExptRetryItemsExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, exptRunLogRepo: runLogs, resultSvc: result, publisher: publisher} + } + mode := &finishTickMode{ExptSchedulerMode: realMode} + factory := svcmocks.NewMockSchedulerModeFactory(ctrl) + factory.EXPECT().NewSchedulerMode(gomock.Any()).Return(mode, nil).AnyTimes() + scheduler := &ExptSchedulerImpl{Manager: manager, Configer: config, Publisher: publisher, ResultSvc: result, schedulerModeFactory: factory} + return &finishTickFixture{scheduler: scheduler, mode: mode, manager: manager, result: result, publisher: publisher, event: event, expt: expt} +} + +// Removing finish-error rearming strands the normal cadence even though calibration blocks completion. +func TestFinishCalibrationTickLiveness(t *testing.T) { + for _, dispatch := range []string{entity.ExptDispatchModeLegacy, entity.ExptDispatchModeEnforce} { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeSubmit, entity.EvaluationModeFailRetry, entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems} { + t.Run(fmt.Sprintf("%s/mode%d", dispatch, mode), func(t *testing.T) { + f := newFinishTickFixture(t, mode, dispatch) + for _, state := range []string{"calibration_pending", "calibration_error", "finish_pending"} { + t.Run(state, func(t *testing.T) { + f.result.ready = state == "finish_pending" + f.result.err = nil + if state == "calibration_error" { + f.result.err = errors.New("projection unavailable") + } + before := *f.event + var published *entity.ExptScheduleEvent + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, next *entity.ExptScheduleEvent, delay *time.Duration) error { + require.NoError(t, ctx.Err()) + require.NotSame(t, f.event, next, "broker event must retain its original retry counters") + require.Equal(t, before, *next) + require.Equal(t, 17*time.Second, *delay) + published = next + return nil + }) + err := f.scheduler.HandleEventErr(f.scheduler.schedule)(context.Background(), f.event) + require.NoError(t, err, "successful next-tick publication should ACK current event") + require.NotNil(t, published) + require.Equal(t, before, *f.event) + require.Equal(t, entity.ExptStatus_Processing, f.expt.Status) + if state != "finish_pending" { + require.Zero(t, f.manager.calls, "calibration must block the finish transaction") + } + }) + } + require.Equal(t, 3, f.result.calls) + require.Equal(t, 1, f.manager.calls, "only calibrated pending may reach finish composition") + }) + } + } +} + +func TestFinishCalibrationTickBrokerFallback(t *testing.T) { + for _, fault := range []string{"publish_failure", "cancel_before_end", "cancel_during_publish", "nil_publisher", "fatal_end"} { + t.Run(fault, func(t *testing.T) { + f := newFinishTickFixture(t, entity.EvaluationModeSubmit, entity.ExptDispatchModeEnforce) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sentinel := errors.New("publisher unavailable") + switch fault { + case "publish_failure", "cancel_during_publish": + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.ExptScheduleEvent, *time.Duration) error { + if fault == "cancel_during_publish" { + cancel() + return nil + } + return sentinel + }) + case "cancel_before_end": + f.mode.beforeEnd = cancel + case "nil_publisher": + f.scheduler.Publisher = nil + case "fatal_end": + f.mode.endErr = sentinel + } + before := *f.event + err := f.scheduler.schedule(ctx, f.event) + require.Error(t, err) + if fault == "fatal_end" { + require.Same(t, sentinel, err) + } else { + require.True(t, errno.NeedMQRetry(err), "fallback must stay visible to broker") + } + if fault == "publish_failure" { + require.ErrorIs(t, err, sentinel) + } + if fault == "cancel_before_end" || fault == "cancel_during_publish" { + require.ErrorIs(t, err, context.Canceled) + } + require.Equal(t, before, *f.event) + require.Zero(t, f.manager.calls) + }) + } +} + +func TestFinishCalibrationTickExcludedModesAndOldManager(t *testing.T) { + for _, scope := range []string{"online", "append", "trial", "old_manager", "noncurrent"} { + t.Run(scope, func(t *testing.T) { + f := newFinishTickFixture(t, entity.EvaluationModeSubmit, entity.ExptDispatchModeLegacy) + retry := errno.NewMQRetryErr("typed end retry") + f.mode.endErr = retry + switch scope { + case "online": + f.event.ExptType = entity.ExptType_Online + f.expt.ExptType = entity.ExptType_Online + case "append": + f.event.ExptRunMode = entity.EvaluationModeAppend + case "trial": + f.event.ExptRunMode = entity.EvaluationModeTrialRun + case "old_manager": + f.scheduler.Manager = f.manager.IExptManager + case "noncurrent": + f.expt.LatestRunID = 99 + } + err := f.scheduler.HandleEventErr(f.scheduler.schedule)(context.Background(), f.event) + require.Same(t, retry, err) + require.Zero(t, f.manager.calls) + }) + } +} + +func TestFinishCalibrationTickPriorErrorsKeepBrokerRetry(t *testing.T) { + for _, stage := range []string{"start", "schedule_start", "scan", "lock_busy"} { + t.Run(stage, func(t *testing.T) { + f := newFinishTickFixture(t, entity.EvaluationModeSubmit, entity.ExptDispatchModeLegacy) + retry := errno.NewMQRetryErr("typed earlier retry") + endpoint := f.scheduler.schedule + switch stage { + case "start": + f.mode.startErr = retry + case "schedule_start": + f.mode.scheduleErr = retry + case "scan": + f.mode.scanErr = retry + case "lock_busy": + locker := lockmocks.NewMockILocker(gomock.NewController(t)) + locker.EXPECT().LockBackoffWithRenew(gomock.Any(), "expt_run_exec_lock:1:2", gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, retry) + f.scheduler.Mutex = locker + endpoint = f.scheduler.HandleEventLock(endpoint) + } + err := f.scheduler.HandleEventErr(endpoint)(context.Background(), f.event) + require.Same(t, retry, err) + require.Zero(t, f.result.calls) + require.Zero(t, f.manager.calls) + }) + } +} + +// A broker duplicate must still enter the existing run mutex before any calibration or tick publication. +func TestFinishCalibrationTickDuplicateUsesRunLock(t *testing.T) { + f := newFinishTickFixture(t, entity.EvaluationModeRetryItems, entity.ExptDispatchModeEnforce) + locker := lockmocks.NewMockILocker(gomock.NewController(t)) + held := false + locker.EXPECT().LockBackoffWithRenew(gomock.Any(), "expt_run_exec_lock:1:2", gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ string, _, _ time.Duration) (bool, context.Context, func(), error) { + require.False(t, held) + held = true + return true, ctx, func() {}, nil + }).Times(2) + locker.EXPECT().Unlock("expt_run_exec_lock:1:2").DoAndReturn(func(string) (bool, error) { require.True(t, held); held = false; return true, nil }).Times(2) + f.scheduler.Mutex = locker + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.ExptScheduleEvent, *time.Duration) error { + require.True(t, held) + return nil + }).Times(2) + endpoint := f.scheduler.HandleEventErr(f.scheduler.HandleEventCheck(f.scheduler.HandleEventLock(f.scheduler.schedule))) + for i := 0; i < 2; i++ { + require.NoError(t, endpoint(context.Background(), f.event)) + require.False(t, held) + } + require.Equal(t, 2, f.result.calls) + require.Zero(t, f.manager.calls) +} + +var _ lock.ILocker = (*lockmocks.MockILocker)(nil) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index 465c93dbd2..aede438d25 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -24,6 +24,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" "github.com/coze-dev/coze-loop/backend/pkg/lang/conv" @@ -731,7 +732,7 @@ func (e *ExptSubmitExec) ScanEvalItems(ctx context.Context, event *entity.ExptSc func (e *ExptSubmitExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { if toSubmit == 0 && incomplete == 0 { - base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService) + base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService, e.resultSvc) pending, err := base.hasPendingAfterRescan(ctx, event) if err != nil { return false, err @@ -1020,7 +1021,7 @@ func (e *ExptFailRetryExec) ExptStart(ctx context.Context, event *entity.ExptSch return err } - if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, plan.itemIDs, map[string]any{ + if err := updateNewRunItemResults(ctx, e.manager, e.exptItemResultRepo, event, plan.itemIDs, map[string]any{ "err_msg": nil, "status": int32(entity.ItemRunState_Queueing), "expt_run_id": event.ExptRunID, @@ -1131,7 +1132,7 @@ func (e *ExptFailRetryExec) ScanEvalItems(ctx context.Context, event *entity.Exp func (e *ExptFailRetryExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { if toSubmit == 0 && incomplete == 0 { - base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService) + base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService, e.resultSvc) pending, err := base.hasPendingAfterRescan(ctx, event) if err != nil { return false, err @@ -1353,6 +1354,7 @@ func (e *ExptAppendExec) PublishResult(ctx context.Context, turnEvaluatorRefs [] } type exptBaseExec struct { + resultSvc ExptResultService Manager IExptManager idem idem.IdempotentService configer component.IConfiger @@ -1368,8 +1370,9 @@ func newExptBaseExec( exptItemResultRepo repo.IExptItemResultRepo, publisher events.ExptEventPublisher, evaluatorRecordService EvaluatorRecordService, + resultSvc ...ExptResultService, ) *exptBaseExec { - return &exptBaseExec{ + base := &exptBaseExec{ Manager: manager, idem: idem, configer: configer, @@ -1377,25 +1380,51 @@ func newExptBaseExec( evaluatorRecordService: evaluatorRecordService, publisher: publisher, } + if len(resultSvc) > 0 { + base.resultSvc = resultSvc[0] + } + return base } func (e *exptBaseExec) ScanEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (toSubmit, incomplete, complete []*entity.ExptEvalItem, err error) { - incomplete, complete, err = e.scanIncompleteAndComplete(ctx, event, expt) + rows, err := e.exptItemResultRepo.ScanCurrentRequests(ctx, event.SpaceID, event.ExptID, event.ExptRunID, event.Ext[entity.RetryYieldExtKey] == "true") if err != nil { - return nil, nil, nil, err + return nil, nil, nil, errno.WrapMQRetryErr(err) } - - if submitCnt := e.getItemConcurNum(ctx, expt) - len(incomplete); submitCnt > 0 { - toSubmit, err = e.scanToSubmit(ctx, event, expt, int64(submitCnt)) - if err != nil { - return nil, nil, nil, err + candidates := make([]*entity.ExptEvalItem, 0) + toSubmit = make([]*entity.ExptEvalItem, 0) + incomplete = make([]*entity.ExptEvalItem, 0) + complete = make([]*entity.ExptEvalItem, 0) + for _, source := range rows { + if source == nil { + return nil, nil, nil, errno.NewMQRetryErr("nil current item source") + } + item := &entity.ExptEvalItem{ExptID: event.ExptID, EvalSetVersionID: expt.EvalSet.EvaluationSetVersion.ID, ItemID: source.ItemID, Publication: source.Publication, PublicationObserved: source.PublicationObserved, Source: source.Source, State: source.Status, RequestID: source.RequestID, Mode: source.Mode, RetryTimes: source.RetryTimes, UpdatedAt: source.UpdatedAt} + if source.Status == entity.ItemRunState_Processing || source.QuotaReservationState.IsQuotaReserved() { + incomplete = append(incomplete, item) + } + if source.ResultState == entity.ExptItemResultStateLogged { + complete = append(complete, item) + } + if source.Status == entity.ItemRunState_Queueing && !source.QuotaReservationState.IsQuotaReserved() { + candidates = append(candidates, item) } } - + limit := e.getItemConcurNum(ctx, expt) - len(incomplete) + if limit > 0 { + if limit > len(candidates) { + limit = len(candidates) + } + toSubmit = append(toSubmit, candidates[:limit]...) + } return toSubmit, incomplete, complete, nil } func (e *exptBaseExec) scanIncompleteAndComplete(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (incomplete, complete []*entity.ExptEvalItem, err error) { + if _, ok := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); ok { + _, incomplete, complete, err = e.ScanEvalItems(ctx, event, expt) + return incomplete, complete, err + } rls, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, &entity.ExptItemRunLogFilter{ RawFilter: true, RawCond: clause.Expr{SQL: "status IN (?) OR result_state = ?", Vars: []interface{}{[]int32{int32(entity.ItemRunState_Processing)}, int32(entity.ExptItemResultStateLogged)}}, @@ -1481,6 +1510,49 @@ func (e *exptBaseExec) exptEnd(ctx context.Context, event *entity.ExptScheduleEv return nil } + if event.ExptRunMode != entity.EvaluationModeAppend && event.ExptRunMode != entity.EvaluationModeTrialRun && expt.ExptType != entity.ExptType_Online { + if calibration, ok := e.resultSvc.(ExptManualRetryFilterService); ok { + ready, err := reconcileManualRetryFilterBeforeFinish(ctx, calibration, event.SpaceID, event.ExptID, event.ExptRunID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !ready { + return errno.NewMQRetryErr("manual retry filter calibration pending") + } + } else if _, current := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); current { + return errno.NewMQRetryErr("manual retry filter calibration service missing") + } + } + + if event.ExptRunMode != entity.EvaluationModeAppend && event.ExptRunMode != entity.EvaluationModeTrialRun && expt.ExptType != entity.ExptType_Online { + if finish, ok := e.Manager.(IExptNaturalRunFinisher); ok { + result, err := finish.FinishCurrentRunWithHeldTickLock(ctx, event) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if result == nil { + return errno.NewMQRetryErr("finish returned no disposition") + } + switch result.Disposition { + case entity.ExptRunFinishPending: + return errno.NewMQRetryErr("current run finish pending") + case entity.ExptRunFinishNonCurrent: + return nil + case entity.ExptRunFinishFinished, entity.ExptRunFinishAlreadyFinished: + duration := time.Duration(e.configer.GetExptExecConf(ctx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 + if err := e.idem.Set(ctx, idemKey, duration); err != nil { + logs.CtxWarn(ctx, "finish end idem set failed: %v", err) + } + return nil + default: + return errno.NewMQRetryErr("invalid finish disposition") + } + } + if _, current := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); current { + return errno.NewMQRetryErr("current run finisher missing") + } + } + completeCID := fmt.Sprintf("exptexec:onend:%d", event.ExptRunID) if err := e.Manager.CompleteRun(ctx, event.ExptID, event.ExptRunID, event.SpaceID, event.Session, entity.WithCID(completeCID), entity.WithCompleteInterval(time.Second*2)); err != nil { return err @@ -1516,6 +1588,18 @@ func (e *exptBaseExec) hasPendingAfterRescan(ctx context.Context, event *entity. event.ExptID, event.ExptRunID, rls[0].ItemID, rls[0].Status) return true, nil } + // Legacy test doubles do not implement the production current-result port. + // Production also rescans Logged so publication can retry before calibration. + if _, current := e.exptItemResultRepo.(repo.IExptCurrentResultRepo); current { + logged := entity.ExptItemResultStateLogged + logs, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, &entity.ExptItemRunLogFilter{ResultState: &logged}, 0, 1, event.SpaceID) + if err != nil { + return false, err + } + if len(logs) > 0 { + return true, nil + } + } return false, nil } @@ -1577,6 +1661,7 @@ func makeEndIdemKey(event *entity.ExptScheduleEvent) string { } type retryItemResetDeps struct { + manager IExptManager evaluationSetItemService EvaluationSetItemService exptItemResultRepo repo.IExptItemResultRepo exptTurnResultRepo repo.IExptTurnResultRepo @@ -1732,7 +1817,7 @@ func resetRetryRunLogsForItems(ctx context.Context, deps retryItemResetDeps, eve if len(itemIDs) == 0 { return nil, nil } - if err := deps.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, itemIDs, map[string]any{ + if err := updateNewRunItemResults(ctx, deps.manager, deps.exptItemResultRepo, event, itemIDs, map[string]any{ "err_msg": nil, "status": int32(entity.ItemRunState_Queueing), "expt_run_id": event.ExptRunID, @@ -1908,7 +1993,7 @@ func (e *ExptRetryAllExec) ExptStart(ctx context.Context, event *entity.ExptSche idIdx++ } - if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDs, func(k int64, v bool) int64 { return k }), map[string]any{ + if err := updateNewRunItemResults(ctx, e.manager, e.exptItemResultRepo, event, maps.ToSlice(itemIDs, func(k int64, v bool) int64 { return k }), map[string]any{ "err_msg": nil, "status": int32(entity.ItemRunState_Queueing), "expt_run_id": event.ExptRunID, @@ -2011,7 +2096,7 @@ func (e *ExptRetryAllExec) ScanEvalItems(ctx context.Context, event *entity.Expt func (e *ExptRetryAllExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { if toSubmit == 0 && incomplete == 0 { - base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService) + base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService, e.resultSvc) pending, err := base.hasPendingAfterRescan(ctx, event) if err != nil { return false, err @@ -2088,6 +2173,7 @@ func (e *ExptRetryAllExec) exptStartMultiSet(ctx context.Context, event *entity. } deps := retryItemResetDeps{ + manager: e.manager, evaluationSetItemService: e.evaluationSetItemService, exptItemResultRepo: e.exptItemResultRepo, exptTurnResultRepo: e.exptTurnResultRepo, @@ -2343,7 +2429,7 @@ func (e *ExptRetryItemsExec) resetEvalItems(ctx context.Context, event *entity.E idIdx++ } - if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDMap, func(k int64, v bool) int64 { return k }), map[string]any{ + if err := updateNewRunItemResults(ctx, e.manager, e.exptItemResultRepo, event, maps.ToSlice(itemIDMap, func(k int64, v bool) int64 { return k }), map[string]any{ "err_msg": nil, "status": int32(entity.ItemRunState_Queueing), "expt_run_id": event.ExptRunID, @@ -2397,6 +2483,7 @@ func (e *ExptRetryItemsExec) resetEvalItemsMultiSet(ctx context.Context, event * } deps := retryItemResetDeps{ + manager: e.manager, evaluationSetItemService: e.evaluationSetItemService, exptItemResultRepo: e.exptItemResultRepo, exptTurnResultRepo: e.exptTurnResultRepo, @@ -2492,7 +2579,7 @@ func (e *ExptRetryItemsExec) ExptEnd(ctx context.Context, event *entity.ExptSche } } - base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService) + base := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService, e.resultSvc) pending, err := base.hasPendingAfterRescan(ctx, event) if err != nil { return false, err @@ -2524,8 +2611,48 @@ func (e *ExptRetryItemsExec) ScheduleStart(ctx context.Context, event *entity.Ex } } event.ExecEvalSetItemIDs = all + hasManualReceipt := false + for _, chunk := range rl.ItemIds { + if chunk.ManualRetryReceipt != nil { + hasManualReceipt = true + break + } + } + if hasManualReceipt && len(absence) == 0 { + return nil + } + if len(absence) > 0 { + if hasManualReceipt { + items, err := e.exptItemResultRepo.BatchGet(contexts.WithCtxWriteDB(ctx), event.SpaceID, event.ExptID, absence) + if err != nil { + return err + } + materialized := make(map[int64]bool) + for _, item := range items { + if item == nil { + return fmt.Errorf("manual retry item missing") + } + retry, err := entity.ParseManualRetryContext(item.Ext) + if err != nil { + return err + } + if retry != nil && retry.RunID == event.ExptRunID && item.ExptRunID == event.ExptRunID && item.SpaceID == event.SpaceID && item.ExptID == event.ExptID { + materialized[item.ItemID] = true + } + } + remaining := make([]int64, 0, len(absence)) + for _, id := range absence { + if !materialized[id] { + remaining = append(remaining, id) + } + } + if len(remaining) == 0 { + return nil + } + absence = remaining + } + } logs.CtxInfo(ctx, "ExptRetryItemsExec.ScheduleStart found absent item_id: %v, expt_id: %v", absence, event.ExptID) - return e.resetEvalItems(ctx, event, expt, absence) } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go index 8b0fb5d77c..bc8d4f9a43 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go @@ -569,22 +569,21 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(2) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, - }, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, - }, int64(1), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{ + {Mode: entity.EvaluationModeSubmit, ItemID: 1, Status: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ItemID: 3, Status: entity.ItemRunState_Success, ResultState: entity.ExptItemResultStateLogged}, + + {Mode: entity.EvaluationModeSubmit, ItemID: 2, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, }, wantIncomplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, }, wantComplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 3, State: entity.ItemRunState_Success}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 3, State: entity.ItemRunState_Success}, }, wantErr: false, assertErr: func(t *testing.T, err error) { @@ -605,7 +604,7 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { expt: mockExpt, }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return(nil, errors.New("scan error")).Times(1) }, wantErr: true, assertErr: func(t *testing.T, err error) { @@ -628,15 +627,14 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(2) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{}, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 1, Status: int32(entity.ItemRunState_Queueing)}, - {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, - }, int64(1), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{ + {Mode: entity.EvaluationModeSubmit, ItemID: 1, Status: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ItemID: 2, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Queueing}, - {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, }, wantIncomplete: []*entity.ExptEvalItem{}, wantComplete: []*entity.ExptEvalItem{}, @@ -1026,22 +1024,21 @@ func TestExptFailRetryExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *exptFailRetryExecFields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(2) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, - }, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, - }, int64(1), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{ + {Mode: entity.EvaluationModeSubmit, ItemID: 1, Status: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ItemID: 3, Status: entity.ItemRunState_Success, ResultState: entity.ExptItemResultStateLogged}, + + {Mode: entity.EvaluationModeSubmit, ItemID: 2, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, }, wantIncomplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, }, wantComplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 3, State: entity.ItemRunState_Success}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 3, State: entity.ItemRunState_Success}, }, wantErr: false, assertErr: func(t *testing.T, err error) { @@ -1062,7 +1059,7 @@ func TestExptFailRetryExec_ScanEvalItems(t *testing.T) { expt: mockExpt, }, prepareMock: func(f *exptFailRetryExecFields, ctrl *gomock.Controller, args args) { - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return(nil, errors.New("scan error")).Times(1) }, wantErr: true, assertErr: func(t *testing.T, err error) { @@ -1637,22 +1634,21 @@ func TestExptAppendExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(2) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, - }, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, - }, int64(1), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{ + {Mode: entity.EvaluationModeSubmit, ItemID: 1, Status: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ItemID: 3, Status: entity.ItemRunState_Success, ResultState: entity.ExptItemResultStateLogged}, + + {Mode: entity.EvaluationModeSubmit, ItemID: 2, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, }, wantIncomplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, }, wantComplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 3, State: entity.ItemRunState_Success}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 3, State: entity.ItemRunState_Success}, }, wantErr: false, assertErr: func(t *testing.T, err error) { @@ -1673,7 +1669,7 @@ func TestExptAppendExec_ScanEvalItems(t *testing.T) { expt: mockExpt, }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return(nil, errors.New("scan error")).Times(1) }, wantErr: true, assertErr: func(t *testing.T, err error) { @@ -1785,8 +1781,7 @@ func TestExptAppendExec_ExptEnd(t *testing.T) { ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 1}, } f.configer.EXPECT().GetExptExecConf(gomock.Any(), args.event.SpaceID).Return(execConf).AnyTimes() - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), args.event.ExptID, args.event.ExptRunID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{}, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), args.event.ExptID, args.event.ExptRunID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{}, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{}, nil).Times(1) f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, gomock.Any(), args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -1810,12 +1805,11 @@ func TestExptAppendExec_ExptEnd(t *testing.T) { f.mutex.EXPECT().Unlock(gomock.Any()).Return(true, nil).Times(1) f.manager.EXPECT().GetDetail(gomock.Any(), args.event.ExptID, args.event.SpaceID, gomock.Any()).Return(mockExptWithEvalSet, nil).Times(1) f.configer.EXPECT().GetExptExecConf(gomock.Any(), args.event.SpaceID).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 2}}).AnyTimes() - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), args.event.ExptID, args.event.ExptRunID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), args.event.ExptID, args.event.ExptRunID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, - }, int64(1), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{ + {ItemID: 1, Status: entity.ItemRunState_Processing}, + + {ItemID: 2, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) }, wantNextTick: true, wantErr: false, @@ -1836,8 +1830,7 @@ func TestExptAppendExec_ExptEnd(t *testing.T) { f.mutex.EXPECT().Unlock(gomock.Any()).Return(true, nil).Times(1) f.manager.EXPECT().GetDetail(gomock.Any(), args.event.ExptID, args.event.SpaceID, gomock.Any()).Return(mockExptWithEvalSet, nil).Times(1) f.configer.EXPECT().GetExptExecConf(gomock.Any(), args.event.SpaceID).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 1}}).AnyTimes() - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), args.event.ExptID, args.event.ExptRunID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{}, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), args.event.ExptID, args.event.ExptRunID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{}, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{}, nil).Times(1) f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(true, nil).Times(1) }, wantNextTick: false, @@ -3793,18 +3786,17 @@ func TestExptRetryAllExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *exptRetryAllExecFields, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(2) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, - }, int64(1), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{ + {Mode: entity.EvaluationModeSubmit, ItemID: 1, Status: entity.ItemRunState_Processing}, + + {Mode: entity.EvaluationModeSubmit, ItemID: 2, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, }, wantIncomplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, }, wantComplete: []*entity.ExptEvalItem{}, wantErr: false, @@ -3824,7 +3816,7 @@ func TestExptRetryAllExec_ScanEvalItems(t *testing.T) { expt: mockExpt, }, prepareMock: func(f *exptRetryAllExecFields, args args) { - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return(nil, errors.New("scan error")).Times(1) }, wantErr: true, assertErr: func(t *testing.T, err error) { @@ -4768,18 +4760,17 @@ func TestExptRetryItemsExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *exptRetryItemsExecFields, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(2) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(0), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, - }, int64(1), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return([]*entity.ExptItemCurrentRequest{ + {Mode: entity.EvaluationModeSubmit, ItemID: 1, Status: entity.ItemRunState_Processing}, + + {Mode: entity.EvaluationModeSubmit, ItemID: 2, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, }, wantIncomplete: []*entity.ExptEvalItem{ - {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + {Mode: entity.EvaluationModeSubmit, ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, }, wantComplete: []*entity.ExptEvalItem{}, wantErr: false, @@ -4799,7 +4790,7 @@ func TestExptRetryItemsExec_ScanEvalItems(t *testing.T) { expt: mockExpt, }, prepareMock: func(f *exptRetryItemsExecFields, args args) { - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + f.exptItemResultRepo.EXPECT().ScanCurrentRequests(gomock.Any(), args.event.SpaceID, args.event.ExptID, args.event.ExptRunID, false).Return(nil, errors.New("scan error")).Times(1) }, wantErr: true, assertErr: func(t *testing.T, err error) { diff --git a/backend/modules/evaluation/domain/service/expt_run_termination.go b/backend/modules/evaluation/domain/service/expt_run_termination.go new file mode 100644 index 0000000000..576d110b2e --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_termination.go @@ -0,0 +1,730 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "fmt" + "sort" + "strconv" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +// IExptCurrentRunTerminator accepts the execution source before a worker leaves +// its item lease. The caller must release that lease before calling this port. +// +//go:generate mockgen -destination=mocks/expt_run_termination.go -package=mocks . IExptCurrentRunTerminator +type IExptCurrentRunTerminator interface { + TerminateCurrentRun(context.Context, int64, int64, int64, *entity.Session, *entity.ExptCurrentItemRunSource, ...entity.CompleteExptOptionFn) (bool, error) +} + +func (e *ExptMangerImpl) terminationRepo() (repo.IExptRunTerminationRepo, error) { + port, ok := e.runFinishRepo.(repo.IExptRunTerminationRepo) + if !ok || e.manualRetryLocker == nil { + return nil, errno.NewMQRetryErr("current run termination provider missing") + } + return port, nil +} + +func (e *ExptMangerImpl) withTerminationRunLock(ctx context.Context, expt, run int64, f func(context.Context) error) error { + if e.manualRetryLocker == nil { + return errno.NewMQRetryErr("termination tick locker missing") + } + locker := e.manualRetryLocker() + if locker == nil { + return errno.NewMQRetryErr("termination tick locker missing") + } + key := fmt.Sprintf("expt_run_exec_lock:%d:%d", expt, run) + locked, lockCtx, cancel, err := locker.LockWithRenew(ctx, key, 5*time.Second, 5*time.Minute) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !locked { + return errno.NewMQRetryErr("termination tick lock busy") + } + defer cancel() + if err := lockCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + return f(lockCtx) +} + +func (e *ExptMangerImpl) beginCurrentRunTermination(ctx context.Context, expt, run, space int64) error { + port, err := e.terminationRepo() + if err != nil { + return err + } + return e.withTerminationRunLock(ctx, expt, run, func(ctx context.Context) error { + result, err := port.TerminateCurrentRun(ctx, &entity.ExptRunTerminationRequest{SpaceID: space, ExptID: expt, RunID: run, Begin: true}) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if result == nil { + return errno.NewMQRetryErr("termination begin result missing") + } + if result.NonCurrent { + return errno.NewMQRetryErr("cancellation run replaced") + } + return nil + }) +} + +func (e *ExptMangerImpl) captureTerminationSources(ctx context.Context, port repo.IExptRunTerminationRepo, space, expt, run int64, ids []int64) ([]*entity.ExptCurrentItemRunSource, error) { + seen := map[int64]bool{} + ordered := []int64{} + for _, id := range ids { + if id <= 0 { + return nil, fmt.Errorf("invalid termination item") + } + if !seen[id] { + ordered = append(ordered, id) + seen[id] = true + } + } + sort.Slice(ordered, func(i, j int) bool { return ordered[i] < ordered[j] }) + sources := make([]*entity.ExptCurrentItemRunSource, 0, len(ordered)) + for _, id := range ordered { + s, err := port.GetTerminationItemSource(ctx, space, expt, run, id) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if s == nil { + return nil, errno.NewMQRetryErr("termination source missing") + } + sources = append(sources, s) + } + return sources, nil +} + +func (e *ExptMangerImpl) commitTerminationItems(ctx context.Context, port repo.IExptRunTerminationRepo, request *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + opCtx, stop := context.WithCancel(ctx) + defer stop() + for _, source := range request.Items { + locker := e.manualRetryLocker() + if locker == nil { + return nil, errno.NewMQRetryErr("termination item locker missing") + } + if err := opCtx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", request.ExptID, source.Current.ItemID) + locked, itemCtx, cancel, err := locker.LockWithRenew(opCtx, key, 5*time.Second, 5*time.Minute) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if !locked { + return nil, errno.NewMQRetryErr("termination item lease busy") + } + defer cancel() + unregister := context.AfterFunc(itemCtx, stop) + defer unregister() + if err := itemCtx.Err(); err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if source.Current.Status == entity.ItemRunState_Success || source.Current.Status == entity.ItemRunState_Fail { + continue + } + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Terminal} + if err := prepareMissingRawTurnLogs(opCtx, e.idgenerator, source, write); err != nil { + return nil, err + } + request.MissingTurnLogs = append(request.MissingTurnLogs, write.MissingTurnLogs...) + } + result, err := port.TerminateCurrentRun(opCtx, request) + if err != nil { + return nil, errno.WrapMQRetryErr(err) + } + if result == nil { + return nil, errno.NewMQRetryErr("termination result missing") + } + return result, nil +} + +func (e *ExptMangerImpl) terminateCurrentItems(ctx context.Context, expt, run, space int64, ids []int64, session *entity.Session) error { + port, err := e.terminationRepo() + if err != nil { + return err + } + // Preserve the request's original source even when admission wins the run lease. + sources, err := e.captureTerminationSources(ctx, port, space, expt, run, ids) + if err != nil { + return err + } + return e.withTerminationRunLock(ctx, expt, run, func(ctx context.Context) error { + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), expt, space) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("termination experiment missing") + } + if current.LatestRunID != run { + return nil + } + if current.ExptType != entity.ExptType_Offline || current.Status != entity.ExptStatus_Processing { + return errno.NewMQRetryErr("row termination requires processing offline experiment") + } + result, err := e.commitTerminationItems(ctx, port, &entity.ExptRunTerminationRequest{SpaceID: space, ExptID: expt, RunID: run, Items: sources}) + if err != nil { + return err + } + if result.NonCurrent { + return nil + } + resourceCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) + defer cancel() + if err := e.releaseTerminationResources(resourceCtx, current, run, sources, false, session); err != nil { + return err + } + return nil + }) +} + +func (e *ExptMangerImpl) TerminateCurrentRun(ctx context.Context, expt, run, space int64, session *entity.Session, expected *entity.ExptCurrentItemRunSource, opts ...entity.CompleteExptOptionFn) (bool, error) { + port, err := e.terminationRepo() + if err != nil { + return false, err + } + opt := &entity.CompleteExptOption{Status: entity.ExptStatus_Terminated} + for _, fn := range opts { + fn(opt) + } + applied := false + err = e.withTerminationRunLock(ctx, expt, run, func(ctx context.Context) error { + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), expt, space) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("termination experiment missing") + } + if current.LatestRunID != run { + return nil + } + if expected != nil { + s, err := port.GetTerminationItemSource(ctx, space, expt, run, expected.Current.ItemID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !sameExecutionIdentity(expected, s) { + return nil + } + } + // Include terminal logs on replay, so residual resource debt remains visible. + var ids []int64 + var cursor int64 + for { + logs, next, err := e.itemResultRepo.ScanItemRunLogs(contexts.WithCtxWriteDB(ctx), expt, run, &entity.ExptItemRunLogFilter{}, cursor, 500, space) + if err != nil { + return errno.WrapMQRetryErr(err) + } + for _, log := range logs { + if log == nil { + return errno.NewMQRetryErr("termination log missing") + } + ids = append(ids, log.ItemID) + } + if next == 0 || len(logs) == 0 { + break + } + if next <= cursor { + return errno.NewMQRetryErr("termination scan cursor invalid") + } + cursor = next + } + sources, err := e.captureTerminationSources(ctx, port, space, expt, run, ids) + if err != nil { + return err + } + result, err := e.commitTerminationItems(ctx, port, &entity.ExptRunTerminationRequest{SpaceID: space, ExptID: expt, RunID: run, Finalize: true, Status: opt.Status, StatusMessage: opt.StatusMessage, NoCompleteItemTurn: opt.NoCompleteItemTurn, ExpectedSource: expected, Items: sources}) + if err != nil { + return err + } + if result.NonCurrent { + if expected == nil { + return errno.NewMQRetryErr("termination source advanced while acquiring item leases") + } + return nil + } + if result.Experiment == nil { + return errno.NewMQRetryErr("terminated experiment missing") + } + applied = true + return e.cleanupTerminatedRun(ctx, &entity.ExptScheduleEvent{SpaceID: space, ExptID: expt, ExptRunID: run, Session: session}, result.Experiment, sources, opt) + }) + return applied, err +} + +func (e *ExptMangerImpl) releaseTerminationResources(ctx context.Context, expt *entity.Experiment, run int64, sources []*entity.ExptCurrentItemRunSource, whole bool, session *entity.Session) error { + seen := map[int64]bool{} + records := []int64{} + for _, source := range sources { + if source == nil { + return fmt.Errorf("termination resource source missing") + } + if !whole && (source.Current.Status == entity.ItemRunState_Success || source.Current.Status == entity.ItemRunState_Fail) { + continue + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if e.centralGuard != nil && entity.IsCentralDispatch(expt.ExptDispatchMode) { + if expt.SchedulerScope == "" { + return fmt.Errorf("termination scope missing") + } + // Each ledger IO has its own budget; a healthy large run must reach + // its tail. The caller's lease/cancellation still bounds the loop. + releaseCtx, cancel := context.WithTimeout(ctx, 8*time.Second) + err := e.centralGuard.Release(releaseCtx, expt.SchedulerScope, run, source.Current.ItemID, source.Current.RequestID, "current run item terminated") + if err == nil { + err = releaseCtx.Err() + } + cancel() + if err != nil { + return errno.WrapMQRetryErr(err) + } + } + if whole || !entity.IsItemRunFinished(source.Current.Status) || source.Current.Status == entity.ItemRunState_Terminal { + frozen := frozenTargetRecordIDs(source) + for _, turn := range source.CanonicalTurns { + if turn != nil && turn.TargetResultID > 0 { + frozen = append(frozen, turn.TargetResultID) + } + } + for _, id := range frozen { + if !seen[id] { + records = append(records, id) + seen[id] = true + } + } + } + } + if e.evalTargetService != nil && len(records) > 0 { + releaseCtx, cancel := context.WithTimeout(ctx, 8*time.Second) + _ = e.evalTargetService.TerminateAsyncRecordsAndDestroySandbox(releaseCtx, resolveLoadSpaceID(expt.SpaceID, expt.TargetSpaceID), records, int32(errno.AsyncEvalTargetTerminatedCode), "async eval target terminated: experiment cancelled", false) + err := releaseCtx.Err() + cancel() + if err != nil { + return errno.WrapMQRetryErr(err) + } + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if whole && e.quotaRepo != nil { + releaseCtx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + return NewQuotaService(e.quotaRepo, e.configer).ReleaseExptRun(releaseCtx, expt.ID, expt.SpaceID, session) + } + return nil +} + +func (e *ExptMangerImpl) cleanupTerminatedRun(ctx context.Context, event *entity.ExptScheduleEvent, finished *entity.Experiment, sources []*entity.ExptCurrentItemRunSource, opt *entity.CompleteExptOption) error { + // Only resources can outlive cancellation. SQL, scores and MQ retain ctx. + resourceCtx, resourceCancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) + defer resourceCancel() + if finished == nil || finished.ID != event.ExptID || finished.LatestRunID != event.ExptRunID { + return errno.NewMQRetryErr("termination cleanup frozen experiment missing") + } + locker := e.manualRetryLocker() + if locker == nil { + return errno.NewMQRetryErr("termination lifecycle locker missing") + } + key := e.makeExptMutexLockKey(event.ExptID) + if _, err := locker.UnlockWithValue(resourceCtx, key, strconv.FormatInt(event.ExptRunID, 10)); err != nil { + return errno.WrapMQRetryErr(err) + } + if err := ctx.Err(); err != nil { + return e.cleanupCancelledTerminationResources(ctx, event, finished, sources) + } + locked, leaseCtx, cancel, err := locker.LockWithRenew(ctx, key, 5*time.Second, 5*time.Minute) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !locked { + return errno.NewMQRetryErr("termination lifecycle owner unavailable") + } + defer cancel() + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(leaseCtx), event.ExptID, event.SpaceID) + if err != nil { + if ctx.Err() != nil { + cancel() + if _, unlockErr := locker.Unlock(key); unlockErr != nil { + return errno.WrapMQRetryErr(unlockErr) + } + return e.cleanupCancelledTerminationResources(ctx, event, finished, sources) + } + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("termination cleanup experiment missing") + } + if current.LatestRunID != event.ExptRunID { + return nil + } + if !entity.IsExptFinished(current.Status) { + return errno.NewMQRetryErr("termination cleanup observed active run") + } + if err := e.releaseTerminationResources(leaseCtx, current, event.ExptRunID, sources, true, event.Session); err != nil { + if ctx.Err() != nil { + cancel() + if _, unlockErr := locker.Unlock(key); unlockErr != nil { + return errno.WrapMQRetryErr(unlockErr) + } + return e.cleanupCancelledTerminationResources(ctx, event, current, sources) + } + return err + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + opCtx, stop := context.WithCancel(ctx) + defer stop() + unregister := context.AfterFunc(leaseCtx, stop) + defer unregister() + if err := opCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if !opt.NoCompleteItemTurn { + if err := e.refreshTerminatedRunFilters(opCtx, event, sources); err != nil { + return errno.WrapMQRetryErr(err) + } + } + if e.idem == nil { + return errno.NewMQRetryErr("termination notification idempotency service missing") + } + notificationKey := fmt.Sprintf("finish-cleanup:%d:%d", event.ExptID, event.ExptRunID) + seen, err := e.idem.Exist(opCtx, notificationKey) + if err != nil { + return err + } + if seen { + return nil + } + if current.ExptTemplateMeta != nil && current.ExptTemplateMeta.ID > 0 && e.templateManager != nil { + if err := e.templateManager.UpdateExptInfo(opCtx, current.ExptTemplateMeta.ID, event.SpaceID, event.ExptID, current.Status, 0, nil); err != nil { + return err + } + } + if !opt.NoAggrCalculate && e.exptAggrResultService != nil { + if err := e.exptAggrResultService.PublishExptAggrResultEvent(opCtx, &entity.AggrCalculateEvent{ExperimentID: event.ExptID, SpaceID: event.SpaceID, CalculateMode: entity.CreateAllFields}, gptr.Of(3*time.Second)); err != nil { + return err + } + } + if err := opCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + if e.publisher != nil { + lifecycle := &entity.ExptLifecycleEvent{SpaceID: event.SpaceID, ExptID: event.ExptID, ExptRunID: &event.ExptRunID, FromStatus: entity.ExptStatus_Processing, ToStatus: current.Status, ExptType: current.ExptType, SourceType: current.SourceType} + lifecycle.IdempotentKey = fmt.Sprintf("expt_%d_%d_%d_%d", event.ExptID, event.ExptRunID, lifecycle.FromStatus, lifecycle.ToStatus) + if err := e.publisher.PublishExptLifecycleEvent(opCtx, lifecycle, gptr.Of(3*time.Second), lifecycle.IdempotentKey); err != nil { + return err + } + } + if current.SourceType == entity.SourceType_Workflow { + e.notifyWorkflowPipelineOnExptFinished(opCtx, current, event.SpaceID, current.Status) + } + if e.mtr != nil { + e.mtr.EmitExptExecResult(event.SpaceID, int64(current.ExptType), int64(current.Status), gptr.Indirect(current.StartAt)) + } + e.emitSandboxAgentExperimentFinished(opCtx, current, current.Status, gptr.Indirect(current.EndAt)) + if err := opCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + return e.idem.Set(opCtx, notificationKey, 3*time.Minute) +} + +func (e *ExptMangerImpl) cleanupCancelledTerminationResources(ctx context.Context, event *entity.ExptScheduleEvent, frozen *entity.Experiment, sources []*entity.ExptCurrentItemRunSource) error { + // Cancellation permits only frozen resource IO. A separate holder prevents + // the cancelled operation's asynchronous unlock from deleting this lease. + resourceCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) + defer stop() + locker := e.manualRetryLocker() + if locker == nil { + return errno.NewMQRetryErr("termination resource locker missing") + } + locked, leaseCtx, cancel, err := locker.LockWithRenew(resourceCtx, e.makeExptMutexLockKey(event.ExptID), 5*time.Second, 8*time.Second) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !locked { + return errno.NewMQRetryErr("termination resource owner unavailable") + } + defer cancel() + if err := e.releaseTerminationResources(leaseCtx, frozen, event.ExptRunID, sources, true, event.Session); err != nil { + return err + } + return errno.WrapMQRetryErr(ctx.Err()) +} + +func (e *ExptMangerImpl) refreshTerminatedRunFilters(ctx context.Context, event *entity.ExptScheduleEvent, sources []*entity.ExptCurrentItemRunSource) error { + port, err := e.terminationRepo() + if err != nil { + return err + } + for _, original := range sources { + if original == nil { + return errno.NewMQRetryErr("termination filter source missing") + } + if original.Current.Status == entity.ItemRunState_Success || original.Current.Status == entity.ItemRunState_Fail { + continue + } + local, ok := e.exptResultService.(*ExptResultServiceImpl) + if !ok { + if e.exptResultService == nil { + return errno.NewMQRetryErr("termination filter service missing") + } + if err := e.exptResultService.UpsertExptTurnResultFilter(ctx, event.SpaceID, event.ExptID, []int64{original.Current.ItemID}); err != nil { + return err + } + continue + } + // The termination transaction has released all item leases. Reacquire + // independently and rebuild the payload from primary canonical rows. + if err := local.withFilterItemLock(ctx, event.ExptID, original.Current.ItemID, func(ctx context.Context) error { + current, err := port.GetTerminationItemSource(ctx, event.SpaceID, event.ExptID, event.ExptRunID, original.Current.ItemID) + if err != nil { + return err + } + if current == nil || current.Current.RequestID != original.Current.RequestID || current.Current.ItemVersionID != original.Current.ItemVersionID || current.Current.Mode != original.Current.Mode || current.Current.Status != entity.ItemRunState_Terminal { + return errno.NewMQRetryErr("termination filter current source changed") + } + filters, err := local.buildCurrentItemFilterLocked(ctx, event.SpaceID, event.ExptID, original.Current.ItemID) + if err != nil { + return err + } + if len(filters) == 0 || local.exptTurnResultFilterRepo == nil { + return errno.NewMQRetryErr("termination filter canonical turns missing") + } + if err := ctx.Err(); err != nil { + return err + } + return local.exptTurnResultFilterRepo.Save(ctx, filters) + }); err != nil { + return err + } + } + return nil +} + +func (e *ExptMangerImpl) cleanupCurrentUnscheduledRun(ctx context.Context, expt, run, space int64, cause error) (err error) { + if run <= 0 { + return fmt.Errorf("invalid cleanup run") + } + // Always release the captured admission value, including read/write failures. + // Compare-and-delete cannot release a replacement run's lifecycle lock. + defer func() { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) + defer cancel() + _, releaseErr := e.mutex.UnlockWithValue(releaseCtx, e.makeExptMutexLockKey(expt), strconv.FormatInt(run, 10)) + err = errors.Join(err, releaseErr) + }() + stateCtx, cancel := context.WithTimeout(ctx, exptRunLogPersistTimeout) + defer cancel() + terminate := false + err = e.withTerminationRunLock(stateCtx, expt, run, func(ctx context.Context) error { + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), expt, space) + if err != nil { + return err + } + if current == nil { + return fmt.Errorf("cleanup experiment missing") + } + if current.LatestRunID != run || entity.IsExptFinished(current.Status) { + return nil + } + if current.ExptType == entity.ExptType_Online { + e.cleanupLegacyUnscheduledRun(ctx, expt, run, space, cause) + return nil + } + if current.Status == entity.ExptStatus_Pending { + port, err := e.terminationRepo() + if err != nil { + return err + } + _, err = port.TerminateCurrentRun(ctx, &entity.ExptRunTerminationRequest{SpaceID: space, ExptID: expt, RunID: run, SetupAbort: true, StatusMessage: cause.Error()}) + return err + } + terminate = true + return nil + }) + if err != nil || !terminate { + return err + } + _, err = e.TerminateCurrentRun(ctx, expt, run, space, &entity.Session{}, nil, entity.WithStatus(entity.ExptStatus_Failed), entity.WithStatusMessage(cause.Error()), entity.NoAggrCalculate()) + return err +} + +func (e *ExptMangerImpl) releaseCurrentSupersededRunQuota(ctx context.Context, expt, space, newRun int64, mode entity.ExptRunMode) error { + if !isRetryRunMode(mode) { + return nil + } + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), expt, space) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("superseded experiment missing") + } + if !entity.IsCentralDispatch(current.ExptDispatchMode) || current.LatestRunID == 0 || current.LatestRunID == newRun { + return nil + } + if e.centralGuard == nil || current.SchedulerScope == "" { + return errno.NewMQRetryErr("superseded quota provider or scope missing") + } + port, err := e.terminationRepo() + if err != nil { + return err + } + run := current.LatestRunID + var sources []*entity.ExptCurrentItemRunSource + var cursor int64 + for { + logs, next, err := e.itemResultRepo.ScanItemRunLogs(contexts.WithCtxWriteDB(ctx), expt, run, &entity.ExptItemRunLogFilter{}, cursor, 500, space) + if err != nil { + return errno.WrapMQRetryErr(err) + } + for _, log := range logs { + if log == nil { + return errno.NewMQRetryErr("superseded quota log missing") + } + source, err := port.GetTerminationItemSource(ctx, space, expt, run, log.ItemID) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if source == nil { + return errno.NewMQRetryErr("superseded quota source missing") + } + sources = append(sources, source) + } + if next == 0 || len(logs) == 0 { + break + } + if next <= cursor { + return errno.NewMQRetryErr("superseded quota cursor invalid") + } + cursor = next + } + // The new lifecycle holder excludes old run cleanup until migration completes. + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second) + defer cancel() + for _, source := range sources { + if err := e.centralGuard.Release(releaseCtx, current.SchedulerScope, run, source.Current.ItemID, source.Current.RequestID, "superseded run residual reservation"); err != nil { + return errno.WrapMQRetryErr(err) + } + } + return ctx.Err() +} + +// IExptNewRunItemResetter protects only canonical migration; the scheduler owns +// the run tick lease around the surrounding new-run initializer. +// +//go:generate mockgen -destination=mocks/expt_new_run_reset.go -package=mocks . IExptNewRunItemResetter +type IExptNewRunItemResetter interface { + ResetItemsForNewRun(context.Context, int64, int64, int64, []int64, map[string]any) error +} + +func (e *ExptMangerImpl) ResetItemsForNewRun(ctx context.Context, space, expt, newRun int64, ids []int64, fields map[string]any) error { + if e.runFinishRepo == nil { + return e.itemResultRepo.UpdateItemsResult(ctx, space, expt, ids, fields) + } + if err := ctx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + current, err := e.exptRepo.GetByID(contexts.WithCtxWriteDB(ctx), expt, space) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if current == nil { + return errno.NewMQRetryErr("new-run reset experiment missing") + } + if current.LatestRunID != newRun { + return errno.NewMQRetryErr("new-run reset replaced") + } + selected := map[int64]bool{} + ordered := []int64{} + for _, id := range ids { + if id <= 0 { + return fmt.Errorf("invalid new-run reset item") + } + if !selected[id] { + ordered = append(ordered, id) + selected[id] = true + } + } + sort.Slice(ordered, func(i, j int) bool { return ordered[i] < ordered[j] }) + for _, id := range ordered { + err := func() error { + locker := e.manualRetryLocker() + if locker == nil { + return errno.NewMQRetryErr("new-run reset item locker missing") + } + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", expt, id) + locked, itemCtx, cancel, err := locker.LockWithRenew(ctx, key, 5*time.Second, time.Minute) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if !locked { + return errno.NewMQRetryErr("new-run reset item lock busy") + } + defer cancel() + if err := itemCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + items, err := e.itemResultRepo.BatchGet(contexts.WithCtxWriteDB(itemCtx), space, expt, []int64{id}) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if len(items) != 1 || items[0] == nil { + return errno.NewMQRetryErr("new-run reset canonical item missing") + } + item := items[0] + if item.ExptRunID != newRun && entity.IsCentralDispatch(current.ExptDispatchMode) { + if e.centralGuard == nil || current.SchedulerScope == "" { + return errno.NewMQRetryErr("new-run reset quota provider missing") + } + previous, err := entity.ParseManualRetryContext(item.Ext) + if err != nil { + return errno.WrapMQRetryErr(err) + } + token := "" + if previous != nil { + if previous.RunID != item.ExptRunID { + return errno.NewMQRetryErr("new-run reset token run mismatch") + } + token = previous.RequestID + } + result, err := e.centralGuard.ReleaseTerminalReservation(itemCtx, current.SchedulerScope, item.ExptRunID, id, token) + if err != nil { + return errno.WrapMQRetryErr(err) + } + if result != component.TerminalReservationReleased && result != component.TerminalReservationAbsent { + return errno.NewMQRetryErr("new-run reset reservation source mismatch") + } + } + if err := itemCtx.Err(); err != nil { + return errno.WrapMQRetryErr(err) + } + return e.itemResultRepo.UpdateItemsResult(itemCtx, space, expt, []int64{id}, fields) + }() + if err != nil { + return err + } + } + return nil +} + +func updateNewRunItemResults(ctx context.Context, manager IExptManager, items repo.IExptItemResultRepo, event *entity.ExptScheduleEvent, ids []int64, fields map[string]any) error { + if reset, ok := manager.(IExptNewRunItemResetter); ok { + return reset.ResetItemsForNewRun(ctx, event.SpaceID, event.ExptID, event.ExptRunID, ids, fields) + } + return items.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, ids, fields) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_termination_unit_test.go b/backend/modules/evaluation/domain/service/expt_run_termination_unit_test.go new file mode 100644 index 0000000000..3379ff87b9 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_run_termination_unit_test.go @@ -0,0 +1,1913 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/google/uuid" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + rpcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// termDualRepo exposes one object that serves both the finish and the +// termination ports, matching the production wiring contract. +type termDualRepo struct { + *repomocks.MockIExptRunFinishRepo + *repomocks.MockIExptRunTerminationRepo +} + +func termUnitSource(itemID int64, status entity.ItemRunState) *entity.ExptCurrentItemRunSource { + return &entity.ExptCurrentItemRunSource{ + Current: entity.ExptItemCurrentRequest{ + SpaceID: 3, ExptID: 1, RunID: 2, ItemID: itemID, ItemVersionID: 7, + RequestID: "req", Mode: entity.EvaluationModeFailRetry, Status: status, + }, + Turns: map[int64]*entity.ExptTurnResultRunLog{}, + } +} + +type termUnitDeps struct { + ctrl *gomock.Controller + finish *repomocks.MockIExptRunFinishRepo + termination *repomocks.MockIExptRunTerminationRepo + expts *repomocks.MockIExperimentRepo + items *repomocks.MockIExptItemResultRepo + locker *lockmocks.MockILocker + guard *componentmocks.MockICentralReservationGuard + targets *svcmocks.MockIEvalTargetService +} + +func newTermUnitDeps(t *testing.T) *termUnitDeps { + t.Helper() + ctrl := gomock.NewController(t) + return &termUnitDeps{ + ctrl: ctrl, + finish: repomocks.NewMockIExptRunFinishRepo(ctrl), + termination: repomocks.NewMockIExptRunTerminationRepo(ctrl), + expts: repomocks.NewMockIExperimentRepo(ctrl), + items: repomocks.NewMockIExptItemResultRepo(ctrl), + locker: lockmocks.NewMockILocker(ctrl), + guard: componentmocks.NewMockICentralReservationGuard(ctrl), + targets: svcmocks.NewMockIEvalTargetService(ctrl), + } +} + +func (d *termUnitDeps) manager() *ExptMangerImpl { + m := &ExptMangerImpl{ + runFinishRepo: &termDualRepo{MockIExptRunFinishRepo: d.finish, MockIExptRunTerminationRepo: d.termination}, + manualRetryLocker: func() lock.ILocker { return d.locker }, + exptRepo: d.expts, + itemResultRepo: d.items, + centralGuard: d.guard, + evalTargetService: d.targets, + } + return m +} + +func (d *termUnitDeps) expectLock(locked bool, lockCtx context.Context) { + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ any, _ any) (bool, context.Context, func(), error) { + return locked, lockCtx, func() {}, nil + }).AnyTimes() +} + +func TestTerminationRepoResolution(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + port, err := m.terminationRepo() + require.NoError(t, err) + require.NotNil(t, port) + + noPort := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return d.locker }} + _, err = noPort.terminationRepo() + require.True(t, errno.NeedMQRetry(err)) + + noLocker := &ExptMangerImpl{runFinishRepo: &termDualRepo{MockIExptRunTerminationRepo: d.termination}} + _, err = noLocker.terminationRepo() + require.True(t, errno.NeedMQRetry(err)) +} + +func TestWithTerminationRunLockBranches(t *testing.T) { + t.Run("missing_factory", func(t *testing.T) { + m := &ExptMangerImpl{} + require.True(t, errno.NeedMQRetry(m.withTerminationRunLock(context.Background(), 1, 2, func(context.Context) error { return nil }))) + }) + t.Run("nil_locker", func(t *testing.T) { + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return nil }} + require.True(t, errno.NeedMQRetry(m.withTerminationRunLock(context.Background(), 1, 2, func(context.Context) error { return nil }))) + }) + t.Run("lock_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_run_exec_lock:1:2", gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("redis down")) + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(m.withTerminationRunLock(context.Background(), 1, 2, func(context.Context) error { return nil }))) + }) + t.Run("busy", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(m.withTerminationRunLock(context.Background(), 1, 2, func(context.Context) error { return nil }))) + }) + t.Run("lock_ctx_cancelled", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, cancelled, func() {}, nil) + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return locker }} + require.True(t, errno.NeedMQRetry(m.withTerminationRunLock(context.Background(), 1, 2, func(context.Context) error { return nil }))) + }) + t.Run("success", func(t *testing.T) { + ctrl := gomock.NewController(t) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return locker }} + called := false + require.NoError(t, m.withTerminationRunLock(context.Background(), 1, 2, func(context.Context) error { called = true; return nil })) + require.True(t, called) + }) +} + +func TestBeginCurrentRunTermination(t *testing.T) { + t.Run("port_error", func(t *testing.T) { + m := &ExptMangerImpl{} + require.Error(t, m.beginCurrentRunTermination(context.Background(), 1, 2, 3)) + }) + t.Run("terminate_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(nil, errors.New("db down")) + require.True(t, errno.NeedMQRetry(m.beginCurrentRunTermination(context.Background(), 1, 2, 3))) + }) + t.Run("nil_result", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.beginCurrentRunTermination(context.Background(), 1, 2, 3))) + }) + t.Run("non_current", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{NonCurrent: true}, nil) + require.True(t, errno.NeedMQRetry(m.beginCurrentRunTermination(context.Background(), 1, 2, 3))) + }) + t.Run("success", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{Applied: true}, nil) + require.NoError(t, m.beginCurrentRunTermination(context.Background(), 1, 2, 3)) + }) +} + +func TestCaptureTerminationSources(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + + t.Run("invalid_id", func(t *testing.T) { + _, err := m.captureTerminationSources(context.Background(), d.termination, 3, 1, 2, []int64{0}) + require.Error(t, err) + }) + t.Run("source_error", func(t *testing.T) { + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, errors.New("read failed")) + _, err := m.captureTerminationSources(context.Background(), d.termination, 3, 1, 2, []int64{9}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("missing_source", func(t *testing.T) { + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(nil, nil) + _, err := m.captureTerminationSources(context.Background(), d.termination, 3, 1, 2, []int64{9}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("dedupe_and_sort", func(t *testing.T) { + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(11)).Return(termUnitSource(11, entity.ItemRunState_Processing), nil) + sources, err := m.captureTerminationSources(context.Background(), d.termination, 3, 1, 2, []int64{11, 9, 11}) + require.NoError(t, err) + require.Len(t, sources, 2) + require.Equal(t, int64(9), sources[0].Current.ItemID) + require.Equal(t, int64(11), sources[1].Current.ItemID) + }) +} + +func TestCommitTerminationItems(t *testing.T) { + t.Run("terminal_items_skip_raw_logs", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + require.Empty(t, req.MissingTurnLogs) + return &entity.ExptRunTerminationResult{Applied: true}, nil + }) + result, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{ + SpaceID: 3, ExptID: 1, RunID: 2, + Items: []*entity.ExptCurrentItemRunSource{ + termUnitSource(9, entity.ItemRunState_Success), + termUnitSource(10, entity.ItemRunState_Fail), + }, + }) + require.NoError(t, err) + require.True(t, result.Applied) + }) + t.Run("nil_locker", func(t *testing.T) { + d := newTermUnitDeps(t) + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return nil }} + _, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("lock_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock error")) + _, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("busy", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(false, context.Background()) + _, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("cancelled_context", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := m.commitTerminationItems(ctx, d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("terminate_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(nil, errors.New("commit failed")) + _, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("nil_result", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}}) + require.True(t, errno.NeedMQRetry(err)) + }) +} + +func TestTerminateCurrentItems(t *testing.T) { + t.Run("port_missing", func(t *testing.T) { + m := &ExptMangerImpl{} + require.Error(t, m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{})) + }) + t.Run("capture_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + require.Error(t, m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{0}, &entity.Session{})) + }) + t.Run("experiment_read_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, errors.New("read failed")) + require.True(t, errno.NeedMQRetry(m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{}))) + }) + t.Run("experiment_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{}))) + }) + t.Run("non_current_run", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 99, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline}, nil) + require.NoError(t, m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{})) + }) + t.Run("unsupported_experiment", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Online}, nil) + require.True(t, errno.NeedMQRetry(m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{}))) + }) + t.Run("success", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline}, nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{Applied: true}, nil) + require.NoError(t, m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{})) + }) + t.Run("non_current_result", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline}, nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{NonCurrent: true}, nil) + require.NoError(t, m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{})) + }) +} + +func TestTerminateCurrentRunUnit(t *testing.T) { + t.Run("port_missing", func(t *testing.T) { + m := &ExptMangerImpl{} + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.Error(t, err) + }) + t.Run("lock_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("experiment_read_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, errors.New("read")) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("experiment_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, nil) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("non_current_run", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 99}, nil) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.NoError(t, err) + }) + t.Run("expected_source_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, errors.New("read")) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, termUnitSource(9, entity.ItemRunState_Processing)) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("expected_source_changed", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + changed := termUnitSource(9, entity.ItemRunState_Processing) + changed.Current.RequestID = "other" + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(changed, nil) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, termUnitSource(9, entity.ItemRunState_Processing)) + require.False(t, applied) + require.NoError(t, err) + }) + t.Run("scan_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), int64(0), int64(500), int64(3)).Return(nil, int64(0), errors.New("scan")) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("nil_log", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{nil}, int64(0), nil) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("scan_cursor_stalled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(0), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{Applied: true, NonCurrent: true}, nil) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.False(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("cleanup_failure_after_apply", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{Applied: true, Experiment: &entity.Experiment{ID: 99}}, nil) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.True(t, applied) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("non_current_with_expected", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + expected := termUnitSource(9, entity.ItemRunState_Processing) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{NonCurrent: true}, nil) + applied, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, expected) + require.False(t, applied) + require.NoError(t, err) + }) +} + +func TestReleaseTerminationResourcesUnit(t *testing.T) { + t.Run("nil_source", func(t *testing.T) { + m := &ExptMangerImpl{} + require.Error(t, m.releaseTerminationResources(context.Background(), &entity.Experiment{}, 2, []*entity.ExptCurrentItemRunSource{nil}, false, &entity.Session{})) + }) + t.Run("finished_skipped", func(t *testing.T) { + m := &ExptMangerImpl{} + require.NoError(t, m.releaseTerminationResources(context.Background(), &entity.Experiment{}, 2, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Success)}, false, &entity.Session{})) + }) + t.Run("cancelled", func(t *testing.T) { + m := &ExptMangerImpl{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, errno.NeedMQRetry(m.releaseTerminationResources(ctx, &entity.Experiment{}, 2, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}, false, &entity.Session{}))) + }) + t.Run("central_scope_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + err := m.releaseTerminationResources(context.Background(), &entity.Experiment{ExptDispatchMode: entity.ExptDispatchModeEnforce}, 2, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}, false, &entity.Session{}) + require.Error(t, err) + }) + t.Run("central_release_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.guard.EXPECT().Release(gomock.Any(), "scope", int64(2), int64(9), "req", gomock.Any()).Return(errors.New("redis")) + err := m.releaseTerminationResources(context.Background(), &entity.Experiment{ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, 2, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}, false, &entity.Session{}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_records_released", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + source := termUnitSource(9, entity.ItemRunState_Processing) + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 71, TurnID: 0, TargetResultID: 88}} + d.targets.EXPECT().TerminateAsyncRecordsAndDestroySandbox(gomock.Any(), int64(3), []int64{88}, gomock.Any(), gomock.Any(), false).Return(errors.New("rpc failed")) + require.NoError(t, m.releaseTerminationResources(context.Background(), &entity.Experiment{ID: 1, SpaceID: 3}, 2, []*entity.ExptCurrentItemRunSource{source}, false, &entity.Session{})) + }) + t.Run("whole_releases_quota", func(t *testing.T) { + d := newTermUnitDeps(t) + quota := repomocks.NewMockQuotaRepo(d.ctrl) + quota.EXPECT().CreateOrUpdate(gomock.Any(), int64(3), gomock.Any(), gomock.Any()).Return(nil) + m := d.manager() + m.quotaRepo = quota + m.configer = nil + require.NoError(t, m.releaseTerminationResources(context.Background(), &entity.Experiment{ID: 1, SpaceID: 3}, 2, nil, true, &entity.Session{})) + }) + t.Run("whole_quota_error", func(t *testing.T) { + d := newTermUnitDeps(t) + quota := repomocks.NewMockQuotaRepo(d.ctrl) + quota.EXPECT().CreateOrUpdate(gomock.Any(), int64(3), gomock.Any(), gomock.Any()).Return(errors.New("quota")) + m := d.manager() + m.quotaRepo = quota + require.Error(t, m.releaseTerminationResources(context.Background(), &entity.Experiment{ID: 1, SpaceID: 3}, 2, nil, true, &entity.Session{})) + }) +} + +func TestCleanupTerminatedRunUnit(t *testing.T) { + expt := func() *entity.Experiment { + return &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Terminated} + } + event := func() *entity.ExptScheduleEvent { + return &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} + } + + t.Run("frozen_experiment_mismatch", func(t *testing.T) { + m := &ExptMangerImpl{} + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), &entity.Experiment{ID: 9}, nil, &entity.CompleteExptOption{}))) + }) + t.Run("nil_locker", func(t *testing.T) { + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return nil }} + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{}))) + }) + t.Run("unlock_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), "expt_run_mutex_lock:1", "2").Return(false, errors.New("unlock")) + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{}))) + }) + t.Run("cancelled_before_owner", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("owner")) + require.Error(t, m.cleanupTerminatedRun(ctx, event(), expt(), nil, &entity.CompleteExptOption{})) + }) + t.Run("owner_unavailable", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{}))) + }) + t.Run("experiment_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{}))) + }) + t.Run("latest_run_changed", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 99}, nil) + require.NoError(t, m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{})) + }) + t.Run("run_still_active", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + active := expt() + active.Status = entity.ExptStatus_Processing + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(active, nil) + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{}))) + }) + t.Run("resource_release_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt(), nil) + err := m.cleanupTerminatedRun(context.Background(), event(), expt(), []*entity.ExptCurrentItemRunSource{nil}, &entity.CompleteExptOption{}) + require.Error(t, err) + }) + t.Run("idem_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt(), nil) + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true}))) + }) + t.Run("idem_exist_error", func(t *testing.T) { + d := newTermUnitDeps(t) + idem := idemmocks.NewMockIdempotentService(d.ctrl) + idem.EXPECT().Exist(gomock.Any(), "finish-cleanup:1:2").Return(false, errors.New("idem")) + m := d.manager() + m.idem = idem + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt(), nil) + require.Error(t, m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true})) + }) + t.Run("already_notified", func(t *testing.T) { + d := newTermUnitDeps(t) + idem := idemmocks.NewMockIdempotentService(d.ctrl) + idem.EXPECT().Exist(gomock.Any(), "finish-cleanup:1:2").Return(true, nil) + m := d.manager() + m.idem = idem + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt(), nil) + require.NoError(t, m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true})) + }) + t.Run("success", func(t *testing.T) { + d := newTermUnitDeps(t) + idem := idemmocks.NewMockIdempotentService(d.ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) + idem.EXPECT().Set(gomock.Any(), "finish-cleanup:1:2", gomock.Any()).Return(nil) + m := d.manager() + m.idem = idem + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt(), nil) + require.NoError(t, m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true})) + }) +} + +func TestCleanupCancelledTerminationResourcesUnit(t *testing.T) { + event := &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} + frozen := &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Terminated} + t.Run("nil_locker", func(t *testing.T) { + m := &ExptMangerImpl{manualRetryLocker: func() lock.ILocker { return nil }} + require.True(t, errno.NeedMQRetry(m.cleanupCancelledTerminationResources(context.Background(), event, frozen, nil))) + }) + t.Run("lock_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + require.True(t, errno.NeedMQRetry(m.cleanupCancelledTerminationResources(context.Background(), event, frozen, nil))) + }) + t.Run("owner_unavailable", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + require.True(t, errno.NeedMQRetry(m.cleanupCancelledTerminationResources(context.Background(), event, frozen, nil))) + }) + t.Run("release_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + require.Error(t, m.cleanupCancelledTerminationResources(context.Background(), event, frozen, []*entity.ExptCurrentItemRunSource{nil})) + }) + t.Run("cancelled_returns_retry", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + require.True(t, errno.NeedMQRetry(m.cleanupCancelledTerminationResources(ctx, event, frozen, nil))) + }) + t.Run("success", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + // The trailing unwrap always reports a retryable envelope, preserving replay. + require.Error(t, m.cleanupCancelledTerminationResources(context.Background(), event, frozen, nil)) + }) +} + +func TestRefreshTerminatedRunFiltersUnit(t *testing.T) { + event := &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} + t.Run("port_missing", func(t *testing.T) { + m := &ExptMangerImpl{} + require.Error(t, m.refreshTerminatedRunFilters(context.Background(), event, nil)) + }) + t.Run("nil_source", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + require.True(t, errno.NeedMQRetry(m.refreshTerminatedRunFilters(context.Background(), event, []*entity.ExptCurrentItemRunSource{nil}))) + }) + t.Run("finished_skipped", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + require.NoError(t, m.refreshTerminatedRunFilters(context.Background(), event, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Fail)})) + }) + t.Run("legacy_service_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.exptResultService = nil + require.True(t, errno.NeedMQRetry(m.refreshTerminatedRunFilters(context.Background(), event, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}))) + }) + t.Run("legacy_upsert_error", func(t *testing.T) { + d := newTermUnitDeps(t) + svc := svcmocks.NewMockExptResultService(d.ctrl) + svc.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), []int64{9}).Return(errors.New("upsert")) + m := d.manager() + m.exptResultService = svc + require.Error(t, m.refreshTerminatedRunFilters(context.Background(), event, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)})) + }) + t.Run("legacy_upsert_success", func(t *testing.T) { + d := newTermUnitDeps(t) + svc := svcmocks.NewMockExptResultService(d.ctrl) + svc.EXPECT().UpsertExptTurnResultFilter(gomock.Any(), int64(3), int64(1), []int64{9}).Return(nil) + m := d.manager() + m.exptResultService = svc + require.NoError(t, m.refreshTerminatedRunFilters(context.Background(), event, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)})) + }) + t.Run("local_source_changed", func(t *testing.T) { + d := newTermUnitDeps(t) + local := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return d.locker }} + m := d.manager() + m.exptResultService = local + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), int64(3), int64(1), int64(2), int64(9)).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.refreshTerminatedRunFilters(context.Background(), event, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}))) + }) + t.Run("local_read_error", func(t *testing.T) { + d := newTermUnitDeps(t) + local := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return d.locker }} + m := d.manager() + m.exptResultService = local + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("read")) + require.Error(t, m.refreshTerminatedRunFilters(context.Background(), event, []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)})) + }) +} + +func TestCleanupCurrentUnscheduledRunUnit(t *testing.T) { + t.Run("invalid_run", func(t *testing.T) { + m := &ExptMangerImpl{} + require.Error(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 0, 3, errors.New("cause"))) + }) + t.Run("experiment_read_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.mutex = d.locker + d.locker.EXPECT().UnlockWithValue(gomock.Any(), "expt_run_mutex_lock:1", "2").Return(true, nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, errors.New("read")) + require.Error(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) + t.Run("latest_run_finished", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.mutex = d.locker + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Success}, nil) + require.NoError(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) + t.Run("online_legacy_cleanup", func(t *testing.T) { + d := newTermUnitDeps(t) + runLogs := repomocks.NewMockIExptRunLogRepo(d.ctrl) + runLogs.EXPECT().Update(gomock.Any(), int64(1), int64(2), int64(3), gomock.Any()).Return(nil) + m := d.manager() + m.mutex = d.locker + m.runLogRepo = runLogs + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Online}, nil) + require.NoError(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) + t.Run("pending_setup_abort", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.mutex = d.locker + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Pending, ExptType: entity.ExptType_Offline}, nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + require.True(t, req.SetupAbort) + return &entity.ExptRunTerminationResult{}, nil + }) + require.NoError(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) + t.Run("unlock_error_joined", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.mutex = d.locker + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, errors.New("unlock")) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 99}, nil) + require.Error(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) +} + +func TestReleaseCurrentSupersededRunQuotaUnit(t *testing.T) { + t.Run("not_retry_mode", func(t *testing.T) { + m := &ExptMangerImpl{} + require.NoError(t, m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeSubmit)) + }) + t.Run("experiment_read_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, errors.New("read")) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("experiment_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("non_central", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5}, nil) + require.NoError(t, m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry)) + }) + t.Run("scope_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce}, nil) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("scan_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(5), gomock.Any(), int64(0), int64(500), int64(3)).Return(nil, int64(0), errors.New("scan")) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("nil_log", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{nil}, int64(0), nil) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("source_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(0), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), int64(3), int64(1), int64(5), int64(9)).Return(nil, errors.New("read")) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("source_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(0), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("release_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(0), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.guard.EXPECT().Release(gomock.Any(), "scope", int64(5), int64(9), "req", gomock.Any()).Return(errors.New("release")) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("success", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(0), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.guard.EXPECT().Release(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + require.NoError(t, m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry)) + }) +} + +func TestResetItemsForNewRunUnit(t *testing.T) { + t.Run("legacy_repo", func(t *testing.T) { + d := newTermUnitDeps(t) + m := &ExptMangerImpl{itemResultRepo: d.items} + d.items.EXPECT().UpdateItemsResult(gomock.Any(), int64(3), int64(1), []int64{9}, map[string]any{"status": 1}).Return(nil) + require.NoError(t, m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, map[string]any{"status": 1})) + }) + t.Run("cancelled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(ctx, 3, 1, 2, []int64{9}, nil))) + }) + t.Run("experiment_read_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, errors.New("read")) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("experiment_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("run_replaced", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 99}, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("invalid_item", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + require.Error(t, m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{0}, nil)) + }) + t.Run("success_dedupe", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.items.EXPECT().BatchGet(gomock.Any(), int64(3), int64(1), []int64{9}).Return([]*entity.ExptItemResult{{ItemID: 9, ExptRunID: 2}}, nil) + d.items.EXPECT().UpdateItemsResult(gomock.Any(), int64(3), int64(1), []int64{9}, gomock.Any()).Return(nil) + require.NoError(t, m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9, 9}, nil)) + }) + t.Run("lock_busy", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("canonical_item_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("central_guard_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.centralGuard = nil + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{{ItemID: 9, ExptRunID: 1}}, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("central_release_mismatch", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{{ItemID: 9, ExptRunID: 1}}, nil) + d.guard.EXPECT().ReleaseTerminalReservation(gomock.Any(), "scope", int64(1), int64(9), "").Return(component.TerminalReservationMismatch, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("central_release_absent", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{{ItemID: 9, ExptRunID: 1}}, nil) + d.guard.EXPECT().ReleaseTerminalReservation(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(component.TerminalReservationAbsent, nil) + d.items.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + require.NoError(t, m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil)) + }) +} + +func TestUpdateNewRunItemResultsUnit(t *testing.T) { + t.Run("resetter", func(t *testing.T) { + d := newTermUnitDeps(t) + manager := svcmocks.NewMockIExptManager(d.ctrl) + resetter := svcmocks.NewMockIExptNewRunItemResetter(d.ctrl) + resetter.EXPECT().ResetItemsForNewRun(gomock.Any(), int64(3), int64(1), int64(2), []int64{9}, map[string]any{"k": "v"}).Return(nil) + // The concrete manager type is required for the resetter assertion. + _ = manager + combined := &termResetterManager{IExptManager: manager, resetter: resetter} + require.NoError(t, updateNewRunItemResults(context.Background(), combined, d.items, &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []int64{9}, map[string]any{"k": "v"})) + }) + t.Run("plain_repo", func(t *testing.T) { + d := newTermUnitDeps(t) + manager := svcmocks.NewMockIExptManager(d.ctrl) + d.items.EXPECT().UpdateItemsResult(gomock.Any(), int64(3), int64(1), []int64{9}, map[string]any{"k": "v"}).Return(nil) + require.NoError(t, updateNewRunItemResults(context.Background(), manager, d.items, &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []int64{9}, map[string]any{"k": "v"})) + }) +} + +type termResetterManager struct { + IExptManager + resetter IExptNewRunItemResetter +} + +func (m *termResetterManager) ResetItemsForNewRun(ctx context.Context, space, expt, newRun int64, ids []int64, fields map[string]any) error { + return m.resetter.ResetItemsForNewRun(ctx, space, expt, newRun, ids, fields) +} + +// TestTerminationRemainingBranches covers failure and edge branches that the +// real-MySQL fixtures do not reach. +func TestTerminationRemainingBranches(t *testing.T) { + event := func() *entity.ExptScheduleEvent { + return &entity.ExptScheduleEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3} + } + expt := func() *entity.Experiment { + return &entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Terminated} + } + + t.Run("commit_item_ctx_cancelled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, cancelled, func() {}, nil) + _, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("commit_raw_turn_prepare_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + source := termUnitSource(9, entity.ItemRunState_Processing) + source.Current.RequestID = "" + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 1, TurnID: 0}} + _, err := m.commitTerminationItems(context.Background(), d.termination, &entity.ExptRunTerminationRequest{Items: []*entity.ExptCurrentItemRunSource{source}}) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("terminate_current_items_release_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline, ExptDispatchMode: entity.ExptDispatchModeEnforce}, nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{Applied: true}, nil) + require.Error(t, m.terminateCurrentItems(context.Background(), 1, 2, 3, []int64{9}, &entity.Session{})) + }) + t.Run("scan_cursor_advance_then_stall", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), int64(0), int64(500), int64(3)).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(5), nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), int64(5), int64(500), int64(3)).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(5), nil) + _, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("capture_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(0), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(nil, errors.New("read")) + _, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("commit_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(0), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(termUnitSource(9, entity.ItemRunState_Processing), nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(nil, errors.New("commit")) + _, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("terminated_experiment_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.termination.EXPECT().TerminateCurrentRun(gomock.Any(), gomock.Any()).Return(&entity.ExptRunTerminationResult{Applied: true}, nil) + _, err := m.TerminateCurrentRun(context.Background(), 1, 2, 3, &entity.Session{}, nil) + require.True(t, errno.NeedMQRetry(err)) + }) + t.Run("target_release_cancelled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + ctx, cancel := context.WithCancel(context.Background()) + source := termUnitSource(9, entity.ItemRunState_Processing) + source.CanonicalTurns = []*entity.ExptTurnResult{{ID: 71, TurnID: 0, TargetResultID: 88}} + d.targets.EXPECT().TerminateAsyncRecordsAndDestroySandbox(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).DoAndReturn(func(context.Context, int64, []int64, int32, string, bool, ...bool) error { + cancel() + return nil + }) + require.True(t, errno.NeedMQRetry(m.releaseTerminationResources(ctx, &entity.Experiment{ID: 1, SpaceID: 3}, 2, []*entity.ExptCurrentItemRunSource{source}, false, &entity.Session{}))) + }) + t.Run("cleanup_lock_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + require.True(t, errno.NeedMQRetry(m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true}))) + }) + t.Run("cleanup_read_error_while_cancelled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + ctx, cancel := context.WithCancel(context.Background()) + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, ctx, func() {}, nil).Times(2) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).DoAndReturn(func(context.Context, int64, int64) (*entity.Experiment, error) { + cancel() + return nil, errors.New("read") + }) + require.Error(t, m.cleanupTerminatedRun(ctx, event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true})) + }) + t.Run("cleanup_release_error_while_cancelled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + ctx, cancel := context.WithCancel(context.Background()) + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, ctx, func() {}, nil).Times(2) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + current := expt() + current.ExptDispatchMode = entity.ExptDispatchModeEnforce + current.SchedulerScope = "scope" + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(current, nil) + d.guard.EXPECT().Release(gomock.Any(), "scope", int64(2), int64(9), "req", gomock.Any()).DoAndReturn(func(context.Context, string, int64, int64, string, string) error { + cancel() + return errors.New("release") + }) + err := m.cleanupTerminatedRun(ctx, event(), expt(), []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}, &entity.CompleteExptOption{NoCompleteItemTurn: true}) + require.Error(t, err) + }) + t.Run("cleanup_template_error", func(t *testing.T) { + d := newTermUnitDeps(t) + idem := idemmocks.NewMockIdempotentService(d.ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) + template := svcmocks.NewMockIExptTemplateManager(d.ctrl) + template.EXPECT().UpdateExptInfo(gomock.Any(), int64(4), int64(3), int64(1), gomock.Any(), int64(0), gomock.Any()).Return(errors.New("template")) + m := d.manager() + m.idem = idem + m.templateManager = template + current := expt() + current.ExptTemplateMeta = &entity.ExptTemplateMeta{ID: 4} + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(current, nil) + require.Error(t, m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true})) + }) + t.Run("cleanup_aggr_error", func(t *testing.T) { + d := newTermUnitDeps(t) + idem := idemmocks.NewMockIdempotentService(d.ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) + aggr := svcmocks.NewMockExptAggrResultService(d.ctrl) + aggr.EXPECT().PublishExptAggrResultEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("aggr")) + m := d.manager() + m.idem = idem + m.exptAggrResultService = aggr + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(expt(), nil) + require.Error(t, m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true})) + }) + t.Run("cleanup_workflow_notify", func(t *testing.T) { + d := newTermUnitDeps(t) + idem := idemmocks.NewMockIdempotentService(d.ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) + idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + pipeline := rpcmocks.NewMockIPipelineListAdapter(d.ctrl) + pipeline.EXPECT().PipelineNodeFinishCallback(gomock.Any(), int64(1), int64(3)).Return(errors.New("pipeline")) + m := d.manager() + m.idem = idem + m.pipelineListAdapter = pipeline + current := expt() + current.SourceType = entity.SourceType_Workflow + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(current, nil) + require.NoError(t, m.cleanupTerminatedRun(context.Background(), event(), expt(), nil, &entity.CompleteExptOption{NoCompleteItemTurn: true})) + }) + t.Run("refresh_local_source_changed", func(t *testing.T) { + d := newTermUnitDeps(t) + local := &ExptResultServiceImpl{resultPublishLocker: func() lock.ILocker { return d.locker }} + m := d.manager() + m.exptResultService = local + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().Unlock(gomock.Any()).Return(true, nil) + changed := termUnitSource(9, entity.ItemRunState_Terminal) + changed.Current.RequestID = "other" + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(9)).Return(changed, nil) + require.True(t, errno.NeedMQRetry(m.refreshTerminatedRunFilters(context.Background(), event(), []*entity.ExptCurrentItemRunSource{termUnitSource(9, entity.ItemRunState_Processing)}))) + }) + t.Run("unscheduled_experiment_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.mutex = d.locker + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(nil, nil) + require.Error(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) + t.Run("unscheduled_pending_port_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := &ExptMangerImpl{runFinishRepo: d.finish, manualRetryLocker: func() lock.ILocker { return d.locker }, exptRepo: d.expts, mutex: d.locker} + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.expectLock(true, context.Background()) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Pending, ExptType: entity.ExptType_Offline}, nil) + require.Error(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) + t.Run("unscheduled_processing_terminates", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.mutex = d.locker + d.locker.EXPECT().UnlockWithValue(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("second lock")) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline}, nil) + require.Error(t, m.cleanupCurrentUnscheduledRun(context.Background(), 1, 2, 3, errors.New("cause"))) + }) + t.Run("superseded_port_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := &ExptMangerImpl{runFinishRepo: d.finish, manualRetryLocker: func() lock.ILocker { return d.locker }, exptRepo: d.expts, itemResultRepo: d.items} + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + require.Error(t, m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry)) + }) + t.Run("superseded_cursor_stalled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 5, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(5), gomock.Any(), int64(0), int64(500), int64(3)).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(5), nil) + d.termination.EXPECT().GetTerminationItemSource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(termUnitSource(9, entity.ItemRunState_Processing), nil).AnyTimes() + d.items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(5), gomock.Any(), int64(5), int64(500), int64(3)).Return([]*entity.ExptItemResultRunLog{{ItemID: 9}}, int64(5), nil) + require.True(t, errno.NeedMQRetry(m.releaseCurrentSupersededRunQuota(context.Background(), 1, 3, 2, entity.EvaluationModeFailRetry))) + }) + t.Run("reset_locker_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + m.manualRetryLocker = func() lock.ILocker { return nil } + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("reset_lock_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil, errors.New("lock")) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("reset_item_ctx_cancelled", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, cancelled, func() {}, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("reset_batch_get_error", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("batch")) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("reset_manual_context_invalid", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{{ItemID: 9, ExptRunID: 1, Ext: map[string]string{entity.ManualRetryExtKey: "not-json"}}}, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("reset_token_run_mismatch", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, ExptDispatchMode: entity.ExptDispatchModeEnforce, SchedulerScope: "scope"}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + ext := map[string]string{entity.ManualRetryExtKey: `{"run_id":99,"request_id":"req","mode":2}`} + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{{ItemID: 9, ExptRunID: 1, Ext: ext}}, nil) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) + t.Run("reset_ctx_cancelled_before_update", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + var itemCancel func() + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2}, nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ string, _ any, _ any) (bool, context.Context, func(), error) { + lockCtx, cancel := context.WithCancel(context.Background()) + itemCancel = cancel + return true, lockCtx, func() {}, nil + }) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, int64, int64, []int64) ([]*entity.ExptItemResult, error) { + itemCancel() + return []*entity.ExptItemResult{{ItemID: 9, ExptRunID: 2}}, nil + }) + require.True(t, errno.NeedMQRetry(m.ResetItemsForNewRun(context.Background(), 3, 1, 2, []int64{9}, nil))) + }) +} + +func manualRetryUnitExperiment() *entity.Experiment { + return &entity.Experiment{ + ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Processing, ExptType: entity.ExptType_Offline, + EvalSetSourceType: entity.ExptEvalSetSourceType_SingleSet, + EvalSetID: 5, EvalSetVersionID: 6, EvalSetSpaceID: 3, + TargetID: 7, TargetVersionID: 8, TargetSpaceID: 3, + } +} + +func manualRetryUnitItem() *entity.ExptItemResult { + return &entity.ExptItemResult{ItemID: 9, SpaceID: 3, ExptID: 1, ExptRunID: 1, ItemVersionID: 7, Status: entity.ItemRunState_Success} +} + +func manualRetryUnitRequest(requestID string) *entity.RetryInCurrentRunRequest { + return &entity.RetryInCurrentRunRequest{SpaceID: 3, ExptID: 1, ExpectedRunID: 2, RequestID: requestID, Mode: entity.EvaluationModeFailRetry} +} + +func TestManualRetryNormalizeAndValidateUnit(t *testing.T) { + t.Run("normalize_nil", func(t *testing.T) { + _, _, err := normalizeCurrentRunRetry(nil) + require.Error(t, err) + }) + t.Run("normalize_dedupes_and_sorts", func(t *testing.T) { + req := &entity.RetryInCurrentRunRequest{SpaceID: 3, ExptID: 1, ExpectedRunID: 2, RequestID: uuid.NewString(), Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{11, 9, 11}, Session: &entity.Session{UserID: "u"}} + normalized, hash, err := normalizeCurrentRunRetry(req) + require.NoError(t, err) + require.Equal(t, []int64{9, 11}, normalized.ItemIDs) + require.Len(t, hash, 64) + }) + for name, mutate := range map[string]func(*entity.RetryInCurrentRunRequest){ + "zero_identity": func(r *entity.RetryInCurrentRunRequest) { r.SpaceID = 0 }, + "bad_uuid": func(r *entity.RetryInCurrentRunRequest) { r.RequestID = "not-a-uuid" }, + "bad_mode": func(r *entity.RetryInCurrentRunRequest) { r.Mode = entity.EvaluationModeRetryAll }, + "fail_with_items": func(r *entity.RetryInCurrentRunRequest) { + r.Mode = entity.EvaluationModeFailRetry + r.ItemIDs = []int64{9} + }, + "items_empty": func(r *entity.RetryInCurrentRunRequest) { r.Mode = entity.EvaluationModeRetryItems }, + "invalid_item": func(r *entity.RetryInCurrentRunRequest) { + r.Mode = entity.EvaluationModeRetryItems + r.ItemIDs = []int64{0} + }, + "internal_ext": func(r *entity.RetryInCurrentRunRequest) { + r.Ext = map[string]string{entity.ManualRetryExtKey: "x"} + }, + } { + t.Run("validate_"+name, func(t *testing.T) { + req := &entity.RetryInCurrentRunRequest{SpaceID: 3, ExptID: 1, ExpectedRunID: 2, RequestID: uuid.NewString(), Mode: entity.EvaluationModeFailRetry} + mutate(req) + require.Error(t, validateCurrentRunRetry(req)) + }) + } + t.Run("validate_ok", func(t *testing.T) { + require.NoError(t, validateCurrentRunRetry(manualRetryUnitRequest(uuid.NewString()))) + }) + t.Run("experiment_validation", func(t *testing.T) { + req := manualRetryUnitRequest(uuid.NewString()) + require.Error(t, validateCurrentRetryExperiment(nil, req)) + require.Error(t, validateCurrentRetryExperiment(&entity.Experiment{ID: 9, SpaceID: 3, LatestRunID: 2}, req)) + require.Error(t, validateCurrentRetryExperiment(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, Status: entity.ExptStatus_Success, ExptType: entity.ExptType_Offline}, req)) + require.NoError(t, validateCurrentRetryExperiment(manualRetryUnitExperiment(), req)) + }) + t.Run("wake", func(t *testing.T) { + m := &ExptMangerImpl{} + require.Error(t, m.wakeCurrentRetryRun(context.Background(), &entity.Experiment{ExptDispatchMode: entity.ExptDispatchModeEnforce}, nil)) + require.Error(t, m.wakeCurrentRetryRun(context.Background(), &entity.Experiment{}, nil)) + }) +} + +func TestRetryInCurrentRunManagerUnit(t *testing.T) { + newDeps := func(t *testing.T) (*termUnitDeps, *repomocks.MockIExptManualRetryRepo, *componentmocks.MockIConfiger, *repomocks.MockIExptRunLogRepo) { + d := newTermUnitDeps(t) + return d, repomocks.NewMockIExptManualRetryRepo(d.ctrl), componentmocks.NewMockIConfiger(d.ctrl), repomocks.NewMockIExptRunLogRepo(d.ctrl) + } + enabled := func(config *componentmocks.MockIConfiger) { + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}).AnyTimes() + } + + t.Run("repo_missing", func(t *testing.T) { + d := newTermUnitDeps(t) + m := d.manager() + _, err := m.RetryInCurrentRun(context.Background(), manualRetryUnitRequest(uuid.NewString())) + require.Error(t, err) + }) + t.Run("existing_receipt", func(t *testing.T) { + d, retryRepo, _, _ := newDeps(t) + receipt := &entity.ManualRetryReceipt{RequestID: "r", AcceptedCount: 1} + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(receipt, nil) + m := d.manager() + m.manualRetryRepo = retryRepo + got, err := m.RetryInCurrentRun(context.Background(), manualRetryUnitRequest(uuid.NewString())) + require.NoError(t, err) + require.Same(t, receipt, got) + }) + t.Run("disabled", func(t *testing.T) { + d, retryRepo, config, _ := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(false)}) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + _, err := m.RetryInCurrentRun(context.Background(), manualRetryUnitRequest(uuid.NewString())) + require.Error(t, err) + }) + t.Run("lock_busy", func(t *testing.T) { + d, retryRepo, config, _ := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil) + d.items.EXPECT().ScanItemResults(gomock.Any(), int64(1), int64(0), int64(500), []int32{int32(entity.ItemRunState_Fail)}, int64(3)).Return(nil, int64(0), nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + _, err := m.RetryInCurrentRun(context.Background(), manualRetryUnitRequest(uuid.NewString())) + require.Error(t, err) + }) + t.Run("run_finished", func(t *testing.T) { + d, retryRepo, config, runs := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil).Times(2) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + runs.EXPECT().Get(gomock.Any(), int64(1), int64(2)).Return(&entity.ExptRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, Status: int64(entity.ExptStatus_Success)}, nil) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + m.runLogRepo = runs + _, err := m.RetryInCurrentRun(context.Background(), manualRetryUnitRequest(uuid.NewString())) + require.Error(t, err) + }) + t.Run("prepare_error", func(t *testing.T) { + d, retryRepo, config, _ := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan")) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + _, err := m.RetryInCurrentRun(context.Background(), manualRetryUnitRequest(uuid.NewString())) + require.Error(t, err) + }) + t.Run("happy_path_empty_plan", func(t *testing.T) { + d, retryRepo, config, runs := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil).Times(2) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + runs.EXPECT().Get(gomock.Any(), int64(1), int64(2)).Return(&entity.ExptRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, Status: int64(entity.ExptStatus_Processing)}, nil) + retryRepo.EXPECT().PreflightTerminalItems(gomock.Any(), gomock.Any()).Return(&entity.ManualRetryPreparation{}, nil) + receipt := &entity.ManualRetryReceipt{RequestID: "r", AcceptedCount: 1} + retryRepo.EXPECT().RequeueTerminalItems(gomock.Any(), gomock.Any()).Return(receipt, nil) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + m.runLogRepo = runs + got, err := m.RetryInCurrentRun(context.Background(), manualRetryUnitRequest(uuid.NewString())) + require.NoError(t, err) + require.Same(t, receipt, got) + }) +} + +func TestPrepareTerminalCandidatesUnit(t *testing.T) { + newDateDeps := func(t *testing.T) (*termUnitDeps, *repomocks.MockIExptTurnResultRepo, *repomocks.MockIExptItemRefRepo) { + d := newTermUnitDeps(t) + return d, repomocks.NewMockIExptTurnResultRepo(d.ctrl), repomocks.NewMockIExptItemRefRepo(d.ctrl) + } + expt := func() *entity.Experiment { return manualRetryUnitExperiment() } + + t.Run("fail_retry_with_items", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeFailRetry, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("fail_retry_scan_error", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan")) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeFailRetry, nil, nil) + require.Error(t, err) + }) + t.Run("fail_retry_cursor_stalled", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), int64(0), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, int64(5), nil) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), int64(5), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, int64(5), nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeFailRetry, nil, nil) + require.Error(t, err) + }) + t.Run("retry_items_empty", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, nil, nil) + require.Error(t, err) + }) + t.Run("retry_items_invalid_id", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{0}, nil) + require.Error(t, err) + }) + t.Run("retry_items_batch_error", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().BatchGet(gomock.Any(), int64(3), int64(1), []int64{9}).Return(nil, errors.New("batch")) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("retry_items_not_found", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().BatchGet(gomock.Any(), int64(3), int64(1), []int64{9}).Return(nil, nil) + plan, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.NoError(t, err) + require.Len(t, plan.RejectedItems, 1) + require.Equal(t, "not_found", plan.RejectedItems[0].Reason) + }) + t.Run("unsupported_mode", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryAll, nil, nil) + require.Error(t, err) + }) + t.Run("nil_item", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{nil}, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("invalid_item_source", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + item := manualRetryUnitItem() + item.SpaceID = 99 + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{item}, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("manual_context_parse_error", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + item := manualRetryUnitItem() + item.Ext = map[string]string{entity.ManualRetryExtKey: "{bad"} + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{item}, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("manual_context_run_mismatch", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + item := manualRetryUnitItem() + item.Ext = map[string]string{entity.ManualRetryExtKey: `{"run_id":99,"request_id":"req","mode":2}`} + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{item}, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("item_log_read_error", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), int64(1), int64(2), []int64{9}, int64(3)).Return(nil, errors.New("logs")) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("duplicate_item_log", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + log := &entity.ExptItemResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateResulted)} + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{log, log}, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("current_log_not_ready", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + log := &entity.ExptItemResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateDefault)} + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{log}, nil) + plan, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.NoError(t, err) + require.Len(t, plan.RejectedItems, 1) + require.Equal(t, "result_not_ready", plan.RejectedItems[0].Reason) + }) + t.Run("current_log_missing", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + item := manualRetryUnitItem() + item.ExptRunID = 2 + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{item}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("single_set_source_missing", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + badExpt := expt() + badExpt.EvalSetID = 0 + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), badExpt, 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("invalid_source_type", func(t *testing.T) { + d, _, _ := newDateDeps(t) + m := d.manager() + badExpt := expt() + badExpt.EvalSetSourceType = entity.ExptEvalSetSourceType(99) + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), badExpt, 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("multiset_ref_error", func(t *testing.T) { + d, _, refs := newDateDeps(t) + m := d.manager() + m.itemRefRepo = refs + badExpt := expt() + badExpt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + refs.EXPECT().GetByExptIDAndItemID(gomock.Any(), int64(3), int64(1), int64(9)).Return(nil, errors.New("ref")) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), badExpt, 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("multiset_ref_mismatch", func(t *testing.T) { + d, _, refs := newDateDeps(t) + m := d.manager() + m.itemRefRepo = refs + badExpt := expt() + badExpt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + refs.EXPECT().GetByExptIDAndItemID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptItemRef{SpaceID: 3, ExptID: 1, ItemID: 9, ItemVersionID: 99}, nil) + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), badExpt, 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("success_single_set", func(t *testing.T) { + d, turns, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().BatchGet(gomock.Any(), int64(3), int64(1), []int64{9}).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + turn := &entity.ExptTurnResult{ID: 10, SpaceID: 3, ExptID: 1, ItemID: 9, ExptRunID: 1, ItemVersionID: 7, TurnID: 0, Status: int32(entity.TurnRunState_Success)} + turns.EXPECT().GetItemTurnResults(gomock.Any(), int64(1), int64(9), int64(3)).Return([]*entity.ExptTurnResult{turn}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), int64(9), int64(3)).Return(nil, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(3), []int64{10}).Return(nil, nil) + m.turnResultRepo = turns + plan, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.NoError(t, err) + require.Len(t, plan.Items, 1) + require.NotNil(t, plan.Items[0].SingleSetSource) + require.Len(t, plan.Items[0].Turns, 1) + }) + t.Run("turns_missing", func(t *testing.T) { + d, turns, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + m.turnResultRepo = turns + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) + t.Run("turn_log_read_error", func(t *testing.T) { + d, turns, _ := newDateDeps(t) + m := d.manager() + d.items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{manualRetryUnitItem()}, nil) + d.items.EXPECT().MGetItemRunLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{{ID: 10, SpaceID: 3, ExptID: 1, ItemID: 9, ExptRunID: 1, ItemVersionID: 7, TurnID: 0}}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("logs")) + m.turnResultRepo = turns + _, err := m.prepareTerminalCandidatesAndReusableRefs(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, []int64{9}, nil) + require.Error(t, err) + }) +} + +func TestManualRetryTurnPreparationUnit(t *testing.T) { + newDeps := func(t *testing.T) (*termUnitDeps, *repomocks.MockIExptTurnResultRepo) { + d := newTermUnitDeps(t) + return d, repomocks.NewMockIExptTurnResultRepo(d.ctrl) + } + expt := func() *entity.Experiment { return manualRetryUnitExperiment() } + preparedItem := func() *entity.PreparedRetryItem { + return &entity.PreparedRetryItem{SourceItem: manualRetryUnitItem(), PreviousRunID: 1, ItemVersionID: 7} + } + turn := func() *entity.ExptTurnResult { + return &entity.ExptTurnResult{ID: 10, SpaceID: 3, ExptID: 1, ItemID: 9, ExptRunID: 1, ItemVersionID: 7, TurnID: 0, Status: int32(entity.TurnRunState_Success)} + } + + t.Run("turn_read_error", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + turns.EXPECT().GetItemTurnResults(gomock.Any(), int64(1), int64(9), int64(3)).Return(nil, errors.New("read")) + require.Error(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, preparedItem(), nil)) + }) + t.Run("invalid_log", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{turn()}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 99, TurnID: 0}}, nil) + require.Error(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, preparedItem(), nil)) + }) + t.Run("invalid_turn", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + bad := turn() + bad.ExptRunID = 99 + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{bad}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + require.Error(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, preparedItem(), nil)) + }) + t.Run("refs_error", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{turn()}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(3), []int64{10}).Return(nil, errors.New("refs")) + require.Error(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, preparedItem(), nil)) + }) + t.Run("invalid_ref", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{turn()}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnEvaluatorResultRef{{SpaceID: 99}}, nil) + require.Error(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, preparedItem(), nil)) + }) + t.Run("missing_current_log", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + item := preparedItem() + item.CurrentItemLog = &entity.ExptItemResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, Status: int32(entity.ItemRunState_Success)} + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{turn()}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + require.Error(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, item, nil)) + }) + t.Run("turn_log_version_mismatch", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{turn()}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 99, TurnID: 0}}, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + require.Error(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, preparedItem(), nil)) + }) + t.Run("fail_retry_preserves_success", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + item := preparedItem() + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{turn()}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), gomock.Any(), []int64{10}).Return([]*entity.ExptTurnEvaluatorResultRef{{ID: 700, SpaceID: 3, ExptID: 1, ExptTurnResultID: 10, EvaluatorVersionID: 51, EvaluatorResultID: 61, Alias: "a", SourceType: int32(entity.EvaluatorRecordSourceTypeBuiltin)}}, nil) + require.NoError(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeFailRetry, item, nil)) + require.Len(t, item.Turns, 1) + require.True(t, item.Turns[0].PreserveSuccess) + require.Equal(t, []int64{700}, item.Turns[0].KeepRefIDs) + }) + t.Run("sort_source_logs", func(t *testing.T) { + d, turns := newDeps(t) + m := d.manager() + m.turnResultRepo = turns + item := preparedItem() + turns.EXPECT().GetItemTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResult{turn()}, nil) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 9, ItemVersionID: 7, TurnID: 0}}, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + require.NoError(t, m.prepareManualRetryTurns(context.Background(), expt(), 2, entity.EvaluationModeRetryItems, item, nil)) + require.Len(t, item.SourceTurnLogs, 1) + }) +} + +func TestManualRetryResultContainsUnit(t *testing.T) { + inlineRef := &entity.ExptTurnEvaluatorResultRef{SourceType: int32(entity.EvaluatorRecordSourceTypeInline), EvaluatorResultID: 61, InlineKey: "k"} + registeredRef := &entity.ExptTurnEvaluatorResultRef{SourceType: int32(entity.EvaluatorRecordSourceTypeBuiltin), EvaluatorResultID: 61, EvaluatorVersionID: 51, Alias: "a"} + require.False(t, manualRetryResultContains(nil, inlineRef)) + require.True(t, manualRetryResultContains(&entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "k", RecordID: 61}}}, inlineRef)) + require.False(t, manualRetryResultContains(&entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "x", RecordID: 61}}}, inlineRef)) + require.True(t, manualRetryResultContains(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}}, registeredRef)) + require.False(t, manualRetryResultContains(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 52, Alias: "a", RecordID: 61}}}, registeredRef)) + legacy := &entity.ExptTurnEvaluatorResultRef{EvaluatorVersionID: 51, EvaluatorResultID: 61} + require.True(t, manualRetryResultContains(&entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 61}}, legacy)) + require.False(t, manualRetryResultContains(&entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 62}}, legacy)) +} + +func TestManualRetryTargetReaderUnit(t *testing.T) { + t.Run("service_missing", func(t *testing.T) { + r := &manualRetryTargetReader{} + _, err := r.GetRecordByID(context.Background(), 3, 61) + require.Error(t, err) + }) + t.Run("record_error", func(t *testing.T) { + ctrl := gomock.NewController(t) + targets := svcmocks.NewMockIEvalTargetService(ctrl) + targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(61)).Return(nil, errors.New("rpc")) + r := &manualRetryTargetReader{IEvalTargetService: targets} + _, err := r.GetRecordByID(context.Background(), 3, 61) + require.Error(t, err) + }) + t.Run("record_mismatch", func(t *testing.T) { + ctrl := gomock.NewController(t) + targets := svcmocks.NewMockIEvalTargetService(ctrl) + targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(61)).Return(&entity.EvalTargetRecord{ID: 61, SpaceID: 3, TargetVersionID: 99, ExperimentRunID: 2, ItemID: 9, TurnID: 0}, nil) + r := &manualRetryTargetReader{IEvalTargetService: targets, versionID: 8, turn: &entity.ExptTurnResult{ItemID: 9, TurnID: 0}} + _, err := r.GetRecordByID(context.Background(), 3, 61) + require.Error(t, err) + }) + t.Run("success_status_without_source_log", func(t *testing.T) { + ctrl := gomock.NewController(t) + targets := svcmocks.NewMockIEvalTargetService(ctrl) + targets.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(61)).Return(&entity.EvalTargetRecord{ID: 61, SpaceID: 3, TargetVersionID: 8, ExperimentRunID: 2, ItemID: 9, TurnID: 0, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil) + r := &manualRetryTargetReader{IEvalTargetService: targets, versionID: 8, turn: &entity.ExptTurnResult{ItemID: 9, TurnID: 0}, source: &retryEvaluatorSourceLogs{repo: repomocks.NewMockIExptTurnResultRepo(ctrl), logsByRun: map[int64]map[int64]*entity.ExptTurnResultRunLog{2: {}}}} + _, err := r.GetRecordByID(context.Background(), 3, 61) + require.Error(t, err) + }) +} + +func TestRetryInCurrentRunRemainingBranches(t *testing.T) { + newDeps := func(t *testing.T) (*termUnitDeps, *repomocks.MockIExptManualRetryRepo, *componentmocks.MockIConfiger, *repomocks.MockIExptRunLogRepo) { + d := newTermUnitDeps(t) + return d, repomocks.NewMockIExptManualRetryRepo(d.ctrl), componentmocks.NewMockIConfiger(d.ctrl), repomocks.NewMockIExptRunLogRepo(d.ctrl) + } + enabled := func(config *componentmocks.MockIConfiger) { + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}).AnyTimes() + } + req := func() *entity.RetryInCurrentRunRequest { return manualRetryUnitRequest(uuid.NewString()) } + + t.Run("find_receipt_error", func(t *testing.T) { + d, retryRepo, _, _ := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("find")) + m := d.manager() + m.manualRetryRepo = retryRepo + _, err := m.RetryInCurrentRun(context.Background(), req()) + require.Error(t, err) + }) + t.Run("locker_nil", func(t *testing.T) { + d, retryRepo, config, _ := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + m.manualRetryLocker = nil + _, err := m.RetryInCurrentRun(context.Background(), req()) + require.Error(t, err) + }) + t.Run("run_missing", func(t *testing.T) { + d, retryRepo, config, runs := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil).Times(2) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + runs.EXPECT().Get(gomock.Any(), int64(1), int64(2)).Return(nil, nil) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + m.runLogRepo = runs + _, err := m.RetryInCurrentRun(context.Background(), req()) + require.Error(t, err) + }) + t.Run("run_mismatch", func(t *testing.T) { + d, retryRepo, config, runs := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil).Times(2) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + runs.EXPECT().Get(gomock.Any(), int64(1), int64(2)).Return(&entity.ExptRunLog{SpaceID: 99, ExptID: 1, ExptRunID: 2}, nil) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + m.runLogRepo = runs + _, err := m.RetryInCurrentRun(context.Background(), req()) + require.Error(t, err) + }) + t.Run("preflight_error", func(t *testing.T) { + d, retryRepo, config, runs := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil).Times(2) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + runs.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, Status: int64(entity.ExptStatus_Processing)}, nil) + retryRepo.EXPECT().PreflightTerminalItems(gomock.Any(), gomock.Any()).Return(nil, errors.New("preflight")) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + m.runLogRepo = runs + _, err := m.RetryInCurrentRun(context.Background(), req()) + require.Error(t, err) + }) + t.Run("requeue_error", func(t *testing.T) { + d, retryRepo, config, runs := newDeps(t) + retryRepo.EXPECT().FindReceipt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + enabled(config) + d.expts.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(manualRetryUnitExperiment(), nil).Times(2) + d.items.EXPECT().ScanItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) + d.locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + runs.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, Status: int64(entity.ExptStatus_Processing)}, nil) + retryRepo.EXPECT().PreflightTerminalItems(gomock.Any(), gomock.Any()).Return(&entity.ManualRetryPreparation{}, nil) + retryRepo.EXPECT().RequeueTerminalItems(gomock.Any(), gomock.Any()).Return(nil, errors.New("requeue")) + m := d.manager() + m.manualRetryRepo = retryRepo + m.configer = config + m.runLogRepo = runs + _, err := m.RetryInCurrentRun(context.Background(), req()) + require.Error(t, err) + }) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_verification_test.go b/backend/modules/evaluation/domain/service/expt_run_verification_test.go index 47b9e9f835..48f9977297 100644 --- a/backend/modules/evaluation/domain/service/expt_run_verification_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_verification_test.go @@ -92,7 +92,8 @@ func TestDefaultExptTurnEvaluationImpl_callTarget_Verification(t *testing.T) { asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "15", gomock.Any()).DoAndReturn( func(_ context.Context, _ string, saved *entity.EvalAsyncCtx) error { require.Equal(t, "sandbox", saved.Callee) - require.Same(t, etec.Event, saved.Event) + require.Equal(t, etec.Event, saved.Event) + require.NotSame(t, etec.Event, saved.Event) return nil }) } diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_consumer_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_consumer_test.go new file mode 100644 index 0000000000..17b85a007e --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_consumer_test.go @@ -0,0 +1,170 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/bytedance/gg/gptr" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestCurrentRequestLockBusyBrokerRetry(t *testing.T) { + ctrl := gomock.NewController(t) + mutex := lockmocks.NewMockILocker(ctrl) + mutex.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, context.Background(), func() {}, nil) + svc := &ExptItemEventEvalServiceImpl{mutex: mutex} + called := false + err := svc.HandleEventLock(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(context.Background(), &entity.ExptItemEvalEvent{ExptID: 1, EvalSetItemID: 2}) + require.True(t, errno.NeedMQRetry(err), "busy lock must reach broker, never ACK") + require.False(t, called) +} + +func TestCurrentRequestBrokerRetryBypassesBudget(t *testing.T) { + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + svc := &ExptItemEventEvalServiceImpl{configer: config} + want := errno.NewMQRetryErr("source read unavailable") + err := svc.HandleEventErr(func(context.Context, *entity.ExptItemEvalEvent) error { return want })(context.Background(), &entity.ExptItemEvalEvent{}) + require.Same(t, want, err, "broker errors must bypass budget, fail and release") +} + +func TestCurrentRequestManualMissingTurnNoNX(t *testing.T) { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeFailRetry, entity.EvaluationModeRetryItems} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + ctx := &entity.ExptItemEvalCtx{Event: &entity.ExptItemEvalEvent{RequestID: "rq1", ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, ExptRunMode: mode}, EvalSetItem: &entity.EvaluationSetItem{Turns: []*entity.Turn{{ID: 21}, {ID: 22}}}, ExistItemEvalResult: &entity.ExptItemEvalResult{ItemResultRunLog: &entity.ExptItemResultRunLog{}, TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{21: {TurnID: 21, ExptID: 1, ExptRunID: 2, ItemID: 11}}}} + var pre RecordEvalMode + if mode == entity.EvaluationModeFailRetry { + pre = &ExptRecordEvalModeFailRetry{} + } else { + pre = &ExptRecordEvalModeRetryIgnoreResult{} + } + err := pre.PreEval(context.Background(), ctx) + require.True(t, errno.NeedMQRetry(err), "missing frozen turn must not repair or ACK") + }) + } +} + +func TestCurrentRequestConsumerSourceGuard(t *testing.T) { + for _, kind := range []string{"stale", "empty_on_manual", "terminal", "read_error", "queueing", "processing"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + src := &entity.ExptItemCurrentRequest{RunID: 2, ItemID: 11, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry, Status: entity.ItemRunState_Processing, RetryTimes: 3} + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, RequestID: "rq1", ExptRunMode: entity.EvaluationModeFailRetry} + readErr := error(nil) + wantCalled := false + switch kind { + case "stale": + event.RequestID = "rq0" + case "empty_on_manual": + event.RequestID = "" + case "terminal": + src.Status = entity.ItemRunState_Success + case "read_error": + readErr = errors.New("DB unavailable") + case "queueing": + src.Status = entity.ItemRunState_Queueing + wantCalled = true + items.EXPECT().MarkProcessingIfCurrent(gomock.Any(), int64(3), int64(1), int64(2), int64(11), "rq1").Return(true, nil) + case "processing": + wantCalled = true + } + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(11)).Return(src, readErr) + svc := &ExptItemEventEvalServiceImpl{exptItemResultRepo: items} + called := false + err := svc.HandleCurrentSource(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(context.Background(), event) + require.Equal(t, wantCalled, called) + if readErr != nil { + require.True(t, errno.NeedMQRetry(err)) + } else { + require.NoError(t, err) + } + if wantCalled { + require.Equal(t, 3, event.RetryTimes) + } + }) + } +} + +func TestCurrentRequestDuplicateInFlightNoInvoke(t *testing.T) { + for _, kind := range []string{"target", "registered", "inline", "old_map"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + dataset := svcmocks.NewMockEvaluationSetItemService(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + evaluators := svcmocks.NewMockEvaluatorRecordService(ctrl) + expt := &entity.Experiment{ID: 1, SpaceID: 3, EvalSet: &entity.EvaluationSet{EvaluationSetVersion: &entity.EvaluationSetVersion{ID: 7, EvaluationSetID: 8}}} + item := &entity.EvaluationSetItem{ID: 11, Turns: []*entity.Turn{{ID: 21}}} + log := &entity.ExptTurnResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 11, TurnID: 21, Status: entity.TurnRunState_Processing} + switch kind { + case "target": + log.TargetResultID = 41 + target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(41)).Return(&entity.EvalTargetRecord{ID: 41, Status: gptr.Of(entity.EvalTargetRunStatusAsyncInvoking)}, nil) + case "registered": + log.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + case "inline": + log.EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{RecordID: 61}}} + case "old_map": + log.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 61}} + } + if kind != "target" { + evaluators.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(61), false).Return(&entity.EvaluatorRecord{ID: 61, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil) + } + manager.EXPECT().GetDetail(gomock.Any(), int64(1), int64(3), gomock.Any()).Return(expt, nil) + items.EXPECT().GetItemRunLog(gomock.Any(), int64(1), int64(2), int64(11), int64(3)).Return(&entity.ExptItemResultRunLog{Status: int32(entity.ItemRunState_Processing)}, nil).Times(2) + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), int64(11), int64(3)).Return([]*entity.ExptTurnResultRunLog{log}, nil) + dataset.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return([]*entity.EvaluationSetItem{item}, nil) + svc := &ExptItemEventEvalServiceImpl{manager: manager, exptItemResultRepo: items, exptTurnResultRepo: turns, evaluationSetItemService: dataset, evaTargetService: target, evaluatorRecordService: evaluators} + require.NoError(t, svc.eval(context.Background(), &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, ExptRunMode: entity.EvaluationModeSubmit})) + }) + } +} + +func TestCurrentRequestManualCompletePreEvalReuse(t *testing.T) { + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, RequestID: "rq1"} + log := &entity.ExptTurnResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 11, TurnID: 21, Status: entity.TurnRunState_Success, TargetResultID: 41} + ctx := &entity.ExptItemEvalCtx{Event: event, EvalSetItem: &entity.EvaluationSetItem{Turns: []*entity.Turn{{ID: 21}}}, ExistItemEvalResult: &entity.ExptItemEvalResult{ItemResultRunLog: &entity.ExptItemResultRunLog{}, TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{21: log}}} + require.NoError(t, (&ExptRecordEvalModeFailRetry{}).PreEval(context.Background(), ctx)) + require.Same(t, log, ctx.GetExistTurnResultRunLog(21)) + require.NoError(t, (&ExptRecordEvalModeRetryIgnoreResult{}).PreEval(context.Background(), ctx)) + require.Equal(t, int64(41), log.TargetResultID) +} + +func TestCurrentRequestConsumerCASLostToCompletion(t *testing.T) { + for _, status := range []entity.ItemRunState{entity.ItemRunState_Processing, entity.ItemRunState_Success} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, ExptRunMode: entity.EvaluationModeSubmit} + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(11)).Return(&entity.ExptItemCurrentRequest{RunID: 2, ItemID: 11, Mode: entity.EvaluationModeSubmit, Status: entity.ItemRunState_Queueing}, nil) + items.EXPECT().MarkProcessingIfCurrent(gomock.Any(), int64(3), int64(1), int64(2), int64(11), "").Return(false, nil) + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(11)).Return(&entity.ExptItemCurrentRequest{RunID: 2, ItemID: 11, Mode: entity.EvaluationModeSubmit, Status: status}, nil) + called := false + svc := &ExptItemEventEvalServiceImpl{exptItemResultRepo: items} + err := svc.HandleCurrentSource(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(context.Background(), event) + require.NoError(t, err) + require.Equal(t, status == entity.ItemRunState_Processing, called) + }) + } +} + +func TestCurrentRequestManualUnknownFrozenTurn(t *testing.T) { + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, RequestID: "rq1"} + eiec := &entity.ExptItemEvalCtx{Event: event, EvalSetItem: &entity.EvaluationSetItem{Turns: []*entity.Turn{{ID: 21}}}, ExistItemEvalResult: &entity.ExptItemEvalResult{ItemResultRunLog: &entity.ExptItemResultRunLog{}, TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{21: {SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 11, TurnID: 21, Status: 9}}}} + require.True(t, errno.NeedMQRetry((&ExptRecordEvalModeFailRetry{}).PreEval(context.Background(), eiec))) +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_dispatch_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_dispatch_test.go new file mode 100644 index 0000000000..c06b0ad8a8 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_dispatch_test.go @@ -0,0 +1,180 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestCurrentRequestProducerMQBeforeCAS(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + mutex := lockmocks.NewMockILocker(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit, ItemRetryTimes: 5, Ext: map[string]string{entity.RetryYieldExtKey: "true", "frozen": "keep"}, Session: &entity.Session{UserID: "original", AppID: 4}} + source := &entity.ExptItemCurrentRequest{RunID: 2, ItemID: 11, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry, Status: entity.ItemRunState_Queueing, RetryTimes: 3} + published := false + mutex.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:1:11", gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + mutex.EXPECT().Unlock("expt_item_eval_run_lock:1:11").Return(true, nil) + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(11)).Return(source, nil) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{IntervalSecond: 2}}) + publisher.EXPECT().BatchPublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, got []*entity.ExptItemEvalEvent, interval *time.Duration) error { + require.Len(t, got, 1) + require.Equal(t, "rq1", got[0].RequestID) + require.Equal(t, entity.EvaluationModeFailRetry, got[0].ExptRunMode) + require.Equal(t, 3, got[0].RetryTimes) + require.Equal(t, 5, got[0].MaxRetryTimes) + require.Equal(t, event.Session, got[0].Session) + require.Equal(t, event.Ext, got[0].Ext) + require.Equal(t, 2*time.Second, *interval) + require.NotSame(t, event.Session, got[0].Session) + got[0].Ext["frozen"] = "mutated" + require.Equal(t, "keep", event.Ext["frozen"]) + published = true + return nil + }) + want := errors.New("DB unavailable after MQ") + items.EXPECT().MarkProcessingIfCurrent(gomock.Any(), int64(3), int64(1), int64(2), int64(11), "rq1").DoAndReturn(func(context.Context, int64, int64, int64, int64, string) (bool, error) { + require.True(t, published) + return false, want + }) + svc := &ExptSchedulerImpl{ExptItemResultRepo: items, Mutex: mutex, itemLocker: func() lock.ILocker { return mutex }, Configer: config, Publisher: publisher} + err := svc.handleToSubmits(context.Background(), event, []*entity.ExptEvalItem{{ItemID: 11, State: entity.ItemRunState_Queueing, RequestID: "rq1"}}) + require.ErrorIs(t, err, want) +} + +func TestCurrentRequestScanOneObservation(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 2, MaxItemConcurNum: 2}}).AnyTimes() + items.EXPECT().ScanCurrentRequests(gomock.Any(), int64(3), int64(1), int64(2), false).Return([]*entity.ExptItemCurrentRequest{{ItemID: 10, Status: entity.ItemRunState_Processing}, {ItemID: 11, Status: entity.ItemRunState_Queueing, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry, RetryTimes: 3}, {ItemID: 12, Status: entity.ItemRunState_Queueing, RequestID: "rq2", Mode: entity.EvaluationModeRetryItems}}, nil) + expt := &entity.Experiment{SpaceID: 3, EvalConf: &entity.EvaluationConfiguration{}, EvalSet: &entity.EvaluationSet{EvaluationSetVersion: &entity.EvaluationSetVersion{ID: 7}}} + exec := &exptBaseExec{exptItemResultRepo: items, configer: config} + to, incomplete, _, err := exec.ScanEvalItems(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, expt) + require.NoError(t, err) + require.Len(t, incomplete, 1) + require.Len(t, to, 1) + require.Equal(t, "rq1", to[0].RequestID) + require.Equal(t, int32(3), to[0].RetryTimes) +} + +func TestCurrentRequestProducerInterleavings(t *testing.T) { + for _, kind := range []string{"publish_error", "publish_unknown", "fast_consumer", "busy", "changed_token", "raw", "fail_retry", "retry_items"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + mutex := lockmocks.NewMockILocker(ctrl) + pub := eventmocks.NewMockExptEventPublisher(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + mode := entity.EvaluationModeSubmit + request := "" + if kind == "fail_retry" { + mode = entity.EvaluationModeFailRetry + request = "rq1" + } + if kind == "retry_items" { + mode = entity.EvaluationModeRetryItems + request = "rq2" + } + src := &entity.ExptItemCurrentRequest{RunID: 2, ItemID: 11, Status: entity.ItemRunState_Queueing, Mode: mode, RequestID: request, RetryTimes: 3} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit, ItemRetryTimes: 5} + mutex.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(kind != "busy", context.Background(), func() {}, nil) + wantErr := false + if kind != "busy" { + mutex.EXPECT().Unlock(gomock.Any()).Return(true, nil) + if kind == "changed_token" { + src.RequestID = "new" + } + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(11)).Return(src, nil) + if kind != "changed_token" { + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}) + pub.EXPECT().BatchPublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, events []*entity.ExptItemEvalEvent, _ *time.Duration) error { + require.Equal(t, mode, events[0].ExptRunMode) + require.NotZero(t, events[0].ExptRunMode) + require.Equal(t, request, events[0].RequestID) + require.Equal(t, 3, events[0].RetryTimes) + if kind == "publish_error" || kind == "publish_unknown" { + return errors.New(kind) + } + if kind == "fast_consumer" { + src.Status = entity.ItemRunState_Success + } + return nil + }) + if kind == "publish_error" || kind == "publish_unknown" { + wantErr = true + } else { + items.EXPECT().MarkProcessingIfCurrent(gomock.Any(), int64(3), int64(1), int64(2), int64(11), request).DoAndReturn(func(context.Context, int64, int64, int64, int64, string) (bool, error) { + return src.Status == entity.ItemRunState_Queueing, nil + }) + } + } + } + svc := &ExptSchedulerImpl{ExptItemResultRepo: items, Publisher: pub, Configer: config, itemLocker: func() lock.ILocker { return mutex }} + err := svc.handleToSubmits(context.Background(), event, []*entity.ExptEvalItem{{ItemID: 11, State: entity.ItemRunState_Queueing, RequestID: request}}) + if wantErr { + require.Error(t, err) + require.Equal(t, entity.ItemRunState_Queueing, src.Status) + } else { + require.NoError(t, err) + } + if kind == "fast_consumer" { + require.Equal(t, entity.ItemRunState_Success, src.Status) + } + require.Equal(t, entity.EvaluationModeSubmit, event.ExptRunMode) + }) + } +} + +func TestCurrentRequestProducerSameBatchModes(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + pub := eventmocks.NewMockExptEventPublisher(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + sources := []*entity.ExptItemCurrentRequest{{RunID: 2, ItemID: 10, Status: entity.ItemRunState_Queueing, Mode: entity.EvaluationModeSubmit, RetryTimes: 0}, {RunID: 2, ItemID: 11, Status: entity.ItemRunState_Queueing, Mode: entity.EvaluationModeFailRetry, RequestID: "rq1", RetryTimes: 3}, {RunID: 2, ItemID: 12, Status: entity.ItemRunState_Queueing, Mode: entity.EvaluationModeRetryItems, RequestID: "rq2", RetryTimes: 1}} + locks := make([]lock.ILocker, 0, 3) + candidates := make([]*entity.ExptEvalItem, 0, 3) + for _, source := range sources { + mutex := lockmocks.NewMockILocker(ctrl) + locks = append(locks, mutex) + mutex.EXPECT().LockWithRenew(gomock.Any(), fmt.Sprintf("expt_item_eval_run_lock:1:%d", source.ItemID), gomock.Any(), gomock.Any()).Return(true, context.Background(), func() {}, nil) + mutex.EXPECT().Unlock(gomock.Any()).Return(true, nil) + candidates = append(candidates, &entity.ExptEvalItem{ItemID: source.ItemID, RequestID: source.RequestID, State: source.Status}) + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), source.ItemID).Return(source, nil) + items.EXPECT().MarkProcessingIfCurrent(gomock.Any(), int64(3), int64(1), int64(2), source.ItemID, source.RequestID).Return(true, nil) + } + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}).Times(3) + got := make([]*entity.ExptItemEvalEvent, 0, 3) + pub.EXPECT().BatchPublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, events []*entity.ExptItemEvalEvent, _ *time.Duration) error { + got = append(got, events...) + return nil + }).Times(3) + n := 0 + svc := &ExptSchedulerImpl{ExptItemResultRepo: items, Publisher: pub, Configer: config, itemLocker: func() lock.ILocker { mutex := locks[n]; n++; return mutex }} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit, ItemRetryTimes: 5, Session: &entity.Session{UserID: "frozen"}} + require.NoError(t, svc.handleToSubmits(context.Background(), event, candidates)) + require.Len(t, got, 3) + require.Equal(t, 3, n) + for i, source := range sources { + require.Equal(t, source.Mode, got[i].ExptRunMode) + require.Equal(t, source.RequestID, got[i].RequestID) + require.Equal(t, int(source.RetryTimes), got[i].RetryTimes) + require.Equal(t, 5, got[i].MaxRetryTimes) + } + require.Equal(t, entity.EvaluationModeSubmit, event.ExptRunMode) +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_filter_batch_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_filter_batch_test.go new file mode 100644 index 0000000000..04b901fcf7 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_filter_batch_test.go @@ -0,0 +1,413 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/driver/clickhouse" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + infraredis "github.com/coze-dev/coze-loop/backend/infra/redis" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentck "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/ck" + experimentckmodel "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/ck/gorm_gen/model" + mysqlmodel "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + mysqlmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +// The public loop must attempt later items after one item fails. A first-item +// early return leaves real prepared INSERT expectations and later reads unmet. +func TestRunningRetryFilterBatch_UpsertContinuesAfterItemFailure(t *testing.T) { + for _, failure := range []string{"busy", "busy_enumerated", "save", "read", "lease_lost", "none"} { + t.Run(failure, func(t *testing.T) { + f := newFilterBatchFixture(t) + firstErr := errors.New("first item source or insert unavailable") + if failure == "busy" || failure == "busy_enumerated" { + f.holdFirst(t) + } + if failure == "read" || failure == "lease_lost" { + f.beforeRead = func(ctx context.Context, item int64) error { + if item != f.item { + return nil + } + if failure == "read" { + return firstErr + } + f.server.Del(f.key()) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(3 * time.Second): + t.Fatal("real Redis lease loss did not cancel item context") + return nil + } + } + } + sqlMock, _ := f.attachCK(t) + start := 0 + if failure == "busy" || failure == "busy_enumerated" || failure == "read" || failure == "lease_lost" { + start = 1 + } + for i := start; i < 3; i++ { + var saveErr error + if i == 0 && failure == "save" { + saveErr = firstErr + } + f.expectInsert(sqlMock, i, saveErr) + } + ids := f.unorderedIDs() + if failure == "busy_enumerated" { + ids = nil // Scheduler/MQ whole-experiment entry enumerates IDs first. + } + err := f.svc.UpsertExptTurnResultFilter(context.Background(), f.space, f.expt, ids) + if failure == "none" { + require.NoError(t, err, "successful batch must return nil, not a wrapper around nil") + } else { + require.True(t, errno.NeedMQRetry(err)) + if failure == "save" || failure == "read" { + require.ErrorIs(t, err, firstErr) + } + } + require.Equal(t, f.ids[start:], f.reads, "later current payloads must still be read after an item failure") + require.Equal(t, f.keys(), f.locks.attempts(), "sort and deduplicate before independent item lock attempts") + require.NoError(t, sqlMock.ExpectationsWereMet()) + t.Log("REAL_MYSQL_BATCH_SOURCE_OSS_CK_PREPARED_INSERT_SQLMOCK_EXECUTED") + }) + } +} + +func TestRunningRetryFilterBatch_CompareContinuesAfterItemFailure(t *testing.T) { + for _, failure := range []string{"busy", "read", "actual_read", "none"} { + t.Run(failure, func(t *testing.T) { + f := newFilterBatchFixture(t) + firstErr := errors.New("first item compare source unavailable") + if failure == "busy" { + f.holdFirst(t) + } + if failure == "read" { + f.beforeRead = func(_ context.Context, item int64) error { + if item == f.item { + return firstErr + } + return nil + } + } + sqlMock, ckConn := f.attachCK(t) + var actualReads []int64 + query := "SELECT etrf\\.\\* FROM .*expt_turn_result_filter AS etrf FINAL WHERE etrf.space_id=\\? AND etrf.expt_id=\\? AND etrf.created_date=\\? AND etrf.item_id IN \\(\\?\\)" + // The actual reader must remain inside the same item's execution lock. + require.NoError(t, ckConn.Callback().Row().Before("gorm:row").Register("batch_actual_item_lock", func(tx *gorm.DB) { + item := f.ids[len(actualReads)] + if failure == "busy" || failure == "read" { + item++ + } + require.True(t, f.server.Exists(f.itemKey(item))) + actualReads = append(actualReads, item) + })) + start := 0 + if failure == "busy" || failure == "read" { + start = 1 + } + for i := start; i < 3; i++ { + q := sqlMock.ExpectQuery(query).WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), "2026-10-20", filterLockedSQLArg{f.server, f.itemKey(f.ids[i]), fmt.Sprint(f.ids[i])}) + if i == 0 && failure == "actual_read" { + q.WillReturnError(firstErr) + } else { + q.WillReturnRows(sqlmock.NewRows([]string{"space_id", "expt_id", "item_id"})) + } + } + ctrl := gomock.NewController(t) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + f.svc.publisher = publisher + var events []int64 + publisher.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptTurnResultFilterEvent, delay *time.Duration) error { + require.Len(t, event.ItemID, 1) + require.True(t, f.server.Exists(f.itemKey(event.ItemID[0]))) + require.Equal(t, int32(2), *event.RetryTimes) + require.Equal(t, entity.UpsertExptTurnResultFilterTypeCheck, *event.FilterType) + require.Equal(t, 10*time.Second, *delay) + events = append(events, event.ItemID[0]) + return nil + }).AnyTimes() + err := f.svc.CompareExptTurnResultFilters(context.Background(), f.space, f.expt, f.unorderedIDs(), 1) + if failure == "none" { + require.NoError(t, err) + } else { + require.True(t, errno.NeedMQRetry(err)) + if failure != "busy" { + require.ErrorIs(t, err, firstErr) + } + } + wantEvents := f.ids[start:] + if failure == "actual_read" { + wantEvents = f.ids[1:] + } + require.Equal(t, wantEvents, events, "later comparisons must publish missing current projections") + require.Equal(t, f.ids[start:], actualReads) + require.Equal(t, f.ids[start:], f.reads) + require.Equal(t, f.keys(), f.locks.attempts()) + require.NoError(t, sqlMock.ExpectationsWereMet()) + }) + } +} + +func TestRunningRetryFilterBatch_ParentCancelStopsLaterItems(t *testing.T) { + for _, compare := range []bool{false, true} { + t.Run(fmt.Sprint(compare), func(t *testing.T) { + f := newFilterBatchFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f.beforeRead = func(ctx context.Context, item int64) error { + require.Equal(t, f.item, item) + cancel() + return ctx.Err() + } + var err error + if compare { + err = f.svc.CompareExptTurnResultFilters(ctx, f.space, f.expt, f.unorderedIDs(), 1) + } else { + err = f.svc.UpsertExptTurnResultFilter(ctx, f.space, f.expt, f.unorderedIDs()) + } + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, f.keys()[:1], f.locks.attempts(), "cancelled parent may not acquire later item locks") + require.Empty(t, f.reads, "parent cancellation may not start any later source IO") + }) + } +} + +func TestRunningRetryFilterBatch_FirstErrorPreserved(t *testing.T) { + for _, compare := range []bool{false, true} { + t.Run(fmt.Sprint(compare), func(t *testing.T) { + f := newFilterBatchFixture(t) + firstErr, lastErr := errors.New("first item read failed"), errors.New("last item read failed") + f.beforeRead = func(_ context.Context, item int64) error { + if item == f.item { + return firstErr + } + if item == f.item+2 { + return lastErr + } + return nil + } + sqlMock, _ := f.attachCK(t) + var err error + if compare { + sqlMock.ExpectQuery("SELECT etrf\\.\\* FROM .*expt_turn_result_filter AS etrf FINAL").WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), "2026-10-20", filterLockedSQLArg{f.server, f.itemKey(f.item + 1), fmt.Sprint(f.item + 1)}).WillReturnRows(sqlmock.NewRows([]string{"space_id", "expt_id", "item_id"})) + publisher := eventmocks.NewMockExptEventPublisher(gomock.NewController(t)) + f.svc.publisher = publisher + publisher.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptTurnResultFilterEvent, _ *time.Duration) error { + require.Equal(t, []int64{f.item + 1}, event.ItemID) + return nil + }) + err = f.svc.CompareExptTurnResultFilters(context.Background(), f.space, f.expt, f.unorderedIDs(), 1) + } else { + f.expectInsert(sqlMock, 1, nil) + err = f.svc.UpsertExptTurnResultFilter(context.Background(), f.space, f.expt, f.unorderedIDs()) + } + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, firstErr) + require.NotErrorIs(t, err, lastErr) + require.Equal(t, []int64{f.item + 1}, f.reads) + require.Equal(t, f.keys(), f.locks.attempts()) + require.NoError(t, sqlMock.ExpectationsWereMet()) + }) + } +} + +func TestRunningRetryFilterBatch_PreCancelledDoesNotEnumerate(t *testing.T) { + for _, compare := range []bool{false, true} { + t.Run(fmt.Sprint(compare), func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + enumerations, factories := 0, 0 + items.EXPECT().ListItemResultsByExptID(gomock.Any(), int64(1), int64(3), gomock.Any(), false).DoAndReturn(func(context.Context, int64, int64, entity.Page, bool) ([]*entity.ExptItemResult, int64, error) { + enumerations++ + return nil, 0, nil + }).AnyTimes() + svc := ExptResultServiceImpl{ExptItemResultRepo: items, resultPublishLocker: func() lock.ILocker { factories++; return nil }} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var err error + if compare { + err = svc.CompareExptTurnResultFilters(ctx, 3, 1, nil, 1) + } else { + err = svc.UpsertExptTurnResultFilter(ctx, 3, 1, nil) + } + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, enumerations) + require.Zero(t, factories) + }) + } +} + +func TestRunningRetryFilterBatch_CompareMaxRetryKeepsMetrics(t *testing.T) { + f := newFilterBatchFixture(t) + sqlMock, _ := f.attachCK(t) + for _, item := range f.ids { + sqlMock.ExpectQuery("SELECT etrf\\.\\* FROM .*expt_turn_result_filter AS etrf FINAL").WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), "2026-10-20", filterLockedSQLArg{f.server, f.itemKey(item), fmt.Sprint(item)}).WillReturnRows(sqlmock.NewRows([]string{"space_id", "expt_id", "item_id"})) + } + ctrl := gomock.NewController(t) + f.svc.publisher = eventmocks.NewMockExptEventPublisher(ctrl) + metric := metricsmocks.NewMockExptMetric(ctrl) + f.svc.Metric = metric + metric.EXPECT().EmitExptTurnResultFilterQueryLatency(f.space, gomock.Any(), false).Times(3) + metric.EXPECT().EmitExptTurnResultFilterCheck(f.space, false, false, true, true).Times(3) + require.NoError(t, f.svc.CompareExptTurnResultFilters(context.Background(), f.space, f.expt, f.unorderedIDs(), 3)) + require.Equal(t, f.ids, f.reads) + require.NoError(t, sqlMock.ExpectationsWereMet()) +} + +type filterBatchFixture struct { + *filterSyncFixture + ids []int64 + locks *filterBatchRedis + reads []int64 + beforeRead func(context.Context, int64) error +} + +func newFilterBatchFixture(t *testing.T) *filterBatchFixture { + t.Helper() + f := &filterBatchFixture{filterSyncFixture: newFilterSyncFixture(t)} + f.ids = []int64{f.item, f.item + 1, f.item + 2} + for i, item := range f.ids { + index := int64(i) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, item).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "item_version_id": f.version + index + 1}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemRef{}).Where("space_id=? AND item_id=?", f.space, item).Updates(map[string]any{"item_version_id": f.version + index + 1, "eval_set_id": f.base + 92 + index, "eval_set_version_id": f.base + 93 + index}).Error) + if i == 0 { + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResult{}).Where("space_id=? AND item_id=?", f.space, item).Updates(map[string]any{"item_version_id": f.version + 1, "target_result_id": f.base + 91}).Error) + } else { + zero := 0.0 + require.NoError(t, f.conn.Create(&mysqlmodel.ExptTurnResult{ID: f.base + 400 + index, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: f.version + index + 1, TurnID: 0, Status: int32(entity.TurnRunState_Success), TargetResultID: f.base + 91 + index, WeightedScore: &zero}).Error) + } + } + f.locks = &filterBatchRedis{Cmdable: f.client} + f.svc.resultPublishLocker = func() lock.ILocker { return filterRealTestLocker(f.locks) } + f.svc.ExptTurnResultRepo = &filterBatchTurnRepo{IExptTurnResultRepo: f.turnRepo(), before: func(ctx context.Context, item int64) error { + require.True(t, f.server.Exists(f.itemKey(item)), "canonical payload must be rebuilt in its item lock") + // The other later item remains independently available during this read. + for _, other := range f.ids[1:] { + if other == item { + continue + } + probe := filterRealTestLocker(f.client) + ok, err := probe.Lock(ctx, f.itemKey(other), time.Minute) + require.NoError(t, err) + require.True(t, ok, "batch must not hold all item locks together") + _, err = probe.Unlock(f.itemKey(other)) + require.NoError(t, err) + } + if f.beforeRead != nil { + if err := f.beforeRead(ctx, item); err != nil { + return err + } + } + f.reads = append(f.reads, item) + return nil + }} + return f +} + +func (f *filterBatchFixture) itemKey(item int64) string { + return fmt.Sprintf("expt_item_eval_run_lock:%d:%d", f.expt, item) +} + +func (f *filterBatchFixture) keys() []string { + return []string{f.itemKey(f.item), f.itemKey(f.item + 1), f.itemKey(f.item + 2)} +} + +func (f *filterBatchFixture) unorderedIDs() []int64 { + return []int64{f.item + 2, f.item, f.item + 1, f.item + 2, f.item} +} + +func (f *filterBatchFixture) holdFirst(t *testing.T) { + t.Helper() + holder := filterRealTestLocker(f.client) + ok, err := holder.Lock(context.Background(), f.key(), time.Minute) + require.NoError(t, err) + require.True(t, ok) + t.Cleanup(func() { _, _ = holder.Unlock(f.key()) }) +} + +func (f *filterBatchFixture) attachCK(t *testing.T) (sqlmock.Sqlmock, *gorm.DB) { + t.Helper() + sqlDB, mock, err := sqlmock.New(sqlmock.ValueConverterOption(filterSQLConverter{})) + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + conn, err := gorm.Open(clickhouse.New(clickhouse.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), &gorm.Config{DisableAutomaticPing: true, Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + require.NoError(t, conn.Callback().Create().Before("gorm:create").Register("batch_insert_item_lock", func(tx *gorm.DB) { + rows, ok := tx.Statement.Dest.([]*experimentckmodel.ExptTurnResultFilter) + require.True(t, ok) + require.Len(t, rows, 1) + require.True(t, f.server.Exists(fmt.Sprintf("expt_item_eval_run_lock:%d:%s", f.expt, rows[0].ItemID))) + })) + mapping := mysqlmocks.NewMockIExptTurnResultFilterKeyMappingDAO(gomock.NewController(t)) + mapping.EXPECT().GetByExptID(gomock.Any(), f.space, f.expt).Return(nil, nil).AnyTimes() + f.svc.exptTurnResultFilterRepo = experimentrepo.NewExptTurnResultFilterRepo(experimentck.NewExptTurnResultFilterDAO(&filterCKProvider{conn: conn}, nil), mapping) + return mock, conn +} + +func (f *filterBatchFixture) expectInsert(mock sqlmock.Sqlmock, i int, saveErr error) { + index, item := int64(i), f.ids[i] + mock.ExpectBegin() + exec := mock.ExpectPrepare("INSERT INTO `expt_turn_result_filter`").ExpectExec().WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), filterLockedSQLArg{f.server, f.itemKey(item), fmt.Sprint(item)}, fmt.Sprint(f.version+index+1), int64(0), "0", int64(entity.ItemRunState_Success), fmt.Sprintf("{\"actual_output\":\"%d\"}", f.base+91+index), "{}", "{}", "{}", "{}", "{}", int64(0), fmt.Sprint(f.base+92+index), float64(0), fmt.Sprint(f.base+93+index), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()) + if saveErr != nil { + exec.WillReturnError(saveErr) + mock.ExpectRollback() + } else { + exec.WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + } +} + +type filterBatchTurnRepo struct { + repo.IExptTurnResultRepo + before func(context.Context, int64) error +} + +func (r *filterBatchTurnRepo) ListTurnResultByItemIDs(ctx context.Context, space, expt int64, ids []int64, page entity.Page, deleted bool) ([]*entity.ExptTurnResult, int64, error) { + if err := r.before(ctx, ids[0]); err != nil { + return nil, 0, err + } + return r.IExptTurnResultRepo.ListTurnResultByItemIDs(ctx, space, expt, ids, page, deleted) +} + +type filterBatchRedis struct { + infraredis.Cmdable + mu sync.Mutex + keys []string +} + +func (r *filterBatchRedis) SetNX(ctx context.Context, key string, value any, expiration time.Duration) *redis.BoolCmd { + r.mu.Lock() + r.keys = append(r.keys, key) + r.mu.Unlock() + return r.Cmdable.SetNX(ctx, key, value, expiration) +} + +func (r *filterBatchRedis) attempts() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.keys...) +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_filter_consumer_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_filter_consumer_test.go new file mode 100644 index 0000000000..7af4d3a930 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_filter_consumer_test.go @@ -0,0 +1,186 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/driver/clickhouse" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentck "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/ck" + experimentmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + mysqlmodel "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + mysqlmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/mocks" +) + +// Exercise both consumer branches with the real execution mutex and result +// service. Reentering the public refresh skips INSERT while returning nil. +func TestRunningRetryFilterConsumer_HeldLockRefresh(t *testing.T) { + for _, reservation := range []bool{true, false} { + for _, saveFails := range []bool{false, true} { + t.Run(fmt.Sprintf("reserved_%t_save_error_%t", reservation, saveFails), func(t *testing.T) { + testRunningRetryFilterConsumer(t, reservation, saveFails) + }) + } + } +} + +func testRunningRetryFilterConsumer(t *testing.T, reservation, saveFails bool) { + t.Helper() + f := newFilterSyncFixture(t) + ctx := context.Background() + event := &entity.ExptItemEvalEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, EvalSetItemID: f.item, RequestID: "rq2", ExptRunMode: entity.EvaluationModeFailRetry} + initialState, finalState := entity.ItemRunState_Processing, entity.ItemRunState_Queueing + initialTurn, finalTurn := entity.TurnRunState_Processing, entity.TurnRunState_Queueing + reservationState := entity.QuotaReservationStateNone + if reservation { + initialState, finalState = entity.ItemRunState_Queueing, entity.ItemRunState_Processing + initialTurn, finalTurn = entity.TurnRunState_Queueing, entity.TurnRunState_Processing + reservationState = entity.QuotaReservationStateReserved + } + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", initialState).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(initialState), "quota_reservation_state": int32(reservationState)}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", initialTurn).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", initialTurn).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("result_state", entity.ExptItemResultStateDefault).Error) + // Counts follow the actual canonical old bucket before atomic Start/Requeue. + pending, processing := int32(1), int32(2) + if reservation { + pending, processing = 2, 1 + } + require.NoError(t, f.conn.Model(&mysqlmodel.ExptStats{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Updates(map[string]any{"pending_cnt": pending, "processing_cnt": processing}).Error) + + // The ID-only work was queued before the source changed. A separate requeue + // holder publishes the new snapshot before the consumer acquires its lock. + holder := filterRealTestLocker(f.client) + ok, err := holder.Lock(ctx, f.key(), time.Minute) + require.NoError(t, err) + require.True(t, ok) + f.token("rq2") + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"target_result_id": f.base + 91, "item_version_id": f.version + 1}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_version_id", f.version+1).Error) + // The new DB source contract requires both frozen log versions to match. + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_version_id", f.version+1).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"item_version_id": f.version + 1, "target_result_id": f.base + 91}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemRef{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"item_version_id": f.version + 1, "eval_set_id": f.base + 92, "eval_set_version_id": f.base + 93}).Error) + _, err = holder.Unlock(f.key()) + require.NoError(t, err) + + var active atomic.Bool + var payloadReads atomic.Int32 + callback := "consumer_filter_item_lock" + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register(callback, func(tx *gorm.DB) { + if !active.Load() { + return + } + require.True(t, f.server.Exists(f.key()), "consumer source query escaped execution lock") + switch tx.Statement.Table { + case "expt_turn_result", "expt_item_ref", "expt_turn_evaluator_result_ref": + payloadReads.Add(1) + } + })) + t.Cleanup(func() { _ = f.conn.Callback().Query().Remove(callback) }) + + ctrl := gomock.NewController(t) + sqlDB, sqlMock, err := sqlmock.New(sqlmock.ValueConverterOption(filterSQLConverter{})) + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + ckConn, err := gorm.Open(clickhouse.New(clickhouse.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), &gorm.Config{DisableAutomaticPing: true, Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + var saves atomic.Int32 + competitor := filterRealTestLocker(f.client) + require.NoError(t, ckConn.Callback().Create().Before("gorm:create").Register(callback, func(tx *gorm.DB) { + saves.Add(1) + require.True(t, f.server.Exists(f.key()), "OSS CK Save escaped execution lock") + acquired, lockErr := competitor.Lock(tx.Statement.Context, f.key(), time.Minute) + require.NoError(t, lockErr) + if acquired { + _, _ = competitor.Unlock(f.key()) + } + require.False(t, acquired, "independent holder acquired item before prepared INSERT") + })) + mapping := mysqlmocks.NewMockIExptTurnResultFilterKeyMappingDAO(ctrl) + mapping.EXPECT().GetByExptID(gomock.Any(), f.space, f.expt).Return(nil, nil).AnyTimes() + f.svc.exptTurnResultFilterRepo = experimentrepo.NewExptTurnResultFilterRepo(experimentck.NewExptTurnResultFilterDAO(&filterCKProvider{conn: ckConn}, nil), mapping) + sqlMock.ExpectBegin() + insert := sqlMock.ExpectPrepare("INSERT INTO `expt_turn_result_filter`").ExpectExec().WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), filterLockedSQLArg{f.server, f.key(), fmt.Sprint(f.item)}, fmt.Sprint(f.version+1), int64(0), "0", int64(finalState), fmt.Sprintf("{\"actual_output\":\"%d\"}", f.base+91), "{}", "{}", "{}", "{}", "{}", int64(0), fmt.Sprint(f.base+92), float64(0), fmt.Sprint(f.base+93), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()) + if saveFails { + insert.WillReturnError(errors.New("consumer CK insert unavailable")) + sqlMock.ExpectRollback() + } else { + insert.WillReturnResult(sqlmock.NewResult(0, 1)) + sqlMock.ExpectCommit() + } + + // Both services use the same production key and independent real holders. + consumer := &ExptItemEventEvalServiceImpl{ + itemLocker: func() lock.ILocker { return filterRealTestLocker(f.client) }, + centralGuard: &fakeGuard{confirmResult: reservation}, + dispatchRepo: experimentrepo.NewExptItemDispatchRepo(experimentmysql.NewExptItemDispatchDAO(f.provider)), + exptItemResultRepo: f.itemRepo(), + exptTurnResultRepo: f.turnRepo(), + exptStatsRepo: experimentrepo.NewExptStatsRepo(experimentmysql.NewExptStatsDAO(f.provider)), + resultSvc: f.svc, + } + nextCalls := 0 + handler := consumer.HandleEventLock(consumer.HandleCentralReservation(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + nextCalls++ + require.True(t, f.server.Exists(f.key())) + return nil + })) + active.Store(true) + err = handler(admittedEnforceCtx(event), event) + active.Store(false) + require.NoError(t, err, "consumer display refresh errors remain best effort") + require.Equal(t, int32(1), saves.Load(), "consumer must attempt the actual OSS CK INSERT while holding its item lock") + require.GreaterOrEqual(t, payloadReads.Load(), int32(3), "canonical turn, ref and evaluator payload must be reread inside consumer lock") + require.NoError(t, sqlMock.ExpectationsWereMet()) + if reservation { + require.Equal(t, 1, nextCalls) + } else { + require.Zero(t, nextCalls, "absent reservation repairs the projection without executing") + } + items, err := f.itemRepo().BatchGet(ctx, f.space, f.expt, []int64{f.item}) + require.NoError(t, err) + require.Len(t, items, 1) + require.Equal(t, finalState, items[0].Status) + var stats mysqlmodel.ExptStats + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Take(&stats).Error) + if reservation { + require.Equal(t, int32(1), stats.PendingCnt) + require.Equal(t, int32(2), stats.ProcessingCnt) + } else { + require.Equal(t, int32(2), stats.PendingCnt) + require.Equal(t, int32(1), stats.ProcessingCnt) + } + var turn mysqlmodel.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Take(&turn).Error) + require.Equal(t, int32(finalTurn), turn.Status) + require.Equal(t, f.base+91, turn.TargetResultID) + observations, err := consumer.dispatchRepo.MGetDispatchObservations(ctx, f.space, f.expt, f.run, []int64{f.item}) + require.NoError(t, err) + require.Len(t, observations, 1) + require.Equal(t, int32(finalState), observations[0].Status) + require.Equal(t, entity.QuotaReservationStateNone, observations[0].QuotaReservationState) + require.False(t, f.server.Exists(f.key()), "consumer releases execution lock after refresh") + ok, err = competitor.Lock(ctx, f.key(), time.Minute) + require.NoError(t, err) + require.True(t, ok) + _, err = competitor.Unlock(f.key()) + require.NoError(t, err) + t.Log("REAL_MYSQL_SOURCE_CONSUMER_OSS_CK_PREPARED_INSERT_SQLMOCK_EXECUTED; boundary is SQLmock, not real CK/ByteHouse") +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_filter_mysql_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_filter_mysql_test.go new file mode 100644 index 0000000000..c5d9919801 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_filter_mysql_test.go @@ -0,0 +1,362 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/alicebob/miniredis/v2" + "github.com/bytedance/gg/gptr" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/driver/clickhouse" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/lock" + infraredis "github.com/coze-dev/coze-loop/backend/infra/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + evaluatormodel "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/evaluator/mysql/gorm_gen/model" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentck "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/ck" + experimentmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + mysqlmodel "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + mysqlmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +type filterSyncFixture struct { + *lateWriteFixture + svc *ExptResultServiceImpl + client infraredis.Cmdable + server *miniredis.Miniredis +} + +func newFilterSyncFixture(t *testing.T) *filterSyncFixture { + t.Helper() + f := newLateWriteFixture(t, "rq1") + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND turn_id<>0", f.space).Delete(&mysqlmodel.ExptTurnResultRunLog{}).Error) + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND turn_id<>0", f.space).Delete(&mysqlmodel.ExptTurnResult{}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResult{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.TurnRunState_Success), "weighted_score": 0}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("status", entity.TurnRunState_Success).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", entity.ItemRunState_Success).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": int32(entity.ExptItemResultStateResulted)}).Error) + server := miniredis.RunT(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { _ = raw.Close() }) + ctrl := gomock.NewController(t) + experiments := repomocks.NewMockIExperimentRepo(ctrl) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + annotations := repomocks.NewMockIExptAnnotateRepo(ctrl) + now := time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC) + experiments.EXPECT().GetByID(gomock.Any(), f.expt, f.space).Return(&entity.Experiment{ID: f.expt, SpaceID: f.space, StartAt: &now, ExptType: entity.ExptType_Offline, EvalSetID: f.base + 80, EvalSetVersionID: f.base + 81}, nil).AnyTimes() + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).DoAndReturn(func(ctx context.Context, ids []int64, _, _ bool) ([]*entity.EvaluatorRecord, error) { + var rows []*evaluatormodel.EvaluatorRecord + require.NoError(t, f.conn.Where("id IN ?", ids).Find(&rows).Error) + result := []*entity.EvaluatorRecord{} + for _, row := range rows { + result = append(result, &entity.EvaluatorRecord{ID: row.ID, EvaluatorVersionID: row.EvaluatorVersionID, Alias: "alias", Status: entity.EvaluatorRunStatusSuccess, EvaluatorOutputData: &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(0.0)}}}) + } + return result, nil + }).AnyTimes() + target.EXPECT().BatchGetRecordByIDs(gomock.Any(), f.space, gomock.Any()).DoAndReturn(func(ctx context.Context, _ int64, ids []int64) ([]*entity.EvalTargetRecord, error) { + return []*entity.EvalTargetRecord{{ID: ids[0], EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: map[string]*entity.Content{"actual_output": {Text: gptr.Of(fmt.Sprint(ids[0]))}}}}}, nil + }).AnyTimes() + annotations.EXPECT().GetExptTurnAnnotateRecordRefsByTurnResultIDs(gomock.Any(), f.space, gomock.Any()).Return(nil, nil).AnyTimes() + annotations.EXPECT().GetAnnotateRecordsByIDs(gomock.Any(), f.space, gomock.Any()).Return(nil, nil).AnyTimes() + svc := &ExptResultServiceImpl{ExptItemResultRepo: f.itemRepo(), ExptTurnResultRepo: f.turnRepo(), ExperimentRepo: experiments, evaluatorRecordService: records, evalTargetService: target, ExptAnnotateRepo: annotations, scoreCalculator: NewEvaluatorScoreCalculator(nil, nil), resultPublishRepo: experimentrepo.NewExptResultPublishRepo(f.provider), exptItemRefRepo: experimentrepo.NewExptItemRefRepo(experimentmysql.NewExptItemRefDAO(f.provider)), resultPublishLocker: func() lock.ILocker { return filterRealTestLocker(client) }} + return &filterSyncFixture{lateWriteFixture: f, svc: svc, client: client, server: server} +} + +func (f *filterSyncFixture) key() string { + return fmt.Sprintf("expt_item_eval_run_lock:%d:%d", f.expt, f.item) +} + +type filterStore struct { + repo.IExptTurnResultFilterRepo + mu sync.Mutex + rows []*entity.ExptTurnResultFilterEntity + fail bool + visible bool + beforeSave func([]*entity.ExptTurnResultFilterEntity) +} + +func (s *filterStore) GetExptTurnResultFilterKeyMappings(context.Context, int64, int64) ([]*entity.ExptTurnResultFilterKeyMapping, error) { + return nil, nil +} + +func (s *filterStore) Save(ctx context.Context, rows []*entity.ExptTurnResultFilterEntity) error { + if s.beforeSave != nil { + s.beforeSave(rows) + } + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.fail { + return errors.New("CK insert unavailable") + } + b, _ := json.Marshal(rows) + _ = json.Unmarshal(b, &s.rows) + return nil +} + +func (s *filterStore) GetByExptIDItemIDs(ctx context.Context, _, _, _ string, _ []string) ([]*entity.ExptTurnResultFilterEntity, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.visible { + return nil, nil + } + b, _ := json.Marshal(s.rows) + var out []*entity.ExptTurnResultFilterEntity + _ = json.Unmarshal(b, &out) + return out, nil +} + +func (s *filterStore) GetMaterializedByExptIDItemIDs(ctx context.Context, a, b, c string, d []string) ([]*entity.ExptTurnResultFilterEntity, error) { + return s.GetByExptIDItemIDs(ctx, a, b, c, d) +} + +// Requeue between the candidate message and acquisition must replace all RQ1 +// data. The actual OSS repository/DAO emits CK INSERT while the item is held. +func TestRunningRetryFilterSync_RealMySQLCurrentSourceCKInsert(t *testing.T) { + f := newFilterSyncFixture(t) + ctrl := gomock.NewController(t) + sqlDB, mock, err := sqlmock.New(sqlmock.ValueConverterOption(filterSQLConverter{})) + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + conn, err := gorm.Open(clickhouse.New(clickhouse.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), &gorm.Config{DisableAutomaticPing: true, Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + require.NoError(t, conn.Callback().Create().Before("gorm:create").Register("assert_filter_lock", func(tx *gorm.DB) { require.True(t, f.server.Exists(f.key()), "CK INSERT escaped item mutex") })) + provider := &filterCKProvider{conn: conn} + mapping := mysqlmocks.NewMockIExptTurnResultFilterKeyMappingDAO(ctrl) + mapping.EXPECT().GetByExptID(gomock.Any(), f.space, f.expt).Return(nil, nil) + f.svc.exptTurnResultFilterRepo = experimentrepo.NewExptTurnResultFilterRepo(experimentck.NewExptTurnResultFilterDAO(provider, nil), mapping) + // A stale ID-only event is prepared now; no RQ1 payload is supplied. + holder := filterRealTestLocker(f.client) + ok, err := holder.Lock(context.Background(), f.key(), time.Minute) + require.NoError(t, err) + require.True(t, ok) + require.True(t, errno.NeedMQRetry(f.svc.UpsertExptTurnResultFilter(context.Background(), f.space, f.expt, []int64{f.item}))) + f.token("rq2") + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResult{}).Where("space_id=?", f.space).Updates(map[string]any{"target_result_id": f.base + 91, "item_version_id": f.version + 1}).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_version_id", f.version+1).Error) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemRef{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"item_version_id": f.version + 1, "eval_set_id": f.base + 92, "eval_set_version_id": f.base + 93}).Error) + _, err = holder.Unlock(f.key()) + require.NoError(t, err) + mock.ExpectBegin() + mock.ExpectPrepare("INSERT INTO `expt_turn_result_filter`").ExpectExec().WithArgs(fmt.Sprint(f.space), fmt.Sprint(f.expt), filterLockedSQLArg{f.server, f.key(), fmt.Sprint(f.item)}, fmt.Sprint(f.version+1), int64(0), "0", int64(entity.ItemRunState_Success), fmt.Sprintf("{\"actual_output\":\"%d\"}", f.base+91), "{}", "{}", "{}", "{}", "{}", int64(0), fmt.Sprint(f.base+92), float64(0), fmt.Sprint(f.base+93), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + require.NoError(t, f.svc.UpsertExptTurnResultFilter(context.Background(), f.space, f.expt, []int64{f.item})) + require.NoError(t, mock.ExpectationsWereMet()) + t.Log("REAL_MYSQL_SOURCE_OSS_CK_INSERT_SQLMOCK_EXECUTED; no real CK server") +} + +type filterCKProvider struct{ conn *gorm.DB } + +func (p *filterCKProvider) NewSession(ctx context.Context) *gorm.DB { return p.conn.WithContext(ctx) } + +func TestRunningRetryFilterSync_ResultedInvisibleWriteNextTick(t *testing.T) { + f := newFilterSyncFixture(t) + store := &filterStore{} + f.svc.exptTurnResultFilterRepo = store + store.beforeSave = func(rows []*entity.ExptTurnResultFilterEntity) { + require.True(t, f.server.Exists(f.key())) + require.Len(t, rows, 1) + require.Equal(t, f.item, rows[0].ItemID) + } + for _, state := range []string{"save_error", "missing", "equal"} { + store.fail = state == "save_error" + store.visible = state == "equal" + ready, err := f.svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), f.space, f.expt, f.run) + if state != "save_error" { + require.NoError(t, err) + require.True(t, ready, "Save ACK permits finish even when physical rows are missing") + } else { + require.True(t, errno.NeedMQRetry(err)) + require.False(t, ready) + } + } + require.NoError(t, f.conn.Model(&mysqlmodel.ExptRunLog{}).Where("space_id=?", f.space).Update("status", entity.ExptStatus_Success).Error) + ready, err := f.svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), f.space, f.expt, f.run) + require.NoError(t, err) + require.True(t, ready, "immutable source remains readable after run ends") +} + +type filterSQLConverter struct{} + +func (filterSQLConverter) ConvertValue(value any) (driver.Value, error) { + switch value.(type) { + case map[string]string, map[string]float64, map[string]int8, map[string]int64: + raw, err := json.Marshal(value) + return string(raw), err + } + return driver.DefaultParameterConverter.ConvertValue(value) +} + +type filterLockedSQLArg struct { + server *miniredis.Miniredis + key, value string +} + +func (a filterLockedSQLArg) Match(value driver.Value) bool { + return a.server.Exists(a.key) && value == a.value +} + +func TestRunningRetryFilterSync_EnumerationAcrossPages(t *testing.T) { + f := newFilterSyncFixture(t) + scanner := f.itemRepo().(repo.IExptManualRetryFilterRepo) + ext, err := entity.MergeManualRetryContext(nil, &entity.ManualRetryContext{RunID: f.run, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry}) + require.NoError(t, err) + rows := make([]*mysqlmodel.ExptItemResult, 0, 405) + for i := 0; i < 405; i++ { + rows = append(rows, &mysqlmodel.ExptItemResult{ID: f.base + 10000 + int64(i), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1000 + int64(i), Status: int32(entity.ItemRunState_Success), Ext: &ext}) + } + require.NoError(t, f.conn.CreateInBatches(rows, 100).Error) + var ids []int64 + cursor := int64(0) + pages := 0 + for { + page, next, err := scanner.ScanManualRetryFilterItems(context.Background(), f.space, f.expt, f.run, cursor, 200) + require.NoError(t, err) + ids = append(ids, page...) + pages++ + if next == 0 { + break + } + require.Greater(t, next, cursor) + cursor = next + } + require.Len(t, ids, 406) + require.Equal(t, 3, pages) + require.Contains(t, ids, f.item+1404, "last page Resulted candidates must not disappear") + oldExt, err := entity.MergeManualRetryContext(nil, &entity.ManualRetryContext{RunID: f.run - 1, RequestID: "old", Mode: entity.EvaluationModeFailRetry}) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, rows[0].ItemID).Updates(map[string]any{"expt_run_id": f.run - 1, "ext": oldExt}).Error) + _, _, err = scanner.ScanManualRetryFilterItems(context.Background(), f.space, f.expt, f.run, 0, 200) + require.NoError(t, err, "old run is outside current enumeration") + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, rows[1].ItemID).Update("ext", []byte(`{"manual_retry":"broken"}`)).Error) + _, _, err = scanner.ScanManualRetryFilterItems(context.Background(), f.space, f.expt, f.run, 0, 200) + require.Error(t, err, "damaged context must not become an empty set") +} + +func TestRunningRetryFilterSync_NoManualNoFilterIO(t *testing.T) { + f := newFilterSyncFixture(t) + f.token("") + f.svc.exptTurnResultFilterRepo = nil + f.svc.resultPublishRepo = nil + ready, err := f.svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), f.space, f.expt, f.run) + require.NoError(t, err) + require.True(t, ready) +} + +func TestRunningRetryFilterSync_PendingPublicationNeverFinishes(t *testing.T) { + for _, status := range []entity.ItemRunState{entity.ItemRunState_Queueing, entity.ItemRunState_Processing, entity.ItemRunState_Success} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + f := newFilterSyncFixture(t) + store := &filterStore{visible: true} + f.svc.exptTurnResultFilterRepo = store + result := entity.ExptItemResultStateDefault + if status == entity.ItemRunState_Success { + result = entity.ExptItemResultStateLogged + } + require.NoError(t, f.conn.Model(&mysqlmodel.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(status), "result_state": int32(result)}).Error) + ready, err := f.svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), f.space, f.expt, f.run) + require.True(t, errno.NeedMQRetry(err)) + require.False(t, ready) + require.Nil(t, store.rows, "unfinished source must not be saved or marked complete") + }) + } +} + +// Match the current asynchronous COM-style repo contract: Save can succeed +// while its incomplete comparison reader cannot establish full materialization. +func TestRunningRetryFilterSync_AsyncPartialReaderCannotPass(t *testing.T) { + f := newFilterSyncFixture(t) + store := &filterStore{visible: true} + f.svc.exptTurnResultFilterRepo = &partialFilterStore{store: store} + require.NoError(t, f.svc.UpsertExptTurnResultFilter(context.Background(), f.space, f.expt, []int64{f.item})) + ready, err := f.svc.materializedFilterMatchesLocked(context.Background(), f.space, f.expt, f.item) + require.NoError(t, err) + require.False(t, ready) +} + +type partialFilterStore struct { + repo.IExptTurnResultFilterRepo + store *filterStore +} + +func (s *partialFilterStore) Save(ctx context.Context, rows []*entity.ExptTurnResultFilterEntity) error { + return s.store.Save(ctx, rows) +} + +func (s *partialFilterStore) GetByExptIDItemIDs(ctx context.Context, a, b, c string, d []string) ([]*entity.ExptTurnResultFilterEntity, error) { + return s.store.GetByExptIDItemIDs(ctx, a, b, c, d) +} + +func (s *partialFilterStore) GetExptTurnResultFilterKeyMappings(ctx context.Context, a, b int64) ([]*entity.ExptTurnResultFilterKeyMapping, error) { + return s.store.GetExptTurnResultFilterKeyMappings(ctx, a, b) +} + +// The current payload and primary source must be re-read when an old source +// cache is present; it is useful only as an ID selection, never as projection. +func TestRunningRetryFilterSync_CachedOldSourceIgnored(t *testing.T) { + f := newFilterSyncFixture(t) + old := f.source() + f.token("rq2") + require.NoError(t, f.conn.Model(&mysqlmodel.ExptTurnResult{}).Where("space_id=?", f.space).Update("target_result_id", f.base+99).Error) + store := &filterStore{visible: true} + f.svc.exptTurnResultFilterRepo = store + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: old}) + require.NoError(t, f.svc.UpsertExptTurnResultFilter(ctx, f.space, f.expt, []int64{f.item})) + require.Len(t, store.rows, 1) + require.Equal(t, fmt.Sprint(f.base+99), store.rows[0].EvalTargetData["actual_output"]) +} + +// Source fields are read beneath the item holder, including refs and scores; +// absence remains absence, and zero scores remain explicitly mapped. +func TestRunningRetryFilterSync_ScorePresenceAliasInline(t *testing.T) { + for _, score := range []*float64{nil, gptr.Of(0.0)} { + t.Run(fmt.Sprint(score == nil), func(t *testing.T) { + f := newFilterSyncFixture(t) + ctrl := gomock.NewController(t) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).DoAndReturn(func(ctx context.Context, ids []int64, _, _ bool) ([]*entity.EvaluatorRecord, error) { + require.True(t, f.server.Exists(f.key())) + return []*entity.EvaluatorRecord{{ID: f.base + 61, EvaluatorVersionID: f.base + 51, Alias: "alias", Status: entity.EvaluatorRunStatusSuccess, EvaluatorOutputData: &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: score}}}, {ID: f.base + 62, EvaluatorVersionID: 0, Status: entity.EvaluatorRunStatusSuccess}}, nil + }) + require.NoError(t, f.conn.Create(&mysqlmodel.ExptTurnEvaluatorResultRef{ID: f.base + 802, SpaceID: f.space, ExptID: f.expt, ExptTurnResultID: f.base + 400, SourceType: 2, InlineKey: "inline", EvaluatorResultID: f.base + 62}).Error) + filterRepo := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + filterRepo.EXPECT().GetExptTurnResultFilterKeyMappings(gomock.Any(), f.space, f.expt).Return([]*entity.ExptTurnResultFilterKeyMapping{{FieldType: entity.FieldTypeEvaluator, FromField: entity.EncodeEvaluatorInstanceKey(f.base+51, "alias"), ToKey: "key1"}}, nil) + filterRepo.EXPECT().Save(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, rows []*entity.ExptTurnResultFilterEntity) error { + require.True(t, f.server.Exists(f.key())) + require.Len(t, rows, 1) + if score == nil { + require.Empty(t, rows[0].EvaluatorScore) + } else { + require.Equal(t, map[string]float64{"key1": 0}, rows[0].EvaluatorScore) + } + return nil + }) + f.svc.evaluatorRecordService = records + f.svc.exptTurnResultFilterRepo = filterRepo + require.NoError(t, f.svc.UpsertExptTurnResultFilter(context.Background(), f.space, f.expt, []int64{f.item})) + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_filter_sync_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_filter_sync_test.go new file mode 100644 index 0000000000..4c1f7bc2cc --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_filter_sync_test.go @@ -0,0 +1,590 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + lwtmocks "github.com/coze-dev/coze-loop/backend/infra/platestwrite/mocks" + infraredis "github.com/coze-dev/coze-loop/backend/infra/redis" + idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +// Removing the public item's mutex exposes a source read while requeue owns it. +func TestRunningRetryFilterSync_BusyDoesNotReadPayload(t *testing.T) { + ctrl := gomock.NewController(t) + server := miniredis.RunT(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { require.NoError(t, raw.Close()) }) + const space, expt, item = int64(7384603622000000001), int64(7384603622000000002), int64(7384603622000000011) + key := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", expt, item) + holder := filterRealTestLocker(client) + ok, err := holder.Lock(context.Background(), key, time.Minute) + require.NoError(t, err) + require.True(t, ok) + t.Cleanup(func() { _, _ = holder.Unlock(key) }) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + var reads atomic.Int32 + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), space, expt, gomock.Any(), gomock.Any(), false).DoAndReturn(func(context.Context, int64, int64, []int64, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + reads.Add(1) + return nil, 0, nil + }).AnyTimes() + svc := &ExptResultServiceImpl{ExptTurnResultRepo: turns, resultPublishLocker: func() lock.ILocker { return filterRealTestLocker(client) }} + err = svc.UpsertExptTurnResultFilter(context.Background(), space, expt, []int64{item}) + require.True(t, errno.NeedMQRetry(err), "held item must broker retry, got %v", err) + require.Zero(t, reads.Load(), "current payload was read before acquiring item mutex") +} + +// A Resulted row must still pass through final calibration; missing publication +// dependencies must leave it pending rather than claim completion. +func TestRunningRetryFilterSync_ResultedNeedsCalibration(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET status=? WHERE space_id=? AND expt_id=? AND item_id=?", entity.ItemRunState_Success, f.space, f.expt, f.item).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET status=?,result_state=? WHERE space_id=? AND expt_id=? AND item_id=?", entity.ItemRunState_Success, entity.ExptItemResultStateResulted, f.space, f.expt, f.item).Error) + svc := &ExptResultServiceImpl{ExptItemResultRepo: f.itemRepo()} + port, ok := any(svc).(interface { + ReconcileManualRetryFilterBeforeFinish(context.Context, int64, int64, int64) (bool, error) + }) + require.True(t, ok, "final calibration entry is missing; Resulted is excluded from Logged scans") + if ok { + _, err := port.ReconcileManualRetryFilterBeforeFinish(context.Background(), f.space, f.expt, f.run) + require.True(t, errno.NeedMQRetry(err), "missing filter dependencies must keep scheduler retrying") + } +} + +// This test catches a comparison that only checks scores/status while an +// old per-item snapshot or target field remains materialized. +func TestRunningRetryFilterSync_FullProjectionMismatch(t *testing.T) { + for _, tc := range []struct { + name string + version int64 + missingWeight bool + ready bool + }{{"nonzero_correct_default_strict", 222, false, true}, {"stale_nonzero_default_strict", 111, false, false}, {"missing_version_default_strict", 0, false, false}, {"missing_weight_zero", 222, true, false}} { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + filters := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + experiments := repomocks.NewMockIExperimentRepo(ctrl) + annotations := repomocks.NewMockIExptAnnotateRepo(ctrl) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + now := time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{10}, gomock.Any(), false).Return([]*entity.ExptTurnResult{{ID: 101, ItemID: 10, ItemVersionID: 222, TurnID: 0, WeightedScore: gptr.Of(0.0)}}, int64(1), nil) + items.EXPECT().BatchGet(gomock.Any(), int64(3), int64(1), []int64{10}).Return([]*entity.ExptItemResult{{ItemID: 10, Status: entity.ItemRunState_Success}}, nil) + filters.EXPECT().GetExptTurnResultFilterKeyMappings(gomock.Any(), int64(3), int64(1)).Return(nil, nil) + experiments.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, StartAt: &now, EvalSetID: 55, EvalSetVersionID: 66}, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(3), []int64{101}).Return(nil, nil) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return(nil, nil) + target.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), gomock.Any()).Return(nil, nil) + annotations.EXPECT().GetExptTurnAnnotateRecordRefsByTurnResultIDs(gomock.Any(), int64(3), []int64{101}).Return(nil, nil) + annotations.EXPECT().GetAnnotateRecordsByIDs(gomock.Any(), int64(3), gomock.Any()).Return(nil, nil) + filters.EXPECT().GetByExptIDItemIDs(gomock.Any(), "3", "1", "2026-10-20", []string{"10"}).Return([]*entity.ExptTurnResultFilterEntity{{SpaceID: 3, ExptID: 1, ItemID: 10, TurnID: 0, Status: entity.ItemRunState_Success, ItemVersionID: tc.version, EvaluatorWeightedScore: func() *float64 { + if tc.missingWeight { + return nil + } + return gptr.Of(0.0) + }(), EvalSetID: 55, EvalSetVersionID: 66}}, nil) + sink := &fullFilterTestSink{IExptTurnResultFilterRepo: filters} + svc := ExptResultServiceImpl{ExptTurnResultRepo: turns, ExptItemResultRepo: items, ExperimentRepo: experiments, ExptAnnotateRepo: annotations, evaluatorRecordService: records, evalTargetService: target, exptTurnResultFilterRepo: sink} + equal, err := svc.materializedFilterMatchesLocked(context.Background(), 3, 1, 10) + require.Equal(t, tc.ready, equal) + require.NoError(t, err) + }) + } +} + +type fullFilterTestSink struct{ repo.IExptTurnResultFilterRepo } + +func (s *fullFilterTestSink) GetMaterializedByExptIDItemIDs(ctx context.Context, space, expt, date string, ids []string) ([]*entity.ExptTurnResultFilterEntity, error) { + return s.GetByExptIDItemIDs(ctx, space, expt, date, ids) +} + +func TestRunningRetryFilterSync_SaveACKAllowsFinishWithoutCKQuery(t *testing.T) { + for _, tc := range []struct { + name string + rows []*entity.ExptTurnResultFilterEntity + err error + }{ + {name: "absent"}, + {name: "stale", rows: []*entity.ExptTurnResultFilterEntity{{SpaceID: 3, ExptID: 1, ItemID: 10, ItemVersionID: 111, Status: entity.ItemRunState_Fail}}}, + {name: "query_unavailable", err: errors.New("CK query unavailable")}, + } { + for _, composition := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/finish_composition=%t", tc.name, composition), func(t *testing.T) { + ctrl := gomock.NewController(t) + svc, filters := newFinishACKTestService(t, ctrl) + queries, saves := 0, 0 + filters.EXPECT().GetByExptIDItemIDs(gomock.Any(), "3", "1", "2026-10-20", []string{"10"}).DoAndReturn(func(context.Context, string, string, string, []string) ([]*entity.ExptTurnResultFilterEntity, error) { + queries++ + return tc.rows, tc.err + }).AnyTimes() + filters.EXPECT().Save(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultFilterEntity) error { + saves++ + require.Len(t, rows, 1) + require.Equal(t, int64(10), rows[0].ItemID) + require.Equal(t, int64(222), rows[0].ItemVersionID) + require.Equal(t, entity.ItemRunState_Success, rows[0].Status) + return nil + }) + if composition { + manager := &naturalFinishProbe{IExptManager: svcmocks.NewMockIExptManager(ctrl), disposition: entity.ExptRunFinishFinished} + idem := idemmocks.NewMockIdempotentService(ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) + idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}).AnyTimes() + base := &exptBaseExec{Manager: manager, idem: idem, configer: config, resultSvc: svc} + err := base.exptEnd(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit}, &entity.Experiment{ExptType: entity.ExptType_Offline}) + require.NoError(t, err, "Save ACK must allow finish composition") + require.Equal(t, 1, manager.calls) + } else { + ready, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.NoError(t, err, "Save ACK must complete calibration") + require.True(t, ready) + } + require.Equal(t, 1, saves) + require.Zero(t, queries, "finish calibration must not query CK after Save ACK") + }) + } + } +} + +func newFinishACKTestService(t *testing.T, ctrl *gomock.Controller) (*ExptResultServiceImpl, *repomocks.MockIExptTurnResultFilterRepo) { + t.Helper() + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + filters := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + experiments := repomocks.NewMockIExperimentRepo(ctrl) + annotations := repomocks.NewMockIExptAnnotateRepo(ctrl) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + publication := repomocks.NewMockIExptResultPublishRepo(ctrl) + now := time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{10}, gomock.Any(), false).Return([]*entity.ExptTurnResult{{ID: 101, SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 10, ItemVersionID: 222, TurnID: 0, Status: int32(entity.TurnRunState_Success), WeightedScore: gptr.Of(0.0)}}, int64(1), nil) + items.EXPECT().BatchGet(gomock.Any(), int64(3), int64(1), []int64{10}).Return([]*entity.ExptItemResult{{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 10, ItemVersionID: 222, Status: entity.ItemRunState_Success}}, nil) + filters.EXPECT().GetExptTurnResultFilterKeyMappings(gomock.Any(), int64(3), int64(1)).Return(nil, nil) + experiments.EXPECT().GetByID(gomock.Any(), int64(1), int64(3)).Return(&entity.Experiment{ID: 1, SpaceID: 3, LatestRunID: 2, StartAt: &now, ExptType: entity.ExptType_Offline, EvalSetID: 55, EvalSetVersionID: 66}, nil) + turns.EXPECT().BatchGetTurnEvaluatorResultRef(gomock.Any(), int64(3), []int64{101}).Return(nil, nil) + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).Return(nil, nil) + target.EXPECT().BatchGetRecordByIDs(gomock.Any(), int64(3), gomock.Any()).Return(nil, nil) + annotations.EXPECT().GetExptTurnAnnotateRecordRefsByTurnResultIDs(gomock.Any(), int64(3), []int64{101}).Return(nil, nil) + annotations.EXPECT().GetAnnotateRecordsByIDs(gomock.Any(), int64(3), gomock.Any()).Return(nil, nil) + retryJSON, err := json.Marshal(&entity.ManualRetryContext{RunID: 2, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry}) + require.NoError(t, err) + source := &entity.ExptItemResultPublishSource{ + Source: &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{SpaceID: 3, ExptID: 1, RunID: 2, ItemID: 10, RequestID: "rq1"}}, + Item: &entity.ExptItemResult{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 10, ItemVersionID: 222, Status: entity.ItemRunState_Success, Ext: map[string]string{entity.ManualRetryExtKey: string(retryJSON)}}, + ItemLog: &entity.ExptItemResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 10, ItemVersionID: 222, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateResulted)}, + } + publication.EXPECT().GetItemResultPublishSource(gomock.Any(), int64(3), int64(1), int64(2), int64(10)).Return(source, nil) + svc := &ExptResultServiceImpl{ExptTurnResultRepo: turns, ExptItemResultRepo: &versionCalibrationScanner{items}, ExperimentRepo: experiments, ExptAnnotateRepo: annotations, evaluatorRecordService: records, evalTargetService: target, exptTurnResultFilterRepo: &fullFilterTestSink{filters}, resultPublishRepo: publication} + return svc, filters +} + +func TestRunningRetryFilterSync_SaveACKPreservesContextError(t *testing.T) { + for _, expired := range []bool{false, true} { + for _, leased := range []bool{false, true} { + t.Run(fmt.Sprintf("expired=%t/leased=%t", expired, leased), func(t *testing.T) { + ctrl := gomock.NewController(t) + svc, filters := newFinishACKTestService(t, ctrl) + ctx, cancel := context.WithCancel(context.Background()) + want := context.Canceled + if expired { + cancel() + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + want = context.DeadlineExceeded + } + defer cancel() + caller := ctx + if leased { + caller = context.Background() + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:1:10", gomock.Any(), gomock.Any()).Return(true, ctx, cancel, nil) + locker.EXPECT().Unlock("expt_item_eval_run_lock:1:10").Return(true, nil) + svc.resultPublishLocker = func() lock.ILocker { return locker } + } + filters.EXPECT().Save(gomock.Any(), gomock.Any()).DoAndReturn(func(savedCtx context.Context, _ []*entity.ExptTurnResultFilterEntity) error { + if expired { + <-savedCtx.Done() + } else { + cancel() + } + return nil + }) + ready, err := svc.ReconcileManualRetryFilterBeforeFinish(caller, 3, 1, 2) + require.False(t, ready) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, want) + }) + } + } +} + +func TestRunningRetryFilterSync_SaveErrorKeepsFinishPending(t *testing.T) { + ctrl := gomock.NewController(t) + svc, filters := newFinishACKTestService(t, ctrl) + fault := errors.New("CK save unavailable") + filters.EXPECT().Save(gomock.Any(), gomock.Any()).Return(fault) + ready, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.False(t, ready) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, fault) +} + +func TestRunningRetryFilterSync_CalibrationSourcePending(t *testing.T) { + for _, tc := range []struct { + name string + run int64 + request string + item entity.ItemRunState + log entity.ItemRunState + result entity.ExptItemResultState + }{ + {"request_changed", 2, "rq2", entity.ItemRunState_Success, entity.ItemRunState_Success, entity.ExptItemResultStateResulted}, + {"run_changed", 1, "rq1", entity.ItemRunState_Success, entity.ItemRunState_Success, entity.ExptItemResultStateResulted}, + {"item_queueing", 2, "rq1", entity.ItemRunState_Queueing, entity.ItemRunState_Success, entity.ExptItemResultStateResulted}, + {"item_processing", 2, "rq1", entity.ItemRunState_Processing, entity.ItemRunState_Success, entity.ExptItemResultStateResulted}, + {"log_queueing", 2, "rq1", entity.ItemRunState_Success, entity.ItemRunState_Queueing, entity.ExptItemResultStateResulted}, + {"log_processing", 2, "rq1", entity.ItemRunState_Success, entity.ItemRunState_Processing, entity.ExptItemResultStateResulted}, + {"logged", 2, "rq1", entity.ItemRunState_Success, entity.ItemRunState_Success, entity.ExptItemResultStateLogged}, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + publication := repomocks.NewMockIExptResultPublishRepo(ctrl) + filters := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + retryJSON, err := json.Marshal(&entity.ManualRetryContext{RunID: tc.run, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry}) + require.NoError(t, err) + publication.EXPECT().GetItemResultPublishSource(gomock.Any(), int64(3), int64(1), int64(2), int64(10)).Return(&entity.ExptItemResultPublishSource{ + Source: &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{RequestID: tc.request}}, + Item: &entity.ExptItemResult{Status: tc.item, Ext: map[string]string{entity.ManualRetryExtKey: string(retryJSON)}}, + ItemLog: &entity.ExptItemResultRunLog{Status: int32(tc.log), ResultState: int32(tc.result)}, + }, nil) + svc := &ExptResultServiceImpl{ExptItemResultRepo: &versionCalibrationScanner{items}, resultPublishRepo: publication, exptTurnResultFilterRepo: filters} + ready, err := svc.ReconcileManualRetryFilterBeforeFinish(context.Background(), 3, 1, 2) + require.False(t, ready) + require.True(t, errno.NeedMQRetry(err)) + }) + } +} + +// A producer already owns the execution mutex; using the public refresh here +// would return busy and silently skip the real CK write. +func TestRunningRetryFilterSync_ProducerDoesNotReenter(t *testing.T) { + ctrl := gomock.NewController(t) + server := miniredis.RunT(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { _ = raw.Close() }) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + var reads atomic.Int32 + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{10}, gomock.Any(), false).DoAndReturn(func(context.Context, int64, int64, []int64, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + reads.Add(1) + return nil, 0, nil + }).AnyTimes() + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(10)).Return(&entity.ExptItemCurrentRequest{ItemID: 10, RunID: 2, Mode: entity.EvaluationModeSubmit, Status: entity.ItemRunState_Queueing}, nil) + items.EXPECT().MarkProcessingIfCurrent(gomock.Any(), int64(3), int64(1), int64(2), int64(10), "").Return(true, nil) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + publisher.EXPECT().BatchPublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}) + result := &ExptResultServiceImpl{ExptTurnResultRepo: turns, resultPublishLocker: func() lock.ILocker { return filterRealTestLocker(client) }} + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: items, Publisher: publisher, Configer: config, ResultSvc: result} + require.NoError(t, scheduler.submitCurrentItem(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, &entity.ExptEvalItem{ItemID: 10}, filterRealTestLocker(client))) + require.Equal(t, int32(1), reads.Load(), "producer must refresh under its existing lock without reacquiring it") +} + +// Manual mapping refresh must retain its event and also enter the same locked +// reconstruction path. The old method ignores the supplied item IDs entirely. +func TestRunningRetryFilterSync_ManualRefreshUsesItemLock(t *testing.T) { + ctrl := gomock.NewController(t) + server := miniredis.RunT(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { _ = raw.Close() }) + experiments := repomocks.NewMockIExperimentRepo(ctrl) + annotations := repomocks.NewMockIExptAnnotateRepo(ctrl) + filters := repomocks.NewMockIExptTurnResultFilterRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + lwt := lwtmocks.NewMockILatestWriteTracker(ctrl) + lwt.EXPECT().CheckWriteFlagByID(gomock.Any(), gomock.Any(), int64(1)).Return(false) + experiments.EXPECT().MGetByID(gomock.Any(), []int64{1}, int64(3)).Return([]*entity.Experiment{{ID: 1, SpaceID: 3}}, nil) + annotations.EXPECT().GetExptTurnResultTagRefs(gomock.Any(), int64(1), int64(3)).Return(nil, nil) + filters.EXPECT().InsertExptTurnResultFilterKeyMappings(gomock.Any(), gomock.Any()).Return(nil) + publisher.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + var reads atomic.Int32 + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{10}, gomock.Any(), false).DoAndReturn(func(context.Context, int64, int64, []int64, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + reads.Add(1) + require.True(t, server.Exists("expt_item_eval_run_lock:1:10")) + return nil, 0, nil + }).AnyTimes() + svc := ExptResultServiceImpl{ExperimentRepo: experiments, ExptAnnotateRepo: annotations, ExptTurnResultRepo: turns, exptTurnResultFilterRepo: filters, publisher: publisher, lwt: lwt, resultPublishLocker: func() lock.ILocker { return filterRealTestLocker(client) }} + require.NoError(t, svc.ManualUpsertExptTurnResultFilter(context.Background(), 3, 1, []int64{10})) + require.Equal(t, int32(1), reads.Load(), "manual entry never refreshed supplied candidates") +} + +func TestRunningRetryFilterSync_IndependentItems(t *testing.T) { + ctrl := gomock.NewController(t) + server := miniredis.RunT(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { _ = raw.Close() }) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + entered := make(chan struct{}) + release := make(chan struct{}) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{10}, gomock.Any(), false).DoAndReturn(func(context.Context, int64, int64, []int64, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + close(entered) + <-release + return nil, 0, nil + }) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{11}, gomock.Any(), false).Return(nil, int64(0), nil) + svc := ExptResultServiceImpl{ExptTurnResultRepo: turns, resultPublishLocker: func() lock.ILocker { return filterRealTestLocker(client) }} + first := make(chan error, 1) + go func() { first <- svc.UpsertExptTurnResultFilter(context.Background(), 3, 1, []int64{10}) }() + <-entered + second := make(chan error, 1) + go func() { second <- svc.UpsertExptTurnResultFilter(context.Background(), 3, 1, []int64{11}) }() + select { + case err := <-second: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Error("different item blocked behind experiment mutex") + } + close(release) + require.NoError(t, <-first) +} + +// All offline modes must carry the calibration result into the real completion +// branch. Broker retries bypass the scheduler's business failure completion. +func TestRunningRetryFilterSync_OfflineFinishBlocksUntilCalibrated(t *testing.T) { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeSubmit, entity.EvaluationModeFailRetry, entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + idem := idemmocks.NewMockIdempotentService(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + baseResult := svcmocks.NewMockExptResultService(ctrl) + result := &finishCalibrationTestResult{ExptResultService: baseResult} + items.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), int64(0), int64(1), int64(3)).Return(nil, int64(0), nil).AnyTimes() + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + var complete atomic.Int32 + manager.EXPECT().CompleteRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, int64, int64, int64, *entity.Session, ...entity.CompleteExptOptionFn) error { + complete.Add(1) + return nil + }).AnyTimes() + manager.EXPECT().CompleteExpt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + config.EXPECT().GetExptExecConf(gomock.Any(), int64(3)).Return(&entity.ExptExecConf{}).AnyTimes() + idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: mode} + expt := &entity.Experiment{ExptType: entity.ExptType_Offline} + var end func(context.Context, *entity.ExptScheduleEvent, *entity.Experiment, int, int) (bool, error) + switch mode { + case entity.EvaluationModeSubmit: + end = (&ExptSubmitExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result}).ExptEnd + case entity.EvaluationModeFailRetry: + end = (&ExptFailRetryExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result}).ExptEnd + case entity.EvaluationModeRetryAll: + end = (&ExptRetryAllExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, resultSvc: result}).ExptEnd + case entity.EvaluationModeRetryItems: + logs := repomocks.NewMockIExptRunLogRepo(ctrl) + logs.EXPECT().Get(gomock.Any(), int64(1), int64(2)).Return(&entity.ExptRunLog{}, nil).AnyTimes() + manager.EXPECT().LockCompletingRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + manager.EXPECT().UnlockCompletingRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + end = (&ExptRetryItemsExec{manager: manager, idem: idem, configer: config, exptItemResultRepo: items, exptRunLogRepo: logs, resultSvc: result}).ExptEnd + } + for _, fault := range []error{errors.New("filter publication unavailable"), nil} { + result.err = fault + result.ready = false + scheduler := &ExptSchedulerImpl{Manager: manager} + err := scheduler.HandleEventErr(func(ctx context.Context, event *entity.ExptScheduleEvent) error { + _, err := end(ctx, event, expt, 0, 0) + return err + })(context.Background(), event) + require.True(t, errno.NeedMQRetry(err)) + require.Zero(t, complete.Load(), "uncalibrated filter completed offline run") + } + result.ready = true + result.err = nil + _, err := end(context.Background(), event, expt, 0, 0) + require.NoError(t, err) + require.Equal(t, int32(1), complete.Load()) + }) + } +} + +type finishCalibrationTestResult struct { + ExptResultService + ready bool + err error + panicValue any +} + +func (r *finishCalibrationTestResult) ReconcileManualRetryFilterBeforeFinish(context.Context, int64, int64, int64) (bool, error) { + if r.panicValue != nil { + panic(r.panicValue) + } + return r.ready, r.err +} + +func TestRunningRetryFilterSync_CalibrationPanicBrokerRetry(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + idem := idemmocks.NewMockIdempotentService(ctrl) + idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) + result := &finishCalibrationTestResult{panicValue: "filter runtime failure"} + base := &exptBaseExec{Manager: manager, idem: idem, resultSvc: result} + scheduler := &ExptSchedulerImpl{Manager: manager} + err := scheduler.HandleEventErr(func(ctx context.Context, event *entity.ExptScheduleEvent) error { + return base.exptEnd(ctx, event, &entity.Experiment{ExptType: entity.ExptType_Offline}) + })(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit}) + require.True(t, errno.NeedMQRetry(err), "calibration runtime failure must never enter CompleteRun/Failed branch") +} + +func TestRunningRetryFilterSync_LeaseLostBeforeSave(t *testing.T) { + ctrl := gomock.NewController(t) + server := miniredis.RunT(t) + client, err := infraredis.NewClient(&redis.Options{Addr: server.Addr()}) + require.NoError(t, err) + raw, _ := infraredis.Unwrap(client) + t.Cleanup(func() { _ = raw.Close() }) + base := filterRealTestLocker(client) + lockCtx, cancel := context.WithCancel(context.Background()) + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), "expt_item_eval_run_lock:1:10", gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, string, time.Duration, time.Duration) (bool, context.Context, func(), error) { + ok, err := base.Lock(lockCtx, "expt_item_eval_run_lock:1:10", time.Minute) + return ok, lockCtx, cancel, err + }) + locker.EXPECT().Unlock("expt_item_eval_run_lock:1:10").DoAndReturn(base.Unlock) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + turns.EXPECT().ListTurnResultByItemIDs(gomock.Any(), int64(3), int64(1), []int64{10}, gomock.Any(), false).DoAndReturn(func(context.Context, int64, int64, []int64, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + cancel() + return nil, 0, nil + }) + svc := ExptResultServiceImpl{ExptTurnResultRepo: turns, resultPublishLocker: func() lock.ILocker { return locker }} + err = svc.UpsertExptTurnResultFilter(context.Background(), 3, 1, []int64{10}) + require.True(t, errno.NeedMQRetry(err), "lost lease may not be acknowledged as successful refresh") +} + +// Public Compare must rebuild under the same item lock as Upsert, including +// records whose scores are absent rather than using stale cached DTOs. +func TestRunningRetryFilterSync_CompareReadsInsideItemLock(t *testing.T) { + f := newFilterSyncFixture(t) + store := &filterStore{visible: true} + f.svc.exptTurnResultFilterRepo = store + require.NoError(t, f.svc.UpsertExptTurnResultFilter(context.Background(), f.space, f.expt, []int64{f.item})) + ctrl := gomock.NewController(t) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + f.svc.publisher = publisher + // A missing filter must retain the original check-event retry budget. + store.visible = false + publisher.EXPECT().PublishExptTurnResultFilterEvent(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, event *entity.ExptTurnResultFilterEvent, _ *time.Duration) error { + require.True(t, f.server.Exists(f.key())) + require.Equal(t, []int64{f.item}, event.ItemID) + require.Equal(t, int32(2), *event.RetryTimes) + return nil + }) + require.NoError(t, f.svc.CompareExptTurnResultFilters(context.Background(), f.space, f.expt, []int64{f.item, f.item}, 1)) +} + +// A private wrapper waits for the real lock's renewal goroutine to release its +// holder before test clients are closed. Production lock behavior stays real. +type filterTestRedis struct { + infraredis.Cmdable + mu sync.Mutex + unlocks map[string]int + changed chan struct{} +} + +func (r *filterTestRedis) Eval(ctx context.Context, script string, keys []string, args ...any) *redis.Cmd { + result := r.Cmdable.Eval(ctx, script, keys, args...) + if strings.Contains(script, "redis.call('DEL'") && len(args) > 0 { + r.mu.Lock() + r.unlocks[fmt.Sprint(args[0])]++ + close(r.changed) + r.changed = make(chan struct{}) + r.mu.Unlock() + } + return result +} + +func (r *filterTestRedis) waitUnlocks(holder string, n int) { + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + for { + r.mu.Lock() + count, ch := r.unlocks[holder], r.changed + r.mu.Unlock() + if count >= n { + return + } + select { + case <-ch: + case <-deadline.C: + return + } + } +} + +type filterTestLocker struct { + lock.ILocker + client *filterTestRedis + holder string + releases int +} + +func (l *filterTestLocker) LockWithRenew(ctx context.Context, key string, ttl, hold time.Duration) (bool, context.Context, func(), error) { + ok, child, cancel, err := l.ILocker.LockWithRenew(ctx, key, ttl, hold) + if !ok || err != nil { + return ok, child, cancel, err + } + l.releases += 2 + return ok, child, func() { cancel(); l.client.waitUnlocks(l.holder, l.releases) }, nil +} + +var filterTestHolderSequence atomic.Int64 + +func filterRealTestLocker(client infraredis.Cmdable) lock.ILocker { + observed := &filterTestRedis{Cmdable: client, unlocks: map[string]int{}, changed: make(chan struct{})} + holder := fmt.Sprint("filter-test-", filterTestHolderSequence.Add(1)) + return &filterTestLocker{ILocker: lock.NewRedisLockerWithHolder(observed, holder), client: observed, holder: holder} +} + +type versionCalibrationScanner struct{ repo.IExptItemResultRepo } + +func (*versionCalibrationScanner) ScanManualRetryFilterItems(context.Context, int64, int64, int64, int64, int) ([]int64, int64, error) { + return []int64{10}, 0, nil +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_late_write_mysql_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_late_write_mysql_test.go new file mode 100644 index 0000000000..412dfb2e50 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_late_write_mysql_test.go @@ -0,0 +1,316 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var lateWriteFixtureSeq atomic.Int64 + +type lateWriteFixture struct { + t *testing.T + provider db.Provider + conn *gorm.DB + port repo.IExptCurrentResultRepo + base, space, expt, run, item, version int64 +} + +func (f *lateWriteFixture) experimentRepo() repo.IExperimentRepo { + return experimentrepo.NewExptRepo(experimentmysql.NewExptDAO(f.provider), experimentmysql.NewExptEvaluatorRefDAO(f.provider), nil) +} + +func newLateWriteFixture(t *testing.T, token string) *lateWriteFixture { + t.Helper() + dsn := os.Getenv("EXPT_RETRY_MYSQL_DSN") + if dsn == "" { + t.Skip("EXPT_RETRY_MYSQL_DSN missing; REAL MYSQL NOT EXECUTED") + } + provider, err := db.NewDB(mysql.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + conn := provider.NewSession(context.Background(), db.WithMaster()) + var identity struct{ Version, Schema string } + require.NoError(t, conn.Raw("SELECT VERSION() AS version,DATABASE() AS `schema`").Scan(&identity).Error) + require.Equal(t, "expt_running_retry_test", identity.Schema) + t.Logf("REAL_MYSQL_EXECUTED version=%s schema=%s", identity.Version, identity.Schema) + base := int64(7384603622000000000) + time.Now().UnixNano()%1000000000000 + lateWriteFixtureSeq.Add(1)*10000 + f := &lateWriteFixture{t: t, provider: provider, conn: conn, base: base, space: base + 1, expt: base + 2, run: base + 3, item: base + 11, version: base + 31} + t.Cleanup(func() { + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_item_ref", "expt_run_log", "expt_stats"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + } + require.NoError(t, conn.Exec("DELETE FROM experiment WHERE space_id=? AND id=?", f.space, f.expt).Error) + for _, table := range []string{"eval_target_record", "evaluator_record"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id=?", f.space).Error) + } + sqlDB, err := conn.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + }) + require.NoError(t, conn.Exec("INSERT INTO experiment (id,space_id,latest_run_id,status,expt_type) VALUES (?,?,?,?,?)", f.expt, f.space, f.run, entity.ExptStatus_Processing, entity.ExptType_Offline).Error) + require.NoError(t, conn.Create(&model.ExptRunLog{ID: f.run, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit)), ItemIds: gptr.Of([]byte(`[{"ItemIDs":[],"CreateAt":1700000000}]`))}).Error) + require.NoError(t, conn.Create(&model.ExptStats{ID: base + 4, SpaceID: f.space, ExptID: f.expt, ProcessingCnt: 2, PendingCnt: 1}).Error) + for index, status := range []entity.ItemRunState{entity.ItemRunState_Processing, entity.ItemRunState_Processing, entity.ItemRunState_Queueing} { + item := f.item + int64(index) + require.NoError(t, conn.Create(&model.ExptItemResult{ID: base + 100 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: f.version, Status: int32(status), Ext: gptr.Of([]byte(`{"other":"keep"}`))}).Error) + require.NoError(t, conn.Create(&model.ExptItemResultRunLog{ID: base + 200 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: f.version, Status: int32(status), RetryTimes: 1, ResultState: gptr.Of(int32(entity.ExptItemResultStateDefault))}).Error) + require.NoError(t, conn.Create(&model.ExptItemRef{ID: base + 300 + int64(index), SpaceID: f.space, ExptID: f.expt, ItemID: item, ItemVersionID: f.version}).Error) + } + f.token(token) + refs, _ := json.Marshal(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: base + 51, Alias: "alias", RecordID: base + 61}}, Inline: []*entity.InlineEvalResult{{InlineKey: "inline", RecordID: base + 62}}}) + for index, turn := range []int64{0, base + 22} { + require.NoError(t, conn.Create(&model.ExptTurnResult{ID: base + 400 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: turn, Status: int32(entity.TurnRunState_Processing), TargetResultID: base + 41, WeightedScore: gptr.Of(0.0)}).Error) + require.NoError(t, conn.Create(&model.ExptTurnResultRunLog{ID: base + 500 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: turn, Status: int32(entity.TurnRunState_Processing), TargetResultID: base + 41, EvaluatorResultIds: &refs, Ext: []byte(`{"other":"keep"}`)}).Error) + } + require.NoError(t, conn.Create(&model.ExptTurnEvaluatorResultRef{ID: base + 600, SpaceID: f.space, ExptID: f.expt, ExptTurnResultID: base + 400, EvaluatorVersionID: base + 51, Alias_: "alias", EvaluatorResultID: base + 61}).Error) + require.NoError(t, conn.Exec("INSERT INTO eval_target_record (id,space_id,target_id,target_version_id,experiment_run_id,item_id,item_version_id,turn_id,log_id,trace_id,status) VALUES (?,?,?,?,?,?,?,?,?,?,?)", base+41, f.space, base+42, base+43, f.run, f.item, f.version, 0, "target", "trace", 1).Error) + require.NoError(t, conn.Exec("INSERT INTO evaluator_record (id,space_id,evaluator_version_id,experiment_id,experiment_run_id,item_id,item_version_id,turn_id,trace_id,status,target_record_id) VALUES (?,?,?,?,?,?,?,?,?,?,?)", base+61, f.space, base+51, f.expt, f.run, f.item, f.version, 0, "trace", 1, 0).Error) + port, ok := any(experimentrepo.NewExptItemResultRepo(experimentmysql.NewExptItemResultDAO(provider))).(repo.IExptCurrentResultRepo) + require.True(t, ok, "source conditional execution transaction missing") + f.port = port + return f +} + +func (f *lateWriteFixture) token(token string) { + f.t.Helper() + ext := map[string]string{"other": "keep"} + if token != "" { + raw, err := json.Marshal(&entity.ManualRetryContext{RunID: f.run, RequestID: token, Mode: entity.EvaluationModeFailRetry, Operator: "test", AcceptedAtMS: 1}) + require.NoError(f.t, err) + ext[entity.ManualRetryExtKey] = string(raw) + } + raw, err := json.Marshal(ext) + require.NoError(f.t, err) + require.NoError(f.t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Update("ext", raw).Error) +} + +func (f *lateWriteFixture) source() *entity.ExptCurrentItemRunSource { + f.t.Helper() + s, err := f.port.GetCurrentItemRunSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(f.t, err) + require.NotNil(f.t, s) + return s +} + +func (f *lateWriteFixture) snapshot() map[string]string { + f.t.Helper() + out := map[string]string{} + for _, table := range []string{"experiment", "expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_turn_evaluator_result_ref", "expt_stats", "expt_run_log", "expt_item_ref", "eval_target_record", "evaluator_record"} { + var rows []map[string]any + require.NoError(f.t, f.conn.Table(table).Where("space_id=?", f.space).Order("id").Find(&rows).Error) + raw, err := json.Marshal(rows) + require.NoError(f.t, err) + out[table] = string(raw) + } + return out +} + +func (f *lateWriteFixture) itemRepo() repo.IExptItemResultRepo { + return f.port.(repo.IExptItemResultRepo) +} + +func (f *lateWriteFixture) turnRepo() repo.IExptTurnResultRepo { + return experimentrepo.NewExptTurnResultRepo(nil, experimentmysql.NewExptTurnResultDAO(f.provider), experimentmysql.NewExptTurnEvaluatorResultRefDAO(f.provider)) +} + +func (f *lateWriteFixture) event() *entity.ExptItemEvalEvent { + return &entity.ExptItemEvalEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, EvalSetItemID: f.item, ExptRunMode: entity.EvaluationModeFailRetry, RequestID: "rq1", MaxRetryTimes: 3} +} + +func lateWriteLocker(t *testing.T, ctrl *gomock.Controller) ManualRetryLockerFactory { + t.Helper() + return func() lock.ILocker { + mutex := lockmocks.NewMockILocker(ctrl) + mutex.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), 5*time.Second, gomock.Any()).DoAndReturn(func(ctx context.Context, _ string, _ time.Duration, _ time.Duration) (bool, context.Context, func(), error) { + child, cancel := context.WithCancel(ctx) + return true, child, cancel, nil + }).AnyTimes() + mutex.EXPECT().Unlock(gomock.Any()).Return(true, nil).AnyTimes() + return mutex + } +} + +func TestLateWriteMySQLStartThenAdmission(t *testing.T) { + for _, path := range []string{"store", "complete", "yield", "error", "zombie", "sandbox", "cancel_store", "cancel_complete", "source_read_error"} { + t.Run(path, func(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + source := f.source() + event := f.event() + event.RetryTimes = 1 + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: source}) + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + executor := &ExptItemEvalCtxExecutor{ItemResultRepo: f.itemRepo(), TurnResultRepo: f.turnRepo(), Configer: config} + svc := &ExptItemEventEvalServiceImpl{exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), configer: config, itemLocker: lateWriteLocker(t, ctrl)} + if path == "source_read_error" { + before := f.snapshot() + fault := errors.New("source DB unavailable") + hook := "late_write_primary_read_failure" + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register(hook, func(tx *gorm.DB) { _ = tx.AddError(fault) })) + err := svc.HandleEventErr(func(ctx context.Context, _ *entity.ExptItemEvalEvent) error { + executionSource(ctx).Source = source + return fault + })(context.Background(), event) + require.NoError(t, f.conn.Callback().Query().Remove(hook)) + require.True(t, errno.NeedMQRetry(err)) + require.Equal(t, before, f.snapshot()) + return + } + f.token("rq2") + before := f.snapshot() + if path == "cancel_store" || path == "cancel_complete" { + child, cancel := context.WithCancel(ctx) + cancel() + ctx = child + } + switch path { + case "store", "cancel_store": + etec := &entity.ExptTurnEvalCtx{Turn: &entity.Turn{ID: 0}, ExptItemEvalCtx: &entity.ExptItemEvalCtx{Event: event, Expt: &entity.Experiment{ID: f.expt, SpaceID: f.space}, EvalSetItem: &entity.EvaluationSetItem{ItemID: f.item}, ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: source.Turns}}} + err := executor.storeTurnRunResult(ctx, etec, &entity.ExptTurnRunResult{}) + if path == "store" { + require.ErrorIs(t, err, errExptNonCurrentSource) + } else { + require.True(t, errno.NeedMQRetry(err)) + } + case "complete", "cancel_complete": + err := executor.CompleteItemRun(ctx, &entity.ExptItemEvalCtx{Event: event}, nil) + if path == "complete" { + require.ErrorIs(t, err, errExptNonCurrentSource) + } else { + require.True(t, errno.NeedMQRetry(err)) + } + case "yield": + svc.yieldItemRunForRetry(ctx, event, errors.New("old error")) + case "error": + require.NoError(t, svc.HandleEventErr(func(ctx context.Context, _ *entity.ExptItemEvalEvent) error { + executionSource(ctx).Source = source + return errors.New("old error") + })(context.Background(), event)) + case "zombie", "sandbox": + config.EXPECT().GetConsumerConf(gomock.Any()).Return(&entity.ExptConsumerConf{ExptExecConf: &entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ZombieSecond: 1}}}).AnyTimes() + scheduler := &ExptSchedulerImpl{ExptItemResultRepo: f.itemRepo(), ExptTurnResultRepo: f.turnRepo(), itemLocker: lateWriteLocker(t, ctrl), Configer: config} + old := time.Now().Add(-time.Hour) + items := []*entity.ExptEvalItem{{Source: source, ItemID: f.item, ExptID: f.expt, State: entity.ItemRunState_Processing, UpdatedAt: &old, RequestID: "rq1"}} + if path == "zombie" { + _, completed, err := scheduler.handleZombies(ctx, &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run}, items, nil) + require.NoError(t, err) + require.Empty(t, completed) + } else { + _, completed, err := scheduler.sweepCurrentSandboxItems(ctx, &entity.ExptScheduleEvent{SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run}, items, &entity.Experiment{}) + require.NoError(t, err) + require.Empty(t, completed) + } + } + require.Equal(t, before, f.snapshot(), "old processing source cannot mutate any of eleven real tables") + }) + } +} + +func TestLateWriteMySQLManualBudget(t *testing.T) { + for _, spent := range []int32{1, 3} { + for _, yield := range []bool{false, true} { + t.Run(fmt.Sprintf("%d_of_3_yield_%v", spent, yield), func(t *testing.T) { + f := newLateWriteFixture(t, "rq1") + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("retry_times", spent).Error) + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + config.EXPECT().GetErrRetryConf(gomock.Any(), f.space, gomock.Any()).Return(&entity.RetryConf{RetryTimes: 3}).AnyTimes() + expectedPublish := int(3 - spent) + if yield { + expectedPublish = 0 + } + published := 0 + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + published++ + require.Equal(t, int(spent)+published, event.RetryTimes) + require.Equal(t, "rq1", event.RequestID) + return nil + }).Times(expectedPublish) + svc := &ExptItemEventEvalServiceImpl{experimentRepo: f.experimentRepo(), exptItemResultRepo: f.itemRepo(), exptTurnResultRepo: f.turnRepo(), configer: config, publisher: publisher, itemLocker: lateWriteLocker(t, ctrl)} + event := f.event() + event.RetryTimes = 0 + if yield { + event.Ext = map[string]string{entity.RetryYieldExtKey: "true"} + } + opportunities := 0 + for { + source := f.source() + if entity.IsItemRunFinished(source.Current.Status) { + break + } + if source.Current.Status == entity.ItemRunState_Queueing { + advanced, err := f.itemRepo().MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, f.item, "rq1") + require.NoError(t, err) + require.True(t, advanced) + } + opportunities++ + handler := svc.HandleEventErr(svc.HandleCurrentSource(func(ctx context.Context, event *entity.ExptItemEvalEvent) error { + return errors.New("ordinary retryable provider error") + })) + require.NoError(t, handler(context.Background(), event)) + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&log).Error) + require.GreaterOrEqual(t, log.RetryTimes, spent) + require.LessOrEqual(t, log.RetryTimes, int32(3)) + event.RetryTimes = int(log.RetryTimes) + require.LessOrEqual(t, opportunities, 3) + } + require.Equal(t, 1+int(3-spent), opportunities) + require.Equal(t, expectedPublish, published) + source := f.source() + require.Equal(t, int32(3), source.Current.RetryTimes) + require.Equal(t, entity.ItemRunState_Fail, source.Current.Status) + var b, c model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item+1).Take(&b).Error) + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item+2).Take(&c).Error) + require.Equal(t, int32(entity.ItemRunState_Processing), b.Status) + require.Equal(t, int32(entity.ItemRunState_Queueing), c.Status) + }) + } + } +} + +func TestLateWriteMySQLStoreThenCompleteEmptyRefs(t *testing.T) { + f := newLateWriteFixture(t, "") + source := f.source() + event := f.event() + event.RequestID = "" + event.ExptRunMode = entity.EvaluationModeSubmit + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: source}) + executor := &ExptItemEvalCtxExecutor{ItemResultRepo: f.itemRepo(), TurnResultRepo: f.turnRepo()} + for _, id := range []int64{0, f.base + 22} { + etec := &entity.ExptTurnEvalCtx{Turn: &entity.Turn{ID: id}, ExptItemEvalCtx: &entity.ExptItemEvalCtx{Event: event, Expt: &entity.Experiment{ID: f.expt, SpaceID: f.space}, EvalSetItem: &entity.EvaluationSetItem{ItemID: f.item}, ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: source.Turns}}} + require.NoError(t, executor.storeTurnRunResult(ctx, etec, &entity.ExptTurnRunResult{})) + } + require.NoError(t, executor.CompleteItemRun(ctx, &entity.ExptItemEvalCtx{Event: event}, nil)) + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&log).Error) + require.Equal(t, int32(entity.ItemRunState_Success), log.Status) + require.Equal(t, int32(entity.ExptItemResultStateLogged), gptr.Indirect(log.ResultState)) +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_late_write_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_late_write_test.go new file mode 100644 index 0000000000..ce4acd2184 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_late_write_test.go @@ -0,0 +1,141 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "strconv" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// Removing the source check would let the endpoint confirm quota or call providers. +func TestLateWriteConsumerOriginalResume(t *testing.T) { + for _, kind := range []string{"target_stale", "evaluator_stale", "reader_failure", "turn0", "builtin0", "inline_stale", "alias_stale", "oldmap_stale", "missing_legacy", "incomplete_record", "unknown_record"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + records := svcmocks.NewMockEvaluatorRecordService(ctrl) + target := svcmocks.NewMockIEvalTargetService(ctrl) + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, ExptRunMode: entity.EvaluationModeSubmit, RequestID: "rq1"} + source := &entity.ExptItemCurrentRequest{SpaceID: 3, ExptID: 1, RunID: 2, ItemID: 11, ItemVersionID: 31, Mode: entity.EvaluationModeSubmit, RequestID: "rq1", Status: entity.ItemRunState_Processing} + items.EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(11)).Return(source, nil) + log := &entity.ExptTurnResultRunLog{SpaceID: 3, ExptID: 1, ExptRunID: 2, ItemID: 11, ItemVersionID: 31, TurnID: 0, TargetResultID: 41, Status: entity.TurnRunState_Processing} + log.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 51, Alias: "a", RecordID: 61}}} + event.ResumeSource = &entity.ExptResumeSource{Kind: "evaluator", TurnID: 0, RecordID: 61, InvokeID: strconv.FormatInt(61, 10)} + event.AsyncEvaluatorReportTrigger = true + record := &entity.EvaluatorRecord{ID: 61, SpaceID: 3, ExperimentID: 1, ExperimentRunID: 2, ItemID: 11, ItemVersionID: 31, TurnID: 0, EvaluatorVersionID: 51, Alias: "a", TargetRecordID: 0, Status: entity.EvaluatorRunStatusSuccess} + var readErr error + switch kind { + case "target_stale", "turn0": + event.ResumeSource.Kind = "target" + event.ResumeSource.RecordID = 41 + event.ResumeSource.InvokeID = "41" + event.AsyncEvaluatorReportTrigger = false + event.AsyncReportTrigger = true + target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(41)).Return(&entity.EvalTargetRecord{ID: 41, SpaceID: 3, ExperimentRunID: 2, ItemID: 11, ItemVersionID: 31, TurnID: 0, Status: gptr.Of(entity.EvalTargetRunStatusSuccess)}, nil).AnyTimes() + if kind == "target_stale" { + log.TargetResultID = 42 + } + case "evaluator_stale": + log.EvaluatorResultIds.Registered[0].RecordID = 62 + case "inline_stale": + record.SourceType = entity.EvaluatorRecordSourceTypeInline + record.InlineKey = "a" + record.TargetRecordID = 41 + log.EvaluatorResultIds = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: "a", RecordID: 62}}} + case "alias_stale": + log.EvaluatorResultIds.Registered[0].Alias = "b" + case "oldmap_stale": + record.Alias = "" + log.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 62}} + case "reader_failure": + readErr = errors.New("primary source unavailable") + case "incomplete_record": + record.ExperimentRunID = 0 + case "unknown_record": + record.Status = entity.EvaluatorRunStatusUnknown + case "missing_legacy": + event.ResumeSource = nil + event.RequestID = "" + source.RequestID = "" + } + turns.EXPECT().GetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), int64(11), int64(3)).Return([]*entity.ExptTurnResultRunLog{log}, readErr).AnyTimes() + records.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(61), false).Return(record, nil).AnyTimes() + svc := &ExptItemEventEvalServiceImpl{exptItemResultRepo: items, exptTurnResultRepo: turns, evaluatorRecordService: records, evaTargetService: target} + called := false + err := svc.HandleCurrentSource(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(context.Background(), event) + require.Equal(t, kind == "turn0" || kind == "builtin0", called, "noncurrent original invocation must have zero downstream side effects") + if kind == "reader_failure" || kind == "missing_legacy" || kind == "incomplete_record" || kind == "unknown_record" { + require.True(t, errno.NeedMQRetry(err)) + } else { + require.NoError(t, err) + } + }) + } +} + +// Cancelled lock ownership must stop storage and error handling before any fallback writes. +func TestLateWriteCancelledExecution(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + executor := &ExptItemEvalCtxExecutor{ItemResultRepo: items} + event := &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, ExptRunMode: entity.EvaluationModeSubmit} + require.True(t, errno.NeedMQRetry(executor.CompleteItemRun(ctx, &entity.ExptItemEvalCtx{Event: event}, nil))) + svc := &ExptItemEventEvalServiceImpl{exptItemResultRepo: items} + require.True(t, errno.NeedMQRetry(svc.HandleEventErr(func(context.Context, *entity.ExptItemEvalEvent) error { return errors.New("business error") })(ctx, event))) +} + +func TestLateWriteStaleSweepNoSideEffects(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetConsumerConf(gomock.Any()).Return(&entity.ExptConsumerConf{ExptExecConf: &entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ZombieSecond: 1}}}) + old := time.Now().Add(-time.Hour) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + turns.EXPECT().MGetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + svc := &ExptSchedulerImpl{ExptItemResultRepo: items, ExptTurnResultRepo: turns, Configer: config} + // A same-scan original source is mandatory; an absent source must protect, never write. + _, _, err := svc.handleZombies(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []*entity.ExptEvalItem{{ExptID: 1, ItemID: 11, RequestID: "rq1", State: entity.ItemRunState_Processing, UpdatedAt: &old}}, nil) + require.True(t, errno.NeedMQRetry(err)) +} + +func TestLateWriteArmStopsAfterOwnershipCancellation(t *testing.T) { + ctrl := gomock.NewController(t) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluators := svcmocks.NewMockEvaluatorService(ctrl) + ctx, cancel := context.WithCancel(context.Background()) + turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, []*entity.ExptTurnResultRunLog) error { cancel(); return nil }) + evaluators.EXPECT().ArmEvaluatorResume(gomock.Any(), gomock.Any()).Times(0) + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turns, evaluatorService: evaluators} + etec := &entity.ExptTurnEvalCtx{Turn: &entity.Turn{ID: 0}, ExptItemEvalCtx: &entity.ExptItemEvalCtx{Event: &entity.ExptItemEvalEvent{ExptRunID: 2}, Expt: &entity.Experiment{ID: 1, SpaceID: 3}, EvalSetItem: &entity.EvaluationSetItem{ItemID: 11}, ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{0: {ID: 21, TurnID: 0}}}}} + err := executor.storeTurnRunResult(ctx, etec, &entity.ExptTurnRunResult{EvaluatorResults: []*entity.EvaluatorRecord{{ID: 61, EvaluatorVersionID: 51, Status: entity.EvaluatorRunStatusAsyncInvoking}}}) + require.True(t, errno.NeedMQRetry(err)) +} + +func TestLateWriteZombieUsesFrozenRecordIDs(t *testing.T) { + ctrl := gomock.NewController(t) + turns := repomocks.NewMockIExptTurnResultRepo(ctrl) + records := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + // A reader returning a newer invocation must never supply cleanup identity. + turns.EXPECT().MGetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptTurnResultRunLog{{TargetResultID: 42, EvaluatorResultIds: &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 62}}}}, nil).AnyTimes() + records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{61}, false, false).Return(nil, nil) + source := &entity.ExptCurrentItemRunSource{Turns: map[int64]*entity.ExptTurnResultRunLog{0: {ItemID: 11, TargetResultID: 41, EvaluatorResultIds: &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: 61}}}}} + ctx := context.WithValue(context.Background(), currentItemSourceKey{}, ¤tItemExecution{Source: source}) + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turns, EvaluatorRecordRepo: records} + require.NoError(t, svc.terminateZombieEvaluatorRecords(ctx, &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, []int64{11})) +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_manual_continuation_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_manual_continuation_test.go new file mode 100644 index 0000000000..5772017083 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_manual_continuation_test.go @@ -0,0 +1,281 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "encoding/json" + "strconv" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/infra/external/benefit" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/pkg/ctxcache" +) + +// These spies sit at external boundaries. EvalTurns, result loading, stage +// selection and persistence construction remain real, including on a replay. +type manualContinuationCalls struct { + Targets map[int64]int `json:"targets"` + Evaluators map[int64]map[int64]int `json:"evaluators"` + Benefits int `json:"benefits"` + Stores map[int64]int `json:"stores"` + mu sync.Mutex + targets map[int64]*entity.EvalTargetRecord + evaluators map[int64]*entity.EvaluatorRecord + sequence int64 +} + +func newManualContinuationCalls(t *testing.T, f *turnResumeFixture) *manualContinuationCalls { + t.Helper() + c := &manualContinuationCalls{ + Targets: map[int64]int{}, Evaluators: map[int64]map[int64]int{}, Stores: map[int64]int{}, + targets: map[int64]*entity.EvalTargetRecord{}, evaluators: map[int64]*entity.EvaluatorRecord{}, sequence: 10000, + } + f.item.Event.Ext = map[string]string{entity.ManualRetryExtKey: `{"run_id":103,"request_id":"RQ1","mode":5,"operator":"u","accepted_at_ms":1}`} + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, id int64) (*entity.EvalTargetRecord, error) { + return c.targets[id], nil + }).AnyTimes() + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), false, false).DoAndReturn(func(_ context.Context, ids []int64, _, _ bool) ([]*entity.EvaluatorRecord, error) { + rows := make([]*entity.EvaluatorRecord, 0, len(ids)) + for _, id := range ids { + require.Contains(t, c.evaluators, id, "fixture must contain every persisted ref") + rows = append(rows, c.evaluators[id]) + } + return rows, nil + }).AnyTimes() + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, exec *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.Targets[exec.TurnID]++ + c.sequence++ + record := c.target(c.sequence, exec.TurnID) + c.targets[record.ID] = record + assert.NotContains(t, input.Ext, entity.ManualRetryExtKey) + assert.Len(t, input.HistoryMessages, map[int64]int{0: 0, 20: 1, 30: 2}[exec.TurnID]) + return record, nil + }).AnyTimes() + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).AnyTimes() + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.Evaluators[req.TurnID] == nil { + c.Evaluators[req.TurnID] = map[int64]int{} + } + c.Evaluators[req.TurnID][req.EvaluatorVersionID]++ + c.sequence++ + record := resumeEvaluator(c.sequence, req.EvaluatorVersionID, req.TurnID, req.Alias) + record.Ext = map[string]string{"request_id": "RQ1"} + c.evaluators[record.ID] = record + assert.NotContains(t, req.Ext, entity.ManualRetryExtKey) + return record, nil + }).AnyTimes() + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ *benefit.CheckAndDeductEvalBenefitParams) (*benefit.CheckAndDeductEvalBenefitResult, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.Benefits++ + return &benefit.CheckAndDeductEvalBenefitResult{}, nil + }).AnyTimes() + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + require.Len(t, rows, 1) + row := rows[0] + c.Stores[row.TurnID]++ + assert.Equal(t, entity.TurnRunState_Success, row.Status) + f.item.ExistItemEvalResult.TurnResultRunLogs[row.TurnID] = row + return nil + }).AnyTimes() + return c +} + +func (c *manualContinuationCalls) target(id, turnID int64) *entity.EvalTargetRecord { + record := resumeTarget(id, turnID) + record.SpaceID, record.ExperimentRunID, record.ItemID = 3, 103, 10 + record.Ext = map[string]string{"request_id": "RQ1"} + return record +} + +func (c *manualContinuationCalls) prepare(f *turnResumeFixture, turnID int64, status entity.TurnRunState, failedE2 bool) { + log := f.item.ExistItemEvalResult.TurnResultRunLogs[turnID] + log.Status, log.TargetResultID = status, 201+turnID + log.Ext = f.item.Event.Ext + c.targets[log.TargetResultID] = c.target(log.TargetResultID, turnID) + log.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{ + {VersionID: 401, RecordID: 301 + turnID}, {VersionID: 402, RecordID: 302 + turnID}, + }} + c.evaluators[301+turnID] = resumeEvaluator(301+turnID, 401, turnID, "") + c.evaluators[301+turnID].Ext = map[string]string{"request_id": "RQ1"} + e2 := resumeEvaluator(302+turnID, 402, turnID, "") + e2.Ext = map[string]string{"request_id": "RQ1"} + if failedE2 { + e2.Status = entity.EvaluatorRunStatusFail + } + c.evaluators[e2.ID] = e2 +} + +func setManualContinuationSource(f *turnResumeFixture, kind string, turnID int64) { + recordID := 201 + turnID + if kind == "evaluator" { + recordID = 302 + turnID + } + f.item.Event.AsyncReportTrigger = kind == "target" + f.item.Event.AsyncEvaluatorReportTrigger = kind == "evaluator" + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: kind, TurnID: turnID, RecordID: recordID, InvokeID: strconv.FormatInt(recordID, 10)} +} + +func (c *manualContinuationCalls) log(t *testing.T, phase string) { + t.Helper() + raw, err := json.Marshal(c) + require.NoError(t, err) + t.Logf("actual_call_counts %s %s", phase, raw) +} + +// Catches dropping ResumeSource on a non-source clone: RQ1's completed turn +// would be treated as another user-confirmed manual invocation and billed again. +func TestRunningRetryManualContinuation_EvalTurnsSource20DoesNotRerunHistory(t *testing.T) { + for _, kind := range []string{"target", "evaluator"} { + t.Run(kind, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0, 20, 30) + calls := newManualContinuationCalls(t, f) + calls.prepare(f, 0, entity.TurnRunState_Success, false) + calls.prepare(f, 20, entity.TurnRunState_Processing, false) + setManualContinuationSource(f, kind, 20) + originalSource := *f.item.Event.ResumeSource + first := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + for _, phase := range []string{"resume", "duplicate callback"} { + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + calls.log(t, phase) + assert.Equal(t, map[int64]int{30: 1}, calls.Targets) + assert.Equal(t, map[int64]map[int64]int{30: {401: 1, 402: 1}}, calls.Evaluators) + assert.Equal(t, 2, calls.Benefits) + assert.Equal(t, map[int64]int{20: 1, 30: 1}, calls.Stores) + assert.Same(t, first, f.item.ExistItemEvalResult.TurnResultRunLogs[0], "completed history must not be saved again") + assert.Equal(t, originalSource, *f.item.Event.ResumeSource) + } + }) + } +} + +// Catches the same mode confusion for partial results in the next turn. The +// source flags do not apply there, but RQ1's successful T/E1 must still be reused. +func TestRunningRetryManualContinuation_EvalTurnsSource0OnlyInvokesNextPendingE2(t *testing.T) { + for _, kind := range []string{"target", "evaluator"} { + for _, format := range []string{"registered", "alias", "legacy", "inline"} { + t.Run(kind+"/"+format, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0, 20) + calls := newManualContinuationCalls(t, f) + calls.prepare(f, 0, entity.TurnRunState_Processing, false) + calls.prepare(f, 20, entity.TurnRunState_Fail, true) + setManualContinuationSource(f, kind, 0) + next := f.item.ExistItemEvalResult.TurnResultRunLogs[20] + switch format { + case "alias": + f.item.Expt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + f.item.ItemConfig = &entity.ExptItemConfig{EvaluatorConfs: []*entity.ItemEvaluatorConf{{EvaluatorVersionID: 401, Alias: "a"}, {EvaluatorVersionID: 402, Alias: "b"}}} + for _, turnID := range []int64{0, 20} { + refs := f.item.ExistItemEvalResult.TurnResultRunLogs[turnID].EvaluatorResultIds.Registered + refs[0].Alias, refs[1].Alias = "a", "b" + calls.evaluators[301+turnID].Alias, calls.evaluators[302+turnID].Alias = "a", "b" + } + case "legacy": + next.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{401: 321, 402: 322}} + case "inline": + inline := resumeEvaluator(324, 0, 20, "") + inline.SourceType, inline.InlineKey, inline.TargetRecordID = entity.EvaluatorRecordSourceTypeInline, "quality", 221 + calls.evaluators[324] = inline + next.EvaluatorResultIds.Inline = []*entity.InlineEvalResult{{InlineKey: "quality", RecordID: 324}} + } + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + calls.log(t, "resume") + assert.Empty(t, calls.Targets) + assert.Equal(t, map[int64]map[int64]int{20: {402: 1}}, calls.Evaluators) + assert.Equal(t, 1, calls.Benefits) + assert.Equal(t, map[int64]int{0: 1, 20: 1}, calls.Stores) + saved := f.item.ExistItemEvalResult.TurnResultRunLogs[20] + assert.Equal(t, int64(221), saved.TargetResultID) + require.Len(t, saved.EvaluatorResultIds.Registered, 2) + assert.Equal(t, int64(321), saved.EvaluatorResultIds.Registered[0].RecordID) + assert.NotEqual(t, int64(322), saved.EvaluatorResultIds.Registered[1].RecordID) + if format == "inline" { + require.Len(t, saved.EvaluatorResultIds.Inline, 1) + assert.Equal(t, int64(324), saved.EvaluatorResultIds.Inline[0].RecordID) + } + }) + } + } +} + +func TestRunningRetryManualContinuation_FirstUserSelectionExecutesAllTurns(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0, 20, 30) + calls := newManualContinuationCalls(t, f) + // Admission clears every old ref/status. No callback metadata marks this + // first user-confirmed execution, and every frozen turn must be newly invoked. + require.Nil(t, f.item.Event.ResumeSource) + require.False(t, f.item.Event.AsyncReportTrigger) + require.False(t, f.item.Event.AsyncEvaluatorReportTrigger) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + calls.log(t, "first selection") + assert.Equal(t, map[int64]int{0: 1, 20: 1, 30: 1}, calls.Targets) + assert.Equal(t, map[int64]map[int64]int{0: {401: 1, 402: 1}, 20: {401: 1, 402: 1}, 30: {401: 1, 402: 1}}, calls.Evaluators) + assert.Equal(t, 6, calls.Benefits) + assert.Equal(t, map[int64]int{0: 1, 20: 1, 30: 1}, calls.Stores) +} + +// Retaining source identity must not re-clone between CallTarget and +// CallEvaluators: a genuinely new target invalidates that turn's old evaluators. +func TestRunningRetryManualContinuation_EvalTurnsNewTargetKeepsCalledMarker(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0, 20) + calls := newManualContinuationCalls(t, f) + calls.prepare(f, 0, entity.TurnRunState_Processing, false) + calls.prepare(f, 20, entity.TurnRunState_Fail, false) + setManualContinuationSource(f, "target", 0) + target := calls.targets[221] + *target.Status = entity.EvalTargetRunStatusFail + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + calls.log(t, "new target") + assert.Equal(t, map[int64]int{20: 1}, calls.Targets) + assert.Equal(t, map[int64]map[int64]int{20: {401: 1, 402: 1}}, calls.Evaluators) + assert.Equal(t, 2, calls.Benefits) + assert.Equal(t, map[int64]int{0: 1, 20: 1}, calls.Stores) + saved := f.item.ExistItemEvalResult.TurnResultRunLogs[20] + assert.NotEqual(t, int64(221), saved.TargetResultID) + require.Len(t, saved.EvaluatorResultIds.Registered, 2) + for _, ref := range saved.EvaluatorResultIds.Registered { + assert.NotContains(t, []int64{321, 322}, ref.RecordID) + } +} + +func TestRunningRetryManualContinuation_LegacyCallbackWithoutSourceKeepsRefs(t *testing.T) { + for _, kind := range []string{"target", "evaluator"} { + t.Run(kind, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0) + calls := newManualContinuationCalls(t, f) + calls.prepare(f, 0, entity.TurnRunState_Processing, false) + f.item.Event.AsyncReportTrigger = kind == "target" + f.item.Event.AsyncEvaluatorReportTrigger = kind == "evaluator" + require.Nil(t, f.item.Event.ResumeSource) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + calls.log(t, "legacy callback") + assert.Empty(t, calls.Targets) + assert.Empty(t, calls.Evaluators) + assert.Zero(t, calls.Benefits) + assert.Equal(t, map[int64]int{0: 1}, calls.Stores) + assert.Equal(t, int64(201), f.item.ExistItemEvalResult.TurnResultRunLogs[0].TargetResultID) + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_resume_source_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_resume_source_test.go new file mode 100644 index 0000000000..358f577eab --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_resume_source_test.go @@ -0,0 +1,240 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "sync" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + datasetdto "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/data/dataset" + datasetentity "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/data/domain/dataset" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/rpc" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + datasetadapter "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/rpc/data" + datasetmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/rpc/data/mocks" +) + +func TestAsyncSourceFixArmTurn0(t *testing.T) { + ctrl := gomock.NewController(t) + client := datasetmocks.NewMockClient(ctrl) + client.EXPECT().BatchGetDatasetItems(gomock.Any(), gomock.Any()).Return(&datasetdto.BatchGetDatasetItemsResponse{Items: []*datasetentity.DatasetItem{{ItemID: gptr.Of(int64(4)), Data: []*datasetentity.FieldData{{Name: gptr.Of("input"), Content: gptr.Of("single turn")}}}}}, nil) + items, err := datasetadapter.NewDatasetRPCAdapter(client).BatchGetDatasetItems(context.Background(), &rpc.BatchGetDatasetItemsParam{}) + require.NoError(t, err) + require.Len(t, items, 1) + require.Len(t, items[0].Turns, 1) + turn := items[0].Turns[0] + require.Zero(t, turn.ID) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + original := &entity.ExptItemEvalEvent{ExptID: 2, ExptRunID: 3, EvalSetItemID: turn.ItemID, RequestID: "original-rq"} + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: original, ResumeReady: true}, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, TurnID: turn.ID, Status: entity.EvaluatorRunStatusSuccess}, nil) + published := 0 + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, event *entity.ExptItemEvalEvent, _ *time.Duration, _ func(*entity.ExptItemEvalEvent)) error { + require.NotNil(t, event.ResumeSource) + assert.Zero(t, event.ResumeSource.TurnID) + assert.Equal(t, int64(100), event.ResumeSource.RecordID) + assert.Equal(t, "100", event.ResumeSource.InvokeID) + assert.Equal(t, "evaluator", event.ResumeSource.Kind) + assert.Equal(t, original.RequestID, event.RequestID) + bytes, err := json.Marshal(event) + require.NoError(t, err) + var roundtrip entity.ExptItemEvalEvent + require.NoError(t, json.Unmarshal(bytes, &roundtrip)) + require.NotNil(t, roundtrip.ResumeSource) + assert.Zero(t, roundtrip.ResumeSource.TurnID) + published++ + return nil + }).MaxTimes(1) + svc := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo, exptEventPublisher: publisher} + require.NoError(t, svc.ArmEvaluatorResume(context.Background(), 100)) + assert.Equal(t, 1, published) +} + +func TestAsyncSourceFixArmInvalidSource(t *testing.T) { + for _, source := range []string{"negative turn", "wrong record"} { + t.Run(source, func(t *testing.T) { + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{RequestID: "original-rq"}, ResumeReady: true}, nil) + record := &entity.EvaluatorRecord{ID: 100, TurnID: 0, Status: entity.EvaluatorRunStatusSuccess} + if source == "negative turn" { + record.TurnID = -1 + } else { + record.ID = 200 + } + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(record, nil) + svc := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo, exptEventPublisher: publisher} + require.Error(t, svc.ArmEvaluatorResume(context.Background(), 100)) + }) + } +} + +func TestAsyncSourceArmEvaluatorResume(t *testing.T) { + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + const recordID, turnID = int64(7384603622000000301), int64(7384603622000000302) + original := &entity.ExptItemEvalEvent{ExptID: 2, ExptRunID: 3, EvalSetItemID: 4, RequestID: "original-rq", Ext: map[string]string{"inputExpr": "original"}, Session: &entity.Session{UserID: "owner"}} + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:"+strconv.FormatInt(recordID, 10)).Return(&entity.EvalAsyncCtx{Event: original, ResumeReady: true}, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), recordID, false).Return(&entity.EvaluatorRecord{ID: recordID, TurnID: turnID, Status: entity.EvaluatorRunStatusSuccess}, nil) + var events []*entity.ExptItemEvalEvent + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, ev *entity.ExptItemEvalEvent, _ *time.Duration, modify func(*entity.ExptItemEvalEvent)) error { + assert.NotSame(t, original, ev) + assert.Equal(t, "original-rq", ev.RequestID) + assert.Equal(t, "original", ev.Ext["inputExpr"]) + assert.Equal(t, "owner", ev.Session.UserID) + encoded, err := json.Marshal(ev) + require.NoError(t, err) + var wire struct { + ResumeSource *struct { + Kind string + TurnID int64 + InvokeID string + RecordID int64 + } + } + require.NoError(t, json.Unmarshal(encoded, &wire)) + require.NotNil(t, wire.ResumeSource, "Arm must publish the original evaluator record source") + assert.Equal(t, "evaluator", wire.ResumeSource.Kind) + assert.Equal(t, turnID, wire.ResumeSource.TurnID) + assert.Equal(t, recordID, wire.ResumeSource.RecordID) + assert.Equal(t, strconv.FormatInt(recordID, 10), wire.ResumeSource.InvokeID) + modify(ev) + ev.Ext["inputExpr"] = "mutation" + ev.Session.UserID = "mutation" + events = append(events, ev) + if len(events) == 1 { + ev.ResumeSource.TurnID = 99 + return errors.New("publish retry") + } + return nil + }).Times(2) + s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo, exptEventPublisher: publisher} + require.NoError(t, s.ArmEvaluatorResume(context.Background(), recordID)) + require.Len(t, events, 2) + assert.NotSame(t, events[0], events[1]) + assert.False(t, original.AsyncEvaluatorReportTrigger) + assert.Equal(t, "original", original.Ext["inputExpr"]) + assert.Equal(t, "owner", original.Session.UserID) +} + +// Mutating one stored context must not change the item event or another evaluator's context. +func TestAsyncSourceEvaluatorContextCopies(t *testing.T) { + ctrl := gomock.NewController(t) + metric := metricsmocks.NewMockExptMetric(ctrl) + svc := svcmocks.NewMockEvaluatorService(ctrl) + s := &DefaultExptTurnEvaluationImpl{metric: metric, evaluatorService: svc} + original := &entity.ExptItemEvalEvent{SpaceID: 1, ExptID: 2, ExptRunID: 3, EvalSetItemID: 4, RequestID: "original-rq", Ext: map[string]string{"inputExpr": "original"}, Session: &entity.Session{UserID: "owner"}} + // Decode optional metadata so this test compiles before the new field exists. + require.NoError(t, json.Unmarshal([]byte(`{"ResumeSource":{"Kind":"target","TurnID":7384603622000000401,"RecordID":7384603622000000402,"InvokeID":"7384603622000000402"}}`), original)) + etec := &entity.ExptTurnEvalCtx{ExptItemEvalCtx: &entity.ExptItemEvalCtx{Event: original, EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}}, Turn: &entity.Turn{ID: 5}, Ext: map[string]string{"inputExpr": "eval expression"}} + ev := &entity.Evaluator{EvaluatorType: entity.EvaluatorTypeAgent, AgentEvaluatorVersion: &entity.AgentEvaluatorVersion{ID: 6}} + input := &entity.EvaluatorInputData{Ext: map[string]string{"inputExpr": "input expression"}} + metric.EXPECT().EmitTurnExecEvaluatorResult(int64(1), false).Times(2) + var saved []*entity.EvalAsyncCtx + svc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + require.NotNil(t, req.AsyncCtx) + assert.NotSame(t, original, req.AsyncCtx.Event) + assert.NotSame(t, original.Session, req.AsyncCtx.Session) + assert.NotSame(t, req.AsyncCtx.Session, req.AsyncCtx.Event.Session) + assert.Same(t, input, req.InputData) + assert.Equal(t, "eval expression", req.Ext["inputExpr"]) + saved = append(saved, req.AsyncCtx) + return &entity.EvaluatorRecord{ID: int64(len(saved)), TurnID: 5}, nil + }).Times(2) + collector := &evalRecordCollector{} + require.NoError(t, s.asyncCallEvaluator(context.Background(), ev, &entity.EvaluatorConf{}, etec, input, collector)) + require.NoError(t, s.asyncCallEvaluatorWithAlias(context.Background(), ev, nil, "alias", etec, input, collector)) + require.Len(t, saved, 2) + saved[0].Event.Ext["inputExpr"] = "mutation" + saved[0].Event.Session.UserID = "mutation" + saved[0].Session.UserID = "async mutation" + saved[0].Event.AsyncEvaluatorReportTrigger = true + require.NotNil(t, saved[0].Event.ResumeSource) + saved[0].Event.ResumeSource.TurnID = 99 + assert.Equal(t, int64(7384603622000000401), saved[1].Event.ResumeSource.TurnID) + assert.Equal(t, int64(7384603622000000401), original.ResumeSource.TurnID) + assert.Equal(t, "original", saved[1].Event.Ext["inputExpr"]) + assert.Equal(t, "owner", saved[1].Session.UserID) + assert.Equal(t, "owner", saved[1].Event.Session.UserID) + assert.False(t, saved[1].Event.AsyncEvaluatorReportTrigger) + assert.Equal(t, "original", original.Ext["inputExpr"]) + assert.Equal(t, "owner", original.Session.UserID) + assert.False(t, original.AsyncEvaluatorReportTrigger) +} + +func TestAsyncSourceContextRoundTripAndConcurrentCopies(t *testing.T) { + const recordID, turnID = int64(7384603622000000402), int64(7384603622000000401) + original := &entity.EvalAsyncCtx{ + Event: &entity.ExptItemEvalEvent{SpaceID: 1, ExptID: 2, ExptRunID: 3, EvalSetItemID: 4, ExptRunMode: entity.ExptRunMode(2), RetryTimes: 2, MaxRetryTimes: 3, CreateAt: 123, Ext: map[string]string{"inputExpr": "expression", "manual_retry": `{"request_id":"rq"}`}, Session: &entity.Session{UserID: "owner", AppID: 6}, ResumeSource: &entity.ExptResumeSource{Kind: "target", TurnID: turnID, RecordID: recordID, InvokeID: strconv.FormatInt(recordID, 10)}}, + RecordID: recordID, AsyncUnixMS: 1234, Session: &entity.Session{UserID: "reporter"}, EnableExtractTrajectory: new(bool), ResumeReady: true, CallbackURL: "callback", Callee: "callee", EvaluatorVersionID: 7, TargetID: 8, DatasetID: 9, DatasetVersionID: 10, ItemKey: "item", DatasetKey: "dataset", AgentName: "agent", ApplicationID: "application", + } + encoded, err := json.Marshal(original) + require.NoError(t, err) + var roundtrip entity.EvalAsyncCtx + require.NoError(t, json.Unmarshal(encoded, &roundtrip)) + assert.Equal(t, original, &roundtrip, "all existing fields and 19-digit IDs survive AsyncCtx JSON") + var old entity.EvalAsyncCtx + require.NoError(t, json.Unmarshal([]byte(`{"Event":{"ExptID":2,"ExptRunID":3,"EvalSetItemID":4},"RecordID":7384603622000000402}`), &old)) + assert.Nil(t, old.Event.ResumeSource) + assert.Empty(t, old.Event.RequestID) + oldBytes, err := json.Marshal(&old) + require.NoError(t, err) + assert.NotContains(t, string(oldBytes), "ResumeSource") + assert.NotContains(t, string(oldBytes), "RequestID") + var nilEvent *entity.ExptItemEvalEvent + var nilAsync *entity.EvalAsyncCtx + assert.Nil(t, nilEvent.Clone()) + assert.Nil(t, nilAsync.Clone()) + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + copy := original.Clone() + copy.Event.Ext["inputExpr"] = "mutation" + copy.Event.Session.UserID = "mutation" + copy.Session.UserID = "mutation" + copy.Event.ResumeSource.TurnID = 99 + *copy.EnableExtractTrajectory = true + }() + } + wg.Wait() + assert.Equal(t, "expression", original.Event.Ext["inputExpr"]) + assert.Equal(t, "owner", original.Event.Session.UserID) + assert.Equal(t, "reporter", original.Session.UserID) + assert.Equal(t, turnID, original.Event.ResumeSource.TurnID) + assert.False(t, *original.EnableExtractTrajectory) +} + +func assertIndependentTargetAsyncContext(t *testing.T, original *entity.ExptItemEvalEvent, actx *entity.EvalAsyncCtx) { + t.Helper() + assert.NotSame(t, original, actx.Event) + assert.NotSame(t, original.Session, actx.Session) + assert.NotSame(t, actx.Event.Session, actx.Session) + actx.Event.AsyncReportTrigger = true + actx.Event.Session.UserID = "mutation" + assert.False(t, original.AsyncReportTrigger) + assert.Equal(t, "user", original.Session.UserID) + assert.Equal(t, "user", actx.Session.UserID) +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_run_writer_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_run_writer_test.go new file mode 100644 index 0000000000..4d045dc0f4 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_run_writer_test.go @@ -0,0 +1,133 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/infra/db" + lockMocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repoMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +const writerReceiptJSON = `[{"ItemIDs":[11],"CreateAt":1700000000},{"ItemIDs":null,"CreateAt":1700000001,"manual_retry_receipt":{"request_id":"rq1","payload_hash":"hash","accepted_count":1,"rejected_items":[{"item_id":12,"reason":"busy"}],"snapshot_at_ms":1700000000123},"future":{"id":9223372036854775806}}]` + +func newReceiptWriterManager(t *testing.T) (*ExptMangerImpl, db.Provider) { + t.Helper() + ctrl := gomock.NewController(t) + mgr := newTestExptManager(ctrl) + provider := db.NewTestDB(t, &model.ExptRunLog{}, &model.Experiment{}, &model.ExptStats{}) + require.NotNil(t, provider) + ctx := context.Background() + require.NoError(t, provider.NewSession(ctx).Create(&model.Experiment{ID: 7, SpaceID: 9, LatestRunID: 8, Status: int32(entity.ExptStatus_Processing)}).Error) + require.NoError(t, provider.NewSession(ctx).Create(&model.ExptStats{ID: 10, SpaceID: 9, ExptID: 7, SuccessCnt: 2}).Error) + require.NoError(t, provider.NewSession(ctx).Create(&model.ExptRunLog{ + ID: 8, ExptID: 7, ExptRunID: 8, SpaceID: 9, + ItemIds: gptr.Of([]byte(`[{"ItemIDs":[11],"CreateAt":1700000000}]`)), Status: gptr.Of(int64(entity.ExptStatus_Processing)), + }).Error) + mgr.runLogRepo = experiment.NewExptRunLogRepo(mysql.NewExptRunLogDAO(provider)) + return mgr, provider +} + +func TestRecordExptDataDoesNotReopenFinishedRun(t *testing.T) { + mgr, provider := newReceiptWriterManager(t) + ctx := context.Background() + mgr.turnResultRepo.(*repoMocks.MockIExptTurnResultRepo).EXPECT().ListTurnResult(ctx, int64(9), int64(7), nil, gomock.Any(), false). + DoAndReturn(func(context.Context, int64, int64, *entity.ExptTurnResultFilter, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + injectWriterReceipt(t, provider) + require.NoError(t, provider.NewSession(ctx).Model(&model.ExptRunLog{}).Where("id = ?", 8).Updates(map[string]any{"status": int64(entity.ExptStatus_Terminated), "success_cnt": 2}).Error) + return []*entity.ExptTurnResult{{Status: int32(entity.TurnRunState_Success)}}, int64(1), nil + }) + mgr.exptResultService.(*svcMocks.MockExptResultService).EXPECT().CalculateStats(ctx, int64(7), int64(9), gomock.Any()).Return(&entity.ExptCalculateStats{SuccessItemCnt: 1}, nil).AnyTimes() + mgr.statsRepo.(*repoMocks.MockIExptStatsRepo).EXPECT().UpdateByExptID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + require.NoError(t, mgr.RecordExptData(ctx, 7, 8, 9, &entity.Session{})) + var row model.ExptRunLog + require.NoError(t, provider.NewSession(ctx).First(&row, 8).Error) + require.Equal(t, int64(entity.ExptStatus_Terminated), gptr.Indirect(row.Status)) + require.Equal(t, int32(2), row.SuccessCnt) + var stats model.ExptStats + require.NoError(t, provider.NewSession(ctx).First(&stats, 10).Error) + require.Equal(t, int32(2), stats.SuccessCnt) + requireWriterReceipt(t, provider) +} + +func injectWriterReceipt(t *testing.T, provider db.Provider) { + t.Helper() + require.NoError(t, provider.NewSession(context.Background()).Model(&model.ExptRunLog{}).Where("id = ?", 8). + UpdateColumn("item_ids", []byte(writerReceiptJSON)).Error) +} + +func requireWriterReceipt(t *testing.T, provider db.Provider) { + t.Helper() + var row model.ExptRunLog + require.NoError(t, provider.NewSession(context.Background()).First(&row, 8).Error) + require.JSONEq(t, writerReceiptJSON, string(gptr.Indirect(row.ItemIds))) + log, err := experiment.NewExptRunLogRepo(mysql.NewExptRunLogDAO(provider)).Get(context.Background(), 7, 8) + require.NoError(t, err) + require.Equal(t, []int64{11}, log.GetItemIDs()) + require.Equal(t, "rq1", log.ItemIds[1].ManualRetryReceipt.RequestID) +} + +func TestCompleteRunKeepsReceiptForIndirectWriters(t *testing.T) { + for _, tc := range []struct { + name string + status entity.ExptStatus + }{ + {"scheduler_complete", entity.ExptStatus_Success}, + {"cancel", entity.ExptStatus_Terminated}, + {"scheduler_outer_error", entity.ExptStatus_Failed}, + {"consumer_credit_termination", entity.ExptStatus_SystemTerminated}, + } { + t.Run(tc.name, func(t *testing.T) { + mgr, provider := newReceiptWriterManager(t) + ctx := context.Background() + mgr.turnResultRepo.(*repoMocks.MockIExptTurnResultRepo).EXPECT().ListTurnResult(ctx, int64(9), int64(7), nil, gomock.Any(), false). + DoAndReturn(func(context.Context, int64, int64, *entity.ExptTurnResultFilter, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + injectWriterReceipt(t, provider) + return []*entity.ExptTurnResult{{Status: int32(entity.TurnRunState_Success)}}, int64(1), nil + }) + mgr.mutex.(*lockMocks.MockILocker).EXPECT().UnlockForce(ctx, mgr.makeExptMutexLockKey(7)).Return(true, nil) + require.NoError(t, mgr.CompleteRun(ctx, 7, 8, 9, &entity.Session{UserID: "u"}, entity.WithStatus(tc.status))) + requireWriterReceipt(t, provider) + }) + } +} + +func TestCompleteRunPersistsBeforeLifecycleUnlock(t *testing.T) { + mgr, provider := newReceiptWriterManager(t) + ctx := context.Background() + mgr.turnResultRepo.(*repoMocks.MockIExptTurnResultRepo).EXPECT().ListTurnResult(ctx, int64(9), int64(7), nil, gomock.Any(), false). + Return([]*entity.ExptTurnResult{}, int64(0), nil) + mgr.mutex.(*lockMocks.MockILocker).EXPECT().UnlockForce(ctx, mgr.makeExptMutexLockKey(7)). + DoAndReturn(func(context.Context, string) (bool, error) { + var row model.ExptRunLog + require.NoError(t, provider.NewSession(ctx).First(&row, 8).Error) + require.Equal(t, int64(entity.ExptStatus_Terminated), gptr.Indirect(row.Status)) + return true, nil + }) + require.NoError(t, mgr.CompleteRun(ctx, 7, 8, 9, &entity.Session{}, entity.WithStatus(entity.ExptStatus_Terminated))) +} + +func TestRecordExptDataKeepsReceipt(t *testing.T) { + mgr, provider := newReceiptWriterManager(t) + ctx := context.Background() + mgr.turnResultRepo.(*repoMocks.MockIExptTurnResultRepo).EXPECT().ListTurnResult(ctx, int64(9), int64(7), nil, gomock.Any(), false). + DoAndReturn(func(context.Context, int64, int64, *entity.ExptTurnResultFilter, entity.Page, bool) ([]*entity.ExptTurnResult, int64, error) { + injectWriterReceipt(t, provider) + return []*entity.ExptTurnResult{{Status: int32(entity.TurnRunState_Success)}}, int64(1), nil + }) + mgr.exptResultService.(*svcMocks.MockExptResultService).EXPECT().CalculateStats(ctx, int64(7), int64(9), gomock.Any()).Return(&entity.ExptCalculateStats{SuccessItemCnt: 1}, nil) + require.NoError(t, mgr.RecordExptData(ctx, 7, 8, 9, &entity.Session{})) + requireWriterReceipt(t, provider) +} diff --git a/backend/modules/evaluation/domain/service/expt_running_retry_turn_resume_test.go b/backend/modules/evaluation/domain/service/expt_running_retry_turn_resume_test.go new file mode 100644 index 0000000000..f97a3db97c --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_running_retry_turn_resume_test.go @@ -0,0 +1,950 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/coze-dev/coze-loop/backend/infra/fileserver" + fsmocks "github.com/coze-dev/coze-loop/backend/infra/fileserver/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + evalstorage "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/storage" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/infra/external/benefit" + benefitmocks "github.com/coze-dev/coze-loop/backend/infra/external/benefit/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" + metricsmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" + configmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/pkg/ctxcache" +) + +// External boundaries only: EvalTurns, context construction, CallTarget and +// CallEvaluators stay real, so extra invokes, deductions or stores fail tests. +type turnResumeFixture struct { + evalCalls atomic.Int64 + executor *ExptItemEvalCtxExecutor + item *entity.ExptItemEvalCtx + target *svcmocks.MockIEvalTargetService + evaluator *svcmocks.MockEvaluatorService + records *svcmocks.MockEvaluatorRecordService + benefit *benefitmocks.MockIBenefitService + turns *repomocks.MockIExptTurnResultRepo + config *configmocks.MockIConfiger + items *repomocks.MockIExptItemResultRepo +} + +func newTurnResumeFixture(t *testing.T, mode entity.ExptRunMode, turnIDs ...int64) *turnResumeFixture { + t.Helper() + ctrl := gomock.NewController(t) + f := &turnResumeFixture{target: svcmocks.NewMockIEvalTargetService(ctrl), evaluator: svcmocks.NewMockEvaluatorService(ctrl), records: svcmocks.NewMockEvaluatorRecordService(ctrl), benefit: benefitmocks.NewMockIBenefitService(ctrl), turns: repomocks.NewMockIExptTurnResultRepo(ctrl), config: configmocks.NewMockIConfiger(ctrl), items: repomocks.NewMockIExptItemResultRepo(ctrl)} + metric := metricsmocks.NewMockExptMetric(ctrl) + metric.EXPECT().EmitTurnExecEval(gomock.Any(), gomock.Any()).Do(func(_, _ int64) { f.evalCalls.Add(1) }).AnyTimes() + metric.EXPECT().EmitTurnExecResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + metric.EXPECT().EmitTurnExecTargetResult(gomock.Any(), gomock.Any()).AnyTimes() + metric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), gomock.Any()).AnyTimes() + f.executor = &ExptItemEvalCtxExecutor{ItemResultRepo: f.items, TurnResultRepo: f.turns, Configer: f.config, Metric: metric, evalTargetService: f.target, evaluatorService: f.evaluator, evaluatorRecordService: f.records, benefitService: f.benefit} + expt := buildRetryFailureSingleSetExpt(3, 401, 402) + expt.Target.EvalTargetType = entity.EvalTargetTypeCustomRPCServer + expt.EvalConf.ConnectorConf.TargetConf.IngressConf = &entity.TargetIngressConf{EvalSetAdapter: &entity.FieldAdapter{}} + f.item = &entity.ExptItemEvalCtx{Expt: expt, Event: &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 103, SpaceID: 3, EvalSetItemID: 10, ExptRunMode: mode, RequestID: "RQ1", Session: &entity.Session{UserID: "u"}, MaxRetryTimes: 3}, EvalSetItem: &entity.EvaluationSetItem{ItemID: 10, BaseInfo: &entity.BaseInfo{CreatedAt: gptr.Of(int64(1))}}, ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{}}} + for _, id := range turnIDs { + f.item.EvalSetItem.Turns = append(f.item.EvalSetItem.Turns, &entity.Turn{ID: id, ItemID: 10}) + f.item.ExistItemEvalResult.TurnResultRunLogs[id] = &entity.ExptTurnResultRunLog{ID: 100 + id, ExptID: 1, ExptRunID: 103, SpaceID: 3, ItemID: 10, TurnID: id, Status: entity.TurnRunState_Processing} + } + f.items.EXPECT().GetItemRunLog(gomock.Any(), int64(1), int64(103), int64(10), int64(3)).Return(&entity.ExptItemResultRunLog{Status: int32(entity.ItemRunState_Processing)}, nil).AnyTimes() + f.items.EXPECT().BatchGet(gomock.Any(), int64(3), int64(1), []int64{10}).Return(nil, nil).AnyTimes() + f.config.EXPECT().BuildEvalExt(gomock.Any(), int64(3), gomock.Any()).Return(nil).AnyTimes() + return f +} + +func resumeTarget(id, turnID int64) *entity.EvalTargetRecord { + return &entity.EvalTargetRecord{ID: id, TurnID: turnID, Status: gptr.Of(entity.EvalTargetRunStatusSuccess), EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: map[string]*entity.Content{consts.OutputSchemaKey: {Text: gptr.Of("previous answer")}}}} +} + +func resumeEvaluator(id, versionID, turnID int64, alias string) *entity.EvaluatorRecord { + return &entity.EvaluatorRecord{ID: id, ExperimentID: 1, ExperimentRunID: 103, SpaceID: 3, ItemID: 10, TurnID: turnID, EvaluatorVersionID: versionID, Alias: alias, SourceType: entity.EvaluatorRecordSourceTypeBuiltin, Status: entity.EvaluatorRunStatusSuccess} +} + +func TestRunningRetryTurnResume_SourceCallbackDoesNotMarkTargetCalled(t *testing.T) { + for _, turnID := range []int64{0, 20} { + t.Run(string(rune('a'+turnID)), func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, turnID) + f.item.Event.AsyncReportTrigger = true + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: "target", TurnID: turnID, RecordID: 201, InvokeID: "201"} + f.item.ExistItemEvalResult.TurnResultRunLogs[turnID].TargetResultID = 201 + target := resumeTarget(201, turnID) + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(target, nil) + ctx := ctxcache.Init(context.Background()) + etec, err := f.executor.buildExptTurnEvalCtx(ctx, f.item.EvalSetItem.Turns[0], f.item, nil) + require.NoError(t, err) + got, err := (&DefaultExptTurnEvaluationImpl{evalTargetService: f.target, benefitService: f.benefit}).CallTarget(ctx, etec) + require.NoError(t, err) + require.Same(t, target, got) + assert.False(t, etec.Event.CtxTargetCalled(ctx), "callback reused target; no new invocation") + assert.False(t, etec.Event.IgnoreExistedEvaluatorResult(ctx), "manual selected mode must not discard current callback refs") + }) + } +} + +func TestRunningRetryTurnResume_EventAndCacheIsolation(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 10, 20, 30) + f.item.Event.AsyncEvaluatorReportTrigger = true + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: "evaluator", TurnID: 20, RecordID: 302, InvokeID: "302"} + f.item.Event.Ext = map[string]string{"lane": "ppe_frozen"} + ctx := ctxcache.Init(context.Background()) + first, err := f.executor.buildExptTurnEvalCtx(ctx, f.item.EvalSetItem.Turns[0], f.item, nil) + require.NoError(t, err) + middle, err := f.executor.buildExptTurnEvalCtx(ctx, f.item.EvalSetItem.Turns[1], f.item, nil) + require.NoError(t, err) + last, err := f.executor.buildExptTurnEvalCtx(ctx, f.item.EvalSetItem.Turns[2], f.item, nil) + require.NoError(t, err) + assert.False(t, first.Event.AsyncEvaluatorReportTrigger) + assert.Equal(t, f.item.Event.ResumeSource, first.Event.ResumeSource) + assert.NotSame(t, f.item.Event.ResumeSource, first.Event.ResumeSource) + assert.True(t, middle.Event.AsyncEvaluatorReportTrigger) + assert.Equal(t, int64(20), middle.Event.ResumeSource.TurnID) + assert.False(t, last.Event.AsyncEvaluatorReportTrigger) + assert.Equal(t, f.item.Event.ResumeSource, last.Event.ResumeSource) + require.NotSame(t, f.item.Event, first.Event) + require.NotSame(t, first.Event, last.Event) + first.Event.WithCtxTargetCalled(ctx) + assert.True(t, first.Event.IgnoreExistedEvaluatorResult(ctx)) + assert.False(t, middle.Event.CtxTargetCalled(ctx)) + assert.False(t, last.Event.IgnoreExistedEvaluatorResult(ctx)) + last.Event.Ext["lane"] = "mutated" + assert.Equal(t, "ppe_frozen", f.item.Event.Ext["lane"]) + assert.Equal(t, "RQ1", last.Event.RequestID) + assert.Equal(t, 3, last.Event.MaxRetryTimes) + assert.Equal(t, "u", last.Event.Session.UserID) +} + +func TestRunningRetryTurnResume_FailRetryOnlyInvokesE2(t *testing.T) { + for _, format := range []string{"registered", "alias", "legacy", "inline"} { + t.Run(format, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 20) + log := f.item.ExistItemEvalResult.TurnResultRunLogs[20] + log.TargetResultID = 201 + target := resumeTarget(201, 20) + e1 := resumeEvaluator(301, 401, 20, "") + switch format { + case "legacy": + log.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{401: 301}} + case "alias": + e1.Alias = "judge-a" + f.item.Expt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + f.item.ItemConfig = &entity.ExptItemConfig{EvaluatorConfs: []*entity.ItemEvaluatorConf{{EvaluatorVersionID: 401, Alias: "judge-a"}, {EvaluatorVersionID: 402, Alias: "judge-b"}}} + fallthrough + default: + log.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, Alias: e1.Alias, RecordID: 301}}} + } + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(target, nil) + loaded := []*entity.EvaluatorRecord{e1} + ids := []int64{301} + if format == "inline" { + log.EvaluatorResultIds.Inline = []*entity.InlineEvalResult{{InlineKey: "quality", RecordID: 304}} + inline := resumeEvaluator(304, 0, 20, "") + inline.SourceType = entity.EvaluatorRecordSourceTypeInline + inline.InlineKey = "quality" + inline.TargetRecordID = 201 + loaded = append(loaded, inline) + ids = append(ids, 304) + } + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), ids, false, false).Return(loaded, nil) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(1) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(1) + var calls []int64 + var mu sync.Mutex + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + mu.Lock() + defer mu.Unlock() + calls = append(calls, r.EvaluatorVersionID) + require.Equal(t, int64(402), r.EvaluatorVersionID) + return resumeEvaluator(303, 402, 20, r.Alias), nil + }).Times(1) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + require.Len(t, rows, 1) + assert.Equal(t, int64(201), rows[0].TargetResultID) + refs := rows[0].EvaluatorResultIds.Registered + require.Len(t, refs, 2) + assert.Equal(t, int64(301), refs[0].RecordID) + assert.Equal(t, int64(303), refs[1].RecordID) + if format == "inline" { + require.Len(t, rows[0].EvaluatorResultIds.Inline, 1) + assert.Equal(t, "quality", rows[0].EvaluatorResultIds.Inline[0].InlineKey) + assert.Equal(t, int64(304), rows[0].EvaluatorResultIds.Inline[0].RecordID) + } + return nil + }) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, []int64{402}, calls) + }) + } +} + +func TestRunningRetryTurnResume_RepeatedTargetCallbackKeepsAllRefs(t *testing.T) { + for _, pending := range []bool{false, true} { + t.Run(map[bool]string{false: "success no score", true: "async in flight"}[pending], func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0) + f.item.Event.AsyncReportTrigger = true + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: "target", TurnID: 0, RecordID: 201, InvokeID: "201"} + f.item.Expt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + f.item.ItemConfig = &entity.ExptItemConfig{EvaluatorConfs: []*entity.ItemEvaluatorConf{{EvaluatorVersionID: 401, Alias: "judge-a"}, {EvaluatorVersionID: 402, Alias: "judge-b"}}} + log := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + log.TargetResultID = 201 + log.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, Alias: "judge-a", RecordID: 301}, {VersionID: 402, Alias: "judge-b", RecordID: 302}}, Inline: []*entity.InlineEvalResult{{InlineKey: "quality", RecordID: 304}}} + target := resumeTarget(201, 0) + e1 := resumeEvaluator(301, 401, 0, "judge-a") + e2 := resumeEvaluator(302, 402, 0, "judge-b") + inline := resumeEvaluator(304, 0, 0, "") + inline.SourceType = entity.EvaluatorRecordSourceTypeInline + inline.InlineKey = "quality" + inline.TargetRecordID = 201 + if pending { + e2.Status = entity.EvaluatorRunStatusAsyncInvoking + f.evaluator.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(302)).Return(nil) + } + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(target, nil) + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{301, 302, 304}, false, false).Return([]*entity.EvaluatorRecord{e1, e2, inline}, nil) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + assert.Equal(t, int64(201), rows[0].TargetResultID) + ids := map[int64]bool{} + for _, r := range rows[0].EvaluatorResultIds.Registered { + ids[r.RecordID] = true + } + for _, r := range rows[0].EvaluatorResultIds.Inline { + ids[r.RecordID] = true + } + assert.Equal(t, map[int64]bool{301: true, 302: true, 304: true}, ids) + require.Len(t, rows[0].EvaluatorResultIds.Inline, 1) + assert.Equal(t, "quality", rows[0].EvaluatorResultIds.Inline[0].InlineKey) + return nil + }) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.Equal(t, pending, abort) + }) + } +} + +func TestRunningRetryTurnResume_SuccessTurnOnlyFeedsHistory(t *testing.T) { + for _, score := range []string{"zero", "absent"} { + t.Run(score, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0, 20) + first := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + first.Status = entity.TurnRunState_Success + first.TargetResultID = 201 + first.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}, {VersionID: 402, RecordID: 302}}} + target := resumeTarget(201, 0) + e1 := resumeEvaluator(301, 401, 0, "") + e2 := resumeEvaluator(302, 402, 0, "") + if score == "zero" { + e1.EvaluatorOutputData = &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(float64(0))}} + } + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(target, nil) + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{301, 302}, false, false).Return([]*entity.EvaluatorRecord{e1, e2}, nil) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(2) + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, exec *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + require.Equal(t, int64(20), exec.TurnID) + require.Len(t, input.HistoryMessages, 1) + assert.Equal(t, "previous answer", input.HistoryMessages[0].Content.GetText()) + return resumeTarget(202, 20), nil + }) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(2) + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + require.Equal(t, int64(20), r.TurnID) + return resumeEvaluator(1000+r.EvaluatorVersionID, r.EvaluatorVersionID, 20, ""), nil + }).Times(2) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + assert.Equal(t, int64(20), rows[0].TurnID) + return nil + }).Times(1) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, int64(1), f.evalCalls.Load(), "only the pending turn enters Eval") + assert.Equal(t, entity.TurnRunState_Success, first.Status) + assert.Equal(t, int64(301), first.EvaluatorResultIds.Registered[0].RecordID) + }) + } +} + +func TestRunningRetryTurnResume_ManualRetryExtNeverReachesActors(t *testing.T) { + for _, multi := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy", true: "alias"}[multi], func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0) + ext := map[string]string{entity.ManualRetryExtKey: `{"request_id":"RQ1"}`, "lane": "ppe_frozen", "session": "session-frozen", "topology": "double", "other": "keep"} + f.item.Event.Ext = ext + f.item.ItemConfig = &entity.ExptItemConfig{Ext: ext, EvalTargetConf: &entity.ItemTargetConf{RunConf: &entity.ItemRunConf{MaxTurns: 3}}} + if multi { + f.item.Expt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + f.item.ItemConfig.EvaluatorConfs = []*entity.ItemEvaluatorConf{{EvaluatorVersionID: 401, Alias: "a"}, {EvaluatorVersionID: 402, Alias: "b"}} + } + f.item.Expt.EvalConf.ConnectorConf.EvaluatorsConf.EvaluatorConf[0].RunConf = &entity.EvaluatorRunConfig{EvaluatorRuntimeParam: &entity.RuntimeParam{JSONValue: gptr.Of(`{"model":"fixed"}`)}} + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(2) + check := func(got map[string]string) { + t.Helper() + assert.NotContains(t, got, entity.ManualRetryExtKey) + for _, k := range []string{"lane", "session", "topology", "other"} { + assert.Equal(t, ext[k], got[k]) + } + } + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, _ *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + check(input.Ext) + assert.Contains(t, input.Ext, consts.TargetExecuteExtRunConfKey) + return resumeTarget(201, 0), nil + }) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, bool, error) { + check(r.Ext) + check(r.InputData.Ext) + return nil, false, nil + }).Times(2) + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + check(r.Ext) + check(r.InputData.Ext) + if r.EvaluatorVersionID == 401 { + assert.Equal(t, `{"model":"fixed"}`, r.InputData.Ext[consts.FieldAdapterBuiltinFieldNameRuntimeParam]) + } + return resumeEvaluator(1000+r.EvaluatorVersionID, r.EvaluatorVersionID, 0, r.Alias), nil + }).Times(2) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, `{"request_id":"RQ1"}`, ext[entity.ManualRetryExtKey]) + }) + } +} + +func TestRunningRetryTurnResume_CallbackFlagsOnlyAffectMiddleTurn(t *testing.T) { + for _, kind := range []string{"target", "evaluator"} { + t.Run(kind, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0, 20, 30) + f.item.Event.AsyncReportTrigger = kind == "target" + f.item.Event.AsyncEvaluatorReportTrigger = kind == "evaluator" + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: kind, TurnID: 20, RecordID: 202, InvokeID: "202"} + first := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + first.Status = entity.TurnRunState_Success + first.TargetResultID = 201 + middle := f.item.ExistItemEvalResult.TurnResultRunLogs[20] + middle.TargetResultID = 202 + // The callback turn already owns both evaluator refs; target callback repeats + // must wait/reuse them, while turn 30 retains ordinary execution behavior. + middle.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}, {VersionID: 402, RecordID: 302}}} + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(resumeTarget(201, 0), nil) + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(202)).Return(resumeTarget(202, 20), nil) + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{301, 302}, false, false).Return([]*entity.EvaluatorRecord{resumeEvaluator(301, 401, 20, ""), resumeEvaluator(302, 402, 20, "")}, nil) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(2) + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, r *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + assert.Equal(t, int64(30), r.TurnID) + assert.Len(t, input.HistoryMessages, 2) + return resumeTarget(203, 30), nil + }) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(2) + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + assert.Equal(t, int64(30), r.TurnID) + return resumeEvaluator(1000+r.EvaluatorVersionID, r.EvaluatorVersionID, 30, ""), nil + }).Times(2) + var stored []int64 + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + stored = append(stored, rows[0].TurnID) + return nil + }).Times(2) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, []int64{20, 30}, stored) + assert.Equal(t, int64(20), f.item.Event.ResumeSource.TurnID) + }) + } +} + +func TestRunningRetryTurnResume_NewTargetInOneTurnDoesNotInvalidateNextTurn(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 10, 20) + // First turn needs a new target. Next turn has prepared successful T/E1; + // the first turn's cache must not invalidate E1 and cause a second charge. + next := f.item.ExistItemEvalResult.TurnResultRunLogs[20] + next.TargetResultID = 202 + next.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}}} + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(202)).Return(resumeTarget(202, 20), nil) + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{301}, false, false).Return([]*entity.EvaluatorRecord{resumeEvaluator(301, 401, 20, "")}, nil) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(3) + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(resumeTarget(201, 10), nil).Times(1) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(3) + var mu sync.Mutex + calls := map[int64][]int64{} + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + mu.Lock() + defer mu.Unlock() + calls[r.TurnID] = append(calls[r.TurnID], r.EvaluatorVersionID) + if r.TurnID == 20 { + assert.Equal(t, int64(402), r.EvaluatorVersionID) + } + return resumeEvaluator(2000+r.TurnID+r.EvaluatorVersionID, r.EvaluatorVersionID, r.TurnID, ""), nil + }).Times(3) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + if rows[0].TurnID == 20 { + assert.Equal(t, int64(301), rows[0].EvaluatorResultIds.Registered[0].RecordID) + } + return nil + }).Times(2) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, []int64{402}, calls[20]) + assert.Len(t, calls[10], 2) +} + +func TestRunningRetryTurnResume_ManualSelectionRunsEveryPreparedTurn(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0, 20) + // Admission prepared all turns by clearing refs and success state. + for _, log := range f.item.ExistItemEvalResult.TurnResultRunLogs { + log.Status = entity.TurnRunState_Processing + } + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(4) + var targetCalls []int64 + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, r *entity.ExecuteTargetCtx, _ *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + targetCalls = append(targetCalls, r.TurnID) + return resumeTarget(201+r.TurnID, r.TurnID), nil + }).Times(2) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(4) + var mu sync.Mutex + calls := map[int64]int{} + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + mu.Lock() + defer mu.Unlock() + calls[r.TurnID]++ + return resumeEvaluator(1000+r.TurnID+r.EvaluatorVersionID, r.EvaluatorVersionID, r.TurnID, ""), nil + }).Times(4) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(2) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, []int64{0, 20}, targetCalls) + assert.Equal(t, map[int64]int{0: 2, 20: 2}, calls) +} + +func TestRunningRetryTurnResume_CallbackRefsReadIncompleteDoesNotStartEvaluators(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0) + f.item.Event.AsyncReportTrigger = true + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: "target", TurnID: 0, RecordID: 201, InvokeID: "201"} + log := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + log.TargetResultID = 201 + log.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}}} + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(resumeTarget(201, 0), nil) + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{301}, false, false).Return(nil, nil) + _, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.Error(t, err) +} + +func TestRunningRetryTurnResume_SourceWithEmptyRefsStartsEvaluatorsOnce(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0) + f.item.Event.AsyncReportTrigger = true + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: "target", TurnID: 0, RecordID: 201, InvokeID: "201"} + f.item.ExistItemEvalResult.TurnResultRunLogs[0].TargetResultID = 201 + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(resumeTarget(201, 0), nil) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(1) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(2) + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + return resumeEvaluator(1000+r.EvaluatorVersionID, r.EvaluatorVersionID, 0, ""), nil + }).Times(2) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) +} + +func TestRunningRetryTurnResume_AsyncEvaluatorExtAndIdentity(t *testing.T) { + for _, alias := range []string{"", "judge"} { + t.Run("alias="+alias, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0) + f.item.Event.Ext = map[string]string{entity.ManualRetryExtKey: "internal", "lane": "ppe_frozen"} + ev := f.item.Expt.Evaluators[0] + ev.EvaluatorType = entity.EvaluatorTypeAgent + ev.AgentEvaluatorVersion = &entity.AgentEvaluatorVersion{ID: 401, EvaluatorID: 401} + f.item.Expt.Evaluators = f.item.Expt.Evaluators[:1] + if alias != "" { + f.item.Expt.EvalSetSourceType = entity.ExptEvalSetSourceType_MultiSetConfig + f.item.ItemConfig = &entity.ExptItemConfig{EvaluatorConfs: []*entity.ItemEvaluatorConf{{EvaluatorVersionID: 401, Alias: alias}}} + } + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil) + f.evaluator.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + assert.NotContains(t, r.Ext, entity.ManualRetryExtKey) + assert.NotContains(t, r.InputData.Ext, entity.ManualRetryExtKey) + assert.Equal(t, "ppe_frozen", r.Ext["lane"]) + require.NotNil(t, r.AsyncCtx.Event) + assert.Equal(t, "RQ1", r.AsyncCtx.Event.RequestID) + assert.Equal(t, "internal", r.AsyncCtx.Event.Ext[entity.ManualRetryExtKey]) + assert.Equal(t, alias, r.Alias) + return &entity.EvaluatorRecord{ID: 301, EvaluatorVersionID: 401, Alias: alias, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil + }) + f.records.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(301), false).Return(&entity.EvaluatorRecord{ID: 301, EvaluatorVersionID: 401, Alias: alias, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil) + ctx := ctxcache.Init(context.Background()) + etec, err := f.executor.buildExptTurnEvalCtx(ctx, f.item.EvalSetItem.Turns[0], f.item, nil) + require.NoError(t, err) + results, err := (&DefaultExptTurnEvaluationImpl{metric: f.executor.Metric, evaluatorService: f.evaluator, evaluatorRecordService: f.records, benefitService: f.benefit}).CallEvaluators(ctx, etec, resumeTarget(201, 0)) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, int64(301), results[0].ID) + }) + } +} + +func TestRunningRetryTurnResume_DirectCallerIgnoresOtherTurnCallback(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeRetryItems, 0) + f.item.Event.AsyncEvaluatorReportTrigger = true + f.item.Event.ResumeSource = &entity.ExptResumeSource{Kind: "evaluator", TurnID: 20, RecordID: 301, InvokeID: "301"} + // A callback from turn 20 is still a continuation of RQ1 on turn zero. + // Only its callback flags are ignored here; successful current refs survive. + etec := &entity.ExptTurnEvalCtx{ExptItemEvalCtx: f.item, Turn: f.item.EvalSetItem.Turns[0], Ext: map[string]string{}, ExptTurnRunResult: &entity.ExptTurnRunResult{TargetResult: resumeTarget(201, 0), EvaluatorResults: []*entity.EvaluatorRecord{resumeEvaluator(301, 401, 0, ""), resumeEvaluator(302, 402, 0, "")}}} + ctx := ctxcache.Init(context.Background()) + svc := &DefaultExptTurnEvaluationImpl{metric: f.executor.Metric, evalTargetService: f.target, evaluatorService: f.evaluator, benefitService: f.benefit} + target, err := svc.CallTarget(ctx, etec) + require.NoError(t, err) + assert.Equal(t, int64(201), target.ID) + scoped := etec.Event + results, err := svc.CallEvaluators(ctx, etec, target) + require.NoError(t, err) + require.Len(t, results, 2) + assert.Equal(t, int64(301), results[0].ID) + assert.Equal(t, int64(302), results[1].ID) + assert.Same(t, scoped, etec.Event, "both stage calls share the scoped event") + assert.False(t, etec.Event.AsyncEvaluatorReportTrigger) + assert.Equal(t, int64(20), etec.Event.ResumeSource.TurnID) + assert.True(t, f.item.Event.AsyncEvaluatorReportTrigger) + assert.Equal(t, int64(20), f.item.Event.ResumeSource.TurnID) +} + +func TestRunningRetryTurnResume_EventCopiesConcurrentCache(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0) + ctx := ctxcache.Init(context.Background()) + original := f.item.Event + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + called := original.Clone() + other := original.Clone() + called.WithCtxTargetCalled(ctx) + assert.True(t, called.IgnoreExistedEvaluatorResult(ctx)) + assert.False(t, other.IgnoreExistedEvaluatorResult(ctx)) + called.Ext = map[string]string{"own": "value"} + }() + } + wg.Wait() + assert.False(t, original.CtxTargetCalled(ctx)) + assert.Nil(t, original.Ext) +} + +// Storage boundaries return the same preview record that MySQL returns; the +// item/turn execution and history construction remain real. +func TestRunningRetryHistoryOutput_LoadsFullSuccessOutputBeforeNextTarget(t *testing.T) { + for _, source := range []string{"same space", "frozen target space", "frozen item target space"} { + t.Run(source, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0, 20) + spaceID := int64(3) + switch source { + case "frozen target space": + f.item.Expt.TargetSpaceID = 9 + spaceID = 9 + case "frozen item target space": + f.item.Expt.TargetSpaceID = 8 + f.item.ItemConfig = &entity.ExptItemConfig{TargetSourceSpaceID: 9} + spaceID = 9 + } + first := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + first.Status = entity.TurnRunState_Success + first.TargetResultID = 201 + first.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}, {VersionID: 402, RecordID: 302}}} + full := strings.Repeat("complete previous answer ", 10000) + target := resumeTarget(201, 0) + target.SpaceID = spaceID + target.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = &entity.Content{ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of("preview..."), ContentOmitted: gptr.Of(true), FullContent: &entity.ObjectStorage{URI: gptr.Of("eval_record/history-output")}, FullContentBytes: gptr.Of(int32(len(full)))} + preview := target.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] + require.True(t, preview.IsContentOmitted()) + require.Greater(t, len(full), 204800) + e1 := resumeEvaluator(301, 401, 0, "") + e1.EvaluatorOutputData = &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(float64(0))}} + e2 := resumeEvaluator(302, 402, 0, "") // Successful without a score stays reusable. + f.target.EXPECT().GetRecordByID(gomock.Any(), spaceID, int64(201)).Return(target, nil) + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{301, 302}, false, false).Return([]*entity.EvaluatorRecord{e1, e2}, nil) + var loaded bool + f.target.EXPECT().LoadRecordOutputFields(gomock.Any(), gomock.Any(), []string{consts.OutputSchemaKey}).DoAndReturn(func(_ context.Context, got *entity.EvalTargetRecord, _ []string) error { + require.Same(t, target, got) + assert.Equal(t, spaceID, got.SpaceID) + require.Equal(t, "eval_record/history-output", gptr.Indirect(got.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey].FullContent.URI)) + got.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey].Text = gptr.Of(full) + loaded = true + return nil + }).Times(1) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ *benefit.CheckAndDeductEvalBenefitParams) (*benefit.CheckAndDeductEvalBenefitResult, error) { + assert.True(t, loaded, "success history loads before next turn spends benefit") + return &benefit.CheckAndDeductEvalBenefitResult{}, nil + }).Times(2) + f.target.EXPECT().ExecuteTarget(gomock.Any(), spaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, exec *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + assert.True(t, loaded) + assert.Equal(t, int64(20), exec.TurnID) + require.Len(t, input.HistoryMessages, 1) + assert.Equal(t, entity.RoleAssistant, input.HistoryMessages[0].Role) + assert.Equal(t, full, input.HistoryMessages[0].Content.GetText()) + assert.False(t, input.HistoryMessages[0].Content.IsContentOmitted()) + return resumeTarget(202, 20), nil + }).Times(1) + f.evaluator.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(2) + f.evaluator.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + assert.Equal(t, int64(20), r.TurnID) + return resumeEvaluator(1000+r.EvaluatorVersionID, r.EvaluatorVersionID, 20, ""), nil + }).Times(2) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + require.Len(t, rows, 1) + assert.Equal(t, int64(20), rows[0].TurnID) + return nil + }).Times(1) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, int64(1), f.evalCalls.Load(), "reused success turn never enters Eval") + assert.Equal(t, int64(201), first.TargetResultID) + assert.Equal(t, entity.TurnRunState_Success, first.Status) + assert.Equal(t, []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}, {VersionID: 402, RecordID: 302}}, first.EvaluatorResultIds.Registered) + }) + } +} + +func TestRunningRetryHistoryOutput_IncompleteSuccessStopsBeforeNextTurn(t *testing.T) { + for _, failure := range []string{"missing target record", "storage read error", "still omitted", "field removed", "nil field", "nil output data"} { + t.Run(failure, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0, 20) + first := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + first.Status = entity.TurnRunState_Success + first.TargetResultID = 201 + target := resumeTarget(201, 0) + cause := errors.New("object storage read failed") + if failure == "missing target record" { + target = nil + } else { + target.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = &entity.Content{ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of("preview..."), ContentOmitted: gptr.Of(true), FullContent: &entity.ObjectStorage{URI: gptr.Of("eval_record/history-output")}, FullContentBytes: gptr.Of(int32(240000))} + f.target.EXPECT().LoadRecordOutputFields(gomock.Any(), gomock.Any(), []string{consts.OutputSchemaKey}).DoAndReturn(func(_ context.Context, got *entity.EvalTargetRecord, _ []string) error { + require.Same(t, target, got) + switch failure { + case "storage read error": + return cause + case "field removed": + delete(got.EvalTargetOutputData.OutputFields, consts.OutputSchemaKey) + case "nil field": + got.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = nil + case "nil output data": + got.EvalTargetOutputData = nil + } + return nil + }).Times(1) + } + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(target, nil) + // No expected Eval/store/target/evaluator/benefit calls: a missing or + // incomplete old success must stop, rather than refill it with a new invoke. + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.Error(t, err) + assert.False(t, abort) + if failure == "storage read error" { + assert.ErrorIs(t, err, cause) + } + assert.Zero(t, f.evalCalls.Load()) + assert.Equal(t, int64(201), first.TargetResultID) + assert.Equal(t, entity.TurnRunState_Success, first.Status) + }) + } +} + +func TestRunningRetryHistoryOutput_ShortNontextAndTargetFreeRemainCompatible(t *testing.T) { + for _, shape := range []string{"short text", "nontext", "target-free zero record"} { + t.Run(shape, func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0) + first := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + first.Status = entity.TurnRunState_Success + switch shape { + case "target-free zero record": + f.item.Expt.TargetVersionID = 0 + default: + first.TargetResultID = 201 + target := resumeTarget(201, 0) + if shape == "nontext" { + target.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = &entity.Content{ContentType: gptr.Of(entity.ContentTypeImage), Image: &entity.Image{URL: gptr.Of("image-ref")}, ContentOmitted: gptr.Of(true)} + } + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), int64(201)).Return(target, nil) + } + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Zero(t, f.evalCalls.Load()) + assert.Equal(t, entity.TurnRunState_Success, first.Status) + }) + } +} + +func TestRunningRetryHistoryOutput_FreshOmittedTurnLoadsBeforeNextTarget(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeSubmit, 0, 20) + // ExecuteTarget persists before returning; storage mutates the returned + // Content to a preview. With no evaluators, nothing else hydrates it. + f.item.Expt.EvalConf.ConnectorConf.EvaluatorsConf = nil + full := strings.Repeat("complete fresh answer ", 11000) + first := resumeTarget(201, 0) + first.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = &entity.Content{ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of("preview..."), ContentOmitted: gptr.Of(true), FullContent: &entity.ObjectStorage{URI: gptr.Of("eval_record/fresh-history")}, FullContentBytes: gptr.Of(int32(len(full)))} + require.True(t, first.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey].IsContentOmitted()) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(2) + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, exec *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + if exec.TurnID == 0 { + assert.Empty(t, input.HistoryMessages) + return first, nil + } + require.Len(t, input.HistoryMessages, 1) + assert.Equal(t, full, input.HistoryMessages[0].Content.GetText()) + assert.False(t, input.HistoryMessages[0].Content.IsContentOmitted()) + return resumeTarget(202, 20), nil + }).Times(2) + f.target.EXPECT().LoadRecordOutputFields(gomock.Any(), gomock.Any(), []string{consts.OutputSchemaKey}).DoAndReturn(func(_ context.Context, got *entity.EvalTargetRecord, _ []string) error { + require.Same(t, first, got) + got.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey].Text = gptr.Of(full) + return nil + }).Times(1) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(2) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, int64(2), f.evalCalls.Load()) +} + +func TestRunningRetryHistoryOutput_FreshFinalTurnDoesNotLoadUnusedHistory(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeSubmit, 0) + f.item.Expt.EvalConf.ConnectorConf.EvaluatorsConf = nil + target := resumeTarget(201, 0) + target.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = &entity.Content{ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of("preview..."), ContentOmitted: gptr.Of(true), FullContent: &entity.ObjectStorage{URI: gptr.Of("eval_record/final-output")}, FullContentBytes: gptr.Of(int32(240000))} + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil) + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(target, nil) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil) + // No subsequent turn consumes history, so successful single-turn execution + // must not add an object-storage dependency after persisting its result. + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, int64(1), f.evalCalls.Load()) +} + +func TestRunningRetryHistoryIsolation_ThreeTurnsSurviveActualRecordStorage(t *testing.T) { + for _, reused := range []bool{true, false} { + t.Run(map[bool]string{true: "reused success", false: "fresh output"}[reused], func(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 0, 20, 30) + f.item.Expt.TargetSpaceID = 9 + f.item.Expt.EvalConf.ConnectorConf.EvaluatorsConf = nil + f.config.EXPECT().GetEvaluationRecordStorage(gomock.Any()).Return(&component.EvaluationRecordStorage{Providers: []*component.EvaluationRecordProviderConfig{{Provider: "RDS", MaxSize: 204800}}}).AnyTimes() + full := strings.Repeat("complete first answer ", 11000) + require.Greater(t, len(full), 204800) + firstTarget := resumeTarget(201, 0) + firstTarget.SpaceID = 9 + firstTarget.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = &entity.Content{ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of(full)} + blobs := map[string][]byte{"initial-full-output": []byte(full)} + blob := fsmocks.NewMockBatchObjectStorage(gomock.NewController(t)) + uploadCount := 3 + executeCount := 3 + if reused { + uploadCount = 2 + executeCount = 2 + } + blob.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, key string, body io.Reader, _ ...fileserver.UploadOpt) error { + data, err := io.ReadAll(body) + require.NoError(t, err) + blobs[key] = data + return nil + }).Times(uploadCount) + blob.EXPECT().Read(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, key string, _ ...fileserver.DownloadOpt) (fileserver.Reader, error) { + data, exists := blobs[key] + require.True(t, exists) + return &historyIsolationReader{Reader: bytes.NewReader(data)}, nil + }).Times(1) + storage := evalstorage.NewRecordDataStorage(blob, f.config) + firstLog := f.item.ExistItemEvalResult.TurnResultRunLogs[0] + if reused { + firstLog.Status = entity.TurnRunState_Success + firstLog.TargetResultID = 201 + firstLog.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}, {VersionID: 402, RecordID: 302}}} + output := firstTarget.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] + output.Text = gptr.Of("preview...") + output.ContentOmitted = gptr.Of(true) + output.FullContent = &entity.ObjectStorage{URI: gptr.Of("initial-full-output")} + output.FullContentBytes = gptr.Of(int32(len(full))) + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(9), int64(201)).Return(firstTarget, nil) + e1 := resumeEvaluator(301, 401, 0, "") + e1.EvaluatorOutputData = &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(float64(0))}} + f.records.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{301, 302}, false, false).Return([]*entity.EvaluatorRecord{e1, resumeEvaluator(302, 402, 0, "")}, nil) + } + f.target.EXPECT().LoadRecordOutputFields(gomock.Any(), gomock.Any(), []string{consts.OutputSchemaKey}).DoAndReturn(func(ctx context.Context, record *entity.EvalTargetRecord, keys []string) error { + require.Same(t, firstTarget, record) + return storage.LoadEvalTargetOutputFields(ctx, record, keys) + }).Times(1) + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil).Times(executeCount) + var called []int64 + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(9), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ int64, _ int64, _ int64, exec *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + called = append(called, exec.TurnID) + var record *entity.EvalTargetRecord + if exec.TurnID == 0 { + assert.Empty(t, input.HistoryMessages) + record = firstTarget + } else { + if exec.TurnID == 20 { + require.Len(t, input.HistoryMessages, 1) + } else { + require.Len(t, input.HistoryMessages, 2) + assert.Equal(t, "previous answer", input.HistoryMessages[1].Content.GetText()) + } + original := firstTarget.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] + assert.NotSame(t, original, input.HistoryMessages[0].Content, "invocation storage must receive independent Content") + assert.True(t, full == input.HistoryMessages[0].Content.GetText(), "later target must see complete first output bytes") + assert.False(t, input.HistoryMessages[0].Content.IsContentOmitted()) + record = resumeTarget(201+exec.TurnID, exec.TurnID) + record.SpaceID = 9 + } + record.EvalTargetInputData = input + // Actual storage traverses and clips invocation input history before the + // provider returns its record, matching ExecuteTarget's deferred save. + require.NoError(t, storage.SaveEvalTargetRecordData(ctx, record, nil)) + if exec.TurnID != 0 { + assert.True(t, input.HistoryMessages[0].Content.IsContentOmitted(), "actual invocation storage clipped its copy") + assert.True(t, full == firstTarget.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey].GetText(), "shared first output remains full after provider save") + assert.False(t, firstTarget.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey].IsContentOmitted()) + } + return record, nil + }).Times(executeCount) + f.turns.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rows []*entity.ExptTurnResultRunLog) error { + require.Len(t, rows, 1) + if reused { + assert.NotEqual(t, int64(0), rows[0].TurnID) + } + return nil + }).Times(executeCount) + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Equal(t, int64(executeCount), f.evalCalls.Load()) + if reused { + assert.Equal(t, []int64{20, 30}, called) + assert.Equal(t, int64(201), firstLog.TargetResultID) + assert.Equal(t, entity.TurnRunState_Success, firstLog.Status) + assert.Equal(t, []*entity.RegisteredEvalResult{{VersionID: 401, RecordID: 301}, {VersionID: 402, RecordID: 302}}, firstLog.EvaluatorResultIds.Registered) + } else { + assert.Equal(t, []int64{0, 20, 30}, called) + } + assert.True(t, full == firstTarget.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey].GetText()) + }) + } +} + +func TestRunningRetryHistoryIsolation_TargetInputCopiesNestedContent(t *testing.T) { + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, 20) + content := &entity.Content{ContentType: gptr.Of(entity.ContentTypeMultipart), Format: gptr.Of(entity.FieldDisplayFormat_JSON), Text: gptr.Of("root"), Image: &entity.Image{Name: gptr.Of("image"), URI: gptr.Of("image-uri"), URL: gptr.Of("image-url"), ThumbURL: gptr.Of("image-thumb"), StorageProvider: entity.StorageProviderPtr(entity.StorageProvider_S3)}, Audio: &entity.Audio{Format: gptr.Of("wav"), URI: gptr.Of("audio-uri"), Name: gptr.Of("audio"), URL: gptr.Of("audio-url"), StorageProvider: entity.StorageProviderPtr(entity.StorageProvider_S3)}, Video: &entity.Video{Name: gptr.Of("video"), URI: gptr.Of("video-uri"), URL: gptr.Of("video-url"), ThumbURL: gptr.Of("video-thumb"), StorageProvider: entity.StorageProviderPtr(entity.StorageProvider_S3)}, FullContent: &entity.ObjectStorage{Name: gptr.Of("full"), URI: gptr.Of("full-uri"), URL: gptr.Of("full-url"), ThumbURL: gptr.Of("full-thumb"), Provider: entity.StorageProviderPtr(entity.StorageProvider_S3)}, ContentOmitted: gptr.Of(true), FullContentBytes: gptr.Of(int32(240000)), MultiPart: []*entity.Content{{ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of("nested"), MultiPart: []*entity.Content{{Text: gptr.Of("deep")}}}}} + history := []*entity.Message{{Role: entity.RoleAssistant, Content: content, Ext: map[string]string{"metadata": "frozen"}}, nil, {Role: entity.RoleTool}} + etec := &entity.ExptTurnEvalCtx{ExptItemEvalCtx: f.item, Turn: f.item.EvalSetItem.Turns[0], History: history, Ext: map[string]string{}, ExptTurnRunResult: &entity.ExptTurnRunResult{}} + f.benefit.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil) + f.target.EXPECT().ExecuteTarget(gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, _ int64, _ int64, _ int64, _ *entity.ExecuteTargetCtx, input *entity.EvalTargetInputData) (*entity.EvalTargetRecord, error) { + assert.Equal(t, history, input.HistoryMessages, "all nested input fields and nil messages are preserved") + c := input.HistoryMessages[0].Content + assert.NotSame(t, content, c) + assert.NotSame(t, content.MultiPart[0], c.MultiPart[0]) + assert.NotSame(t, content.MultiPart[0].MultiPart[0], c.MultiPart[0].MultiPart[0]) + assert.NotSame(t, content.Image, c.Image) + assert.NotSame(t, content.Audio, c.Audio) + assert.NotSame(t, content.Video, c.Video) + assert.NotSame(t, content.FullContent, c.FullContent) + *c.Text = "changed" + *c.Format = entity.FieldDisplayFormat_Markdown + *c.ContentType = entity.ContentTypeText + *c.ContentOmitted = false + *c.FullContentBytes = 1 + *c.MultiPart[0].Text = "nested changed" + *c.MultiPart[0].MultiPart[0].Text = "deep changed" + *c.Image.URI = "image changed" + *c.Image.StorageProvider = entity.StorageProvider(99) + *c.Audio.URI = "audio changed" + *c.Audio.Format = "mp3" + *c.Video.URI = "video changed" + *c.Video.ThumbURL = "thumb changed" + *c.FullContent.URI = "full changed" + *c.FullContent.Provider = entity.StorageProvider(99) + input.HistoryMessages[0].Ext["metadata"] = "changed" + input.HistoryMessages[0].Role = entity.RoleUser + return resumeTarget(201, 20), nil + }) + svc := &DefaultExptTurnEvaluationImpl{metric: f.executor.Metric, evalTargetService: f.target, benefitService: f.benefit} + _, err := svc.CallTarget(ctxcache.Init(context.Background()), etec) + require.NoError(t, err) + assert.Equal(t, "root", content.GetText()) + assert.Equal(t, entity.ContentTypeMultipart, *content.ContentType) + assert.Equal(t, entity.FieldDisplayFormat_JSON, *content.Format) + assert.True(t, *content.ContentOmitted) + assert.Equal(t, int32(240000), *content.FullContentBytes) + assert.Equal(t, "nested", content.MultiPart[0].GetText()) + assert.Equal(t, "deep", content.MultiPart[0].MultiPart[0].GetText()) + assert.Equal(t, "image-uri", *content.Image.URI) + assert.Equal(t, entity.StorageProvider_S3, *content.Image.StorageProvider) + assert.Equal(t, "audio-uri", *content.Audio.URI) + assert.Equal(t, "wav", *content.Audio.Format) + assert.Equal(t, "video-uri", *content.Video.URI) + assert.Equal(t, "video-thumb", *content.Video.ThumbURL) + assert.Equal(t, "full-uri", *content.FullContent.URI) + assert.Equal(t, entity.StorageProvider_S3, *content.FullContent.Provider) + assert.Equal(t, "frozen", history[0].Ext["metadata"]) + assert.Equal(t, entity.RoleAssistant, history[0].Role) +} + +func TestRunningRetryHistoryIsolation_LastSuccessDoesNotLoadUnusedHistory(t *testing.T) { + for _, hasPrior := range []bool{false, true} { + t.Run(map[bool]string{false: "single turn zero", true: "last of two successes"}[hasPrior], func(t *testing.T) { + ids := []int64{0} + if hasPrior { + ids = []int64{0, 20} + } + f := newTurnResumeFixture(t, entity.EvaluationModeFailRetry, ids...) + for _, id := range ids { + log := f.item.ExistItemEvalResult.TurnResultRunLogs[id] + log.Status = entity.TurnRunState_Success + log.TargetResultID = 201 + id + target := resumeTarget(201+id, id) + if id == ids[len(ids)-1] { + target.EvalTargetOutputData.OutputFields[consts.OutputSchemaKey] = &entity.Content{ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of("preview..."), ContentOmitted: gptr.Of(true), FullContent: &entity.ObjectStorage{URI: gptr.Of("unused-final-history")}, FullContentBytes: gptr.Of(int32(240000))} + } + f.target.EXPECT().GetRecordByID(gomock.Any(), int64(3), 201+id).Return(target, nil) + } + // No load is expected; lack of a history consumer must not create an + // object-storage dependency, while nonzero target record reads still run. + abort, err := f.executor.EvalTurns(ctxcache.Init(context.Background()), f.item) + require.NoError(t, err) + assert.False(t, abort) + assert.Zero(t, f.evalCalls.Load()) + for _, id := range ids { + assert.Equal(t, entity.TurnRunState_Success, f.item.ExistItemEvalResult.TurnResultRunLogs[id].Status) + } + }) + } +} + +// A test-only object-storage reader retains ReadAt required by the real boundary. +type historyIsolationReader struct{ *bytes.Reader } + +func (r *historyIsolationReader) Close() error { return nil } diff --git a/backend/modules/evaluation/domain/service/expt_scan_transition_mysql_test.go b/backend/modules/evaluation/domain/service/expt_scan_transition_mysql_test.go new file mode 100644 index 0000000000..b2ad59f105 --- /dev/null +++ b/backend/modules/evaluation/domain/service/expt_scan_transition_mysql_test.go @@ -0,0 +1,209 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + mysqldriver "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/db" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" + svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + experimentrepo "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment" + experimentmysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" +) + +func TestScanTransitionMySQLCompleteThenArchive(t *testing.T) { + for _, failed := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) { + ctx := context.Background() + dsn := os.Getenv("EXPT_RETRY_MYSQL_DSN") + if dsn == "" { + t.Skip("EXPT_RETRY_MYSQL_DSN missing; real MySQL not executed") + } + provider, err := db.NewDB(mysqldriver.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + conn := provider.NewSession(ctx, db.WithMaster()) + var identity struct{ Version, Schema string } + require.NoError(t, conn.Raw("SELECT VERSION() AS version,DATABASE() AS `schema`").Scan(&identity).Error) + require.Equal(t, "expt_running_retry_test", identity.Schema) + t.Logf("REAL_MYSQL_EXECUTED version=%s schema=%s", identity.Version, identity.Schema) + base := int64(870000000000000000) + time.Now().UnixNano()%1000000000000 + space, expt, run, item, turn := base+1, base+2, base+3, int64(11), int64(21) + t.Cleanup(func() { + require.NoError(t, conn.Exec("DELETE FROM experiment WHERE space_id=? AND id=?", space, expt).Error) + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_stats", "expt_run_log"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=?", space, expt).Error) + } + sqlDB, err := conn.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + }) + require.NoError(t, conn.Exec("INSERT INTO experiment(id,space_id,latest_run_id,status,expt_type) VALUES(?,?,?,?,?)", expt, space, run, entity.ExptStatus_Processing, entity.ExptType_Offline).Error) + require.NoError(t, conn.Create(&model.ExptRunLog{ID: run, SpaceID: space, ExptID: expt, ExptRunID: run, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit))}).Error) + require.NoError(t, conn.Create(&model.ExptItemResult{ID: base + 4, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: 31, Status: int32(entity.ItemRunState_Processing)}).Error) + require.NoError(t, conn.Create(&model.ExptItemResultRunLog{ID: base + 5, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: 31, Status: int32(entity.ItemRunState_Processing), ResultState: gptr.Of(int32(entity.ExptItemResultStateDefault)), RetryTimes: 3}).Error) + require.NoError(t, conn.Create(&model.ExptStats{ID: base + 6, SpaceID: space, ExptID: expt, ProcessingCnt: 1}).Error) + require.NoError(t, conn.Create(&model.ExptTurnResult{ID: base + 7, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: 31, TurnID: turn, Status: int32(entity.TurnRunState_Processing), WeightedScore: gptr.Of(0.0)}).Error) + status, turnStatus := entity.ItemRunState_Success, entity.TurnRunState_Success + var evalErr error + if failed { + status = entity.ItemRunState_Fail + turnStatus = entity.TurnRunState_Fail + evalErr = errors.New("evaluator final failure") + } + // Invocation stages already persisted this turn; the real completion and archival methods follow. + require.NoError(t, conn.Create(&model.ExptTurnResultRunLog{ID: base + 8, SpaceID: space, ExptID: expt, ExptRunID: run, ItemID: item, ItemVersionID: 31, TurnID: turn, Status: int32(turnStatus)}).Error) + itemRepo := experimentrepo.NewExptItemResultRepo(experimentmysql.NewExptItemResultDAO(provider)) + turnRepo := experimentrepo.NewExptTurnResultRepo(nil, experimentmysql.NewExptTurnResultDAO(provider), experimentmysql.NewExptTurnEvaluatorResultRefDAO(provider)) + statsRepo := experimentrepo.NewExptStatsRepo(experimentmysql.NewExptStatsDAO(provider)) + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), space).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 2}}).AnyTimes() + if failed { + config.EXPECT().GetErrRetryConf(gomock.Any(), space, evalErr).Return(&entity.RetryConf{RetryTimes: 3}).Times(2) + } + event := &entity.ExptItemEvalEvent{SpaceID: space, ExptID: expt, ExptRunID: run, EvalSetItemID: item, ExptRunMode: entity.EvaluationModeSubmit, RetryTimes: 3, MaxRetryTimes: 3} + exptDO := &entity.Experiment{ID: expt, SpaceID: space, EvalConf: &entity.EvaluationConfiguration{}, EvalSet: &entity.EvaluationSet{EvaluationSetVersion: &entity.EvaluationSetVersion{ID: 41}}} + executor := &ExptItemEvalCtxExecutor{ItemResultRepo: itemRepo, Configer: config} + require.NoError(t, executor.CompleteItemRun(ctx, &entity.ExptItemEvalCtx{Event: event, Expt: exptDO}, evalErr)) + canonical, err := itemRepo.BatchGet(ctx, space, expt, []int64{item}) + require.NoError(t, err) + require.Equal(t, status, canonical[0].Status) + beforeArchiveStats, err := statsRepo.Get(ctx, expt, space) + require.NoError(t, err) + require.Zero(t, beforeArchiveStats.ProcessingItemCnt) + if failed { + require.Equal(t, int32(1), beforeArchiveStats.FailItemCnt) + } else { + require.Equal(t, int32(1), beforeArchiveStats.SuccessItemCnt) + } + log, err := itemRepo.GetItemRunLog(ctx, expt, run, item, space) + require.NoError(t, err) + require.Equal(t, int32(status), log.Status) + require.Equal(t, int32(entity.ExptItemResultStateLogged), log.ResultState) + scan := &exptBaseExec{exptItemResultRepo: itemRepo, configer: config} + scheduleEvent := &entity.ExptScheduleEvent{SpaceID: space, ExptID: expt, ExptRunID: run, ExptRunMode: entity.EvaluationModeSubmit} + candidates, occupied, complete, err := scan.ScanEvalItems(ctx, scheduleEvent, exptDO) + require.NoError(t, err) + require.Empty(t, candidates) + require.Empty(t, occupied) + require.Len(t, complete, 1) + require.Equal(t, status, complete[0].State) + source, err := itemRepo.GetCurrentRequest(ctx, space, expt, run, item) + require.NoError(t, err) + require.False(t, source.Matches(run, ""), "completed log cannot invoke again") + consumer := &ExptItemEventEvalServiceImpl{exptItemResultRepo: itemRepo} + invoked := false + require.NoError(t, consumer.HandleCurrentSource(func(context.Context, *entity.ExptItemEvalEvent) error { invoked = true; return nil })(ctx, event)) + require.False(t, invoked, "completed log blocks PreEval and invocation before canonical archival") + + archive := &ExptResultServiceImpl{ExptItemResultRepo: itemRepo, ExptTurnResultRepo: turnRepo, ExptStatsRepo: statsRepo, resultPublishRepo: experimentrepo.NewExptResultPublishRepo(provider), resultPublishLocker: lateWriteLocker(t, ctrl)} + _, err = archive.RecordItemRunLogs(ctx, expt, run, item, space, exptDO) + require.NoError(t, err) + canonical, err = itemRepo.BatchGet(ctx, space, expt, []int64{item}) + require.NoError(t, err) + require.Equal(t, status, canonical[0].Status) + log, err = itemRepo.GetItemRunLog(ctx, expt, run, item, space) + require.NoError(t, err) + require.Equal(t, int32(entity.ExptItemResultStateResulted), log.ResultState) + stats, err := statsRepo.Get(ctx, expt, space) + require.NoError(t, err) + require.Equal(t, *beforeArchiveStats, *stats) + require.Zero(t, stats.ProcessingItemCnt) + if failed { + require.Equal(t, int32(1), stats.FailItemCnt) + } else { + require.Equal(t, int32(1), stats.SuccessItemCnt) + } + before := *stats + candidates, occupied, complete, err = scan.ScanEvalItems(ctx, scheduleEvent, exptDO) + require.NoError(t, err) + require.Empty(t, candidates) + require.Empty(t, occupied) + require.Empty(t, complete) + _, err = archive.RecordItemRunLogs(ctx, expt, run, item, space, exptDO) + require.NoError(t, err) + stats, err = statsRepo.Get(ctx, expt, space) + require.NoError(t, err) + require.Equal(t, before, *stats) + turns, err := itemRepo.GetItemTurnResults(ctx, space, expt, item) + require.NoError(t, err) + require.Nil(t, turns[0].WeightedScore, "no score record writes NULL instead of retaining old0") + readFailure := errors.New("injected primary query unavailable") + callbackName := "scan_transition_read_failure" + require.NoError(t, conn.Callback().Query().Before("gorm:query").Register(callbackName, func(tx *gorm.DB) { _ = tx.AddError(readFailure) })) + manager := svcmocks.NewMockIExptManager(ctrl) + scheduler := &ExptSchedulerImpl{Manager: manager} + brokerErr := scheduler.HandleEventErr(func(ctx context.Context, event *entity.ExptScheduleEvent) error { + _, _, _, err := scan.ScanEvalItems(ctx, event, exptDO) + return err + })(ctx, scheduleEvent) + require.NoError(t, conn.Callback().Query().Remove(callbackName)) + require.True(t, errno.NeedMQRetry(brokerErr)) + require.ErrorIs(t, brokerErr, readFailure) + require.Zero(t, scheduleEvent.InfraErrorRetryTimes) + stats, err = statsRepo.Get(ctx, expt, space) + require.NoError(t, err) + require.Equal(t, before, *stats) + t.Log("REAL_MYSQL_SCAN_READ_ERROR broker retry retained cause, no finish/state/stats effects") + t.Logf("REAL_MYSQL_COMPLETE_SCAN_ARCHIVE status=%d canonical Processing -> logged scan -> Resulted; stats once, score0 preserved", status) + }) + } +} + +func TestScanTransitionStorageErrorBrokerBoundary(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + manager := svcmocks.NewMockIExptManager(ctrl) + want := errors.New("primary source read failed") + items.EXPECT().ScanCurrentRequests(gomock.Any(), int64(3), int64(1), int64(2), false).Return(nil, want) + scan := &exptBaseExec{exptItemResultRepo: items} + scheduler := &ExptSchedulerImpl{Manager: manager} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit, InfraErrorRetryTimes: 7} + err := scheduler.HandleEventErr(func(ctx context.Context, event *entity.ExptScheduleEvent) error { + _, _, _, err := scan.ScanEvalItems(ctx, event, nil) + require.True(t, errno.NeedMQRetry(err), "storage observation failure must use broker retry") + require.ErrorIs(t, err, want) + return err + })(context.Background(), event) + require.True(t, errno.NeedMQRetry(err)) + require.ErrorIs(t, err, want) + require.Equal(t, 7, event.InfraErrorRetryTimes) +} + +func TestScanTransitionBusinessErrorKeepsFinishPolicy(t *testing.T) { + ctrl := gomock.NewController(t) + manager := svcmocks.NewMockIExptManager(ctrl) + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeUnknown} + want := errors.New("unknown expt mode: 0") + manager.EXPECT().CompleteRun(gomock.Any(), int64(1), int64(2), int64(3), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + manager.EXPECT().CompleteExpt(gomock.Any(), int64(1), gomock.Any(), int64(3), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + scheduler := &ExptSchedulerImpl{Manager: manager} + require.NoError(t, scheduler.HandleEventErr(func(context.Context, *entity.ExptScheduleEvent) error { return want })(context.Background(), event)) + require.Zero(t, event.InfraErrorRetryTimes) +} + +func TestScanTransitionNilStorageObservationRetries(t *testing.T) { + ctrl := gomock.NewController(t) + items := repomocks.NewMockIExptItemResultRepo(ctrl) + items.EXPECT().ScanCurrentRequests(gomock.Any(), int64(3), int64(1), int64(2), false).Return([]*entity.ExptItemCurrentRequest{nil}, nil) + scan := &exptBaseExec{exptItemResultRepo: items} + _, _, _, err := scan.ScanEvalItems(context.Background(), &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2}, nil) + require.True(t, errno.NeedMQRetry(err)) +} diff --git a/backend/modules/evaluation/domain/service/expt_stats_consistency_test.go b/backend/modules/evaluation/domain/service/expt_stats_consistency_test.go index d9df0b0887..e8b30a8661 100644 --- a/backend/modules/evaluation/domain/service/expt_stats_consistency_test.go +++ b/backend/modules/evaluation/domain/service/expt_stats_consistency_test.go @@ -6,6 +6,7 @@ package service import ( "context" "errors" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -14,6 +15,7 @@ import ( idemMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" repoMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" "github.com/coze-dev/coze-loop/backend/pkg/ctxcache" ) @@ -38,137 +40,24 @@ func centralAdmittedCtx(t *testing.T) context.Context { return ctx } -// TestCentralReservation_StatsFollowsMainTableNotCAS 核心回归： -// run log CAS 未命中（started=false），但主表还停在 Queueing 时，**stats 记账仍然必须发生**。 -// -// 旧代码在这里直接跳过，于是那条 item 的 Queueing 永远不减 —— 这正是线上 pending 虚高的来源。 -func TestCentralReservation_StatsFollowsMainTableNotCAS(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - items := repoMocks.NewMockIExptItemResultRepo(ctrl) - stats := repoMocks.NewMockIExptStatsRepo(ctrl) - - // CAS 未命中：重复投递或已被 repair 修正。 - dispatch.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(false, nil) - // 执行链返回后的释放路径会查一次投影，与本用例无关。 - dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil, nil).AnyTimes() - // 但主表仍是 Queueing —— 说明这条 item 从来没被记进 Processing 桶。 - items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: entity.ItemRunState_Queueing}}, nil) - items.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - - var gotOp *entity.StatsCntArithOp - stats.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, _, _ int64, op *entity.StatsCntArithOp) error { - gotOp = op - return nil +// Start commits stats with canonical; consumer must not write either again. +func TestCentralReservation_NoSecondStatsWrite(t *testing.T) { + for _, started := range []bool{false, true} { + t.Run(fmt.Sprint(started), func(t *testing.T) { + ctrl := gomock.NewController(t) + dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) + dispatch.EXPECT().StartReservedItem(gomock.Any(), int64(3), int64(1), int64(2), int64(4), "rq1").Return(repo.StartReservedItemResult{CurrentRequest: true, Started: started}, nil) + dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + svc := &ExptItemEventEvalServiceImpl{ + centralGuard: &fakeGuard{confirmResult: true}, dispatchRepo: dispatch, + exptItemResultRepo: repoMocks.NewMockIExptItemResultRepo(ctrl), exptStatsRepo: repoMocks.NewMockIExptStatsRepo(ctrl), + } + svc.exptItemResultRepo.(*repoMocks.MockIExptItemResultRepo).EXPECT().GetCurrentRequest(gomock.Any(), int64(3), int64(1), int64(2), int64(4)).Return(&entity.ExptItemCurrentRequest{RunID: 2, ItemID: 4, RequestID: "rq1", Status: entity.ItemRunState_Processing}, nil) + called := false + require.NoError(t, svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { called = true; return nil })(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 4, RequestID: "rq1"})) + require.True(t, called) }) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: &fakeGuard{confirmResult: true}, - dispatchRepo: dispatch, - exptItemResultRepo: items, - exptStatsRepo: stats, - } - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) - - require.NotNil(t, gotOp, "CAS 未命中但主表仍在 Queueing 时必须记账 —— 跳过就是线上 pending 虚高的成因") - assert.Equal(t, 1, gotOp.OpStatusCnt[entity.ItemRunState_Processing]) - assert.Equal(t, -1, gotOp.OpStatusCnt[entity.ItemRunState_Queueing]) -} - -// TestCentralReservation_NoStatsOpWhenMainTableAlreadyProcessing 主表已是 Processing 时 -// 不得重复记账 —— 那会从"少计"变成"多计"，同样让计数行歪掉。 -func TestCentralReservation_NoStatsOpWhenMainTableAlreadyProcessing(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - items := repoMocks.NewMockIExptItemResultRepo(ctrl) - stats := repoMocks.NewMockIExptStatsRepo(ctrl) - - dispatch.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(true, nil) // CAS 命中，但主表已经是 Processing - dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil, nil).AnyTimes() - items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: entity.ItemRunState_Processing}}, nil) - // 主表不该被重写，stats 不该被记账 - items.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - stats.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: &fakeGuard{confirmResult: true}, dispatchRepo: dispatch, - exptItemResultRepo: items, exptStatsRepo: stats, - } - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) -} - -// TestCentralReservation_StatsDecrementsActualBucket 主表停在非 Queueing 的状态（repair / -// 重投留下的）时，减的必须是**那个**桶，不能写死 Queueing —— 减错桶与不减一样会让计数行歪。 -func TestCentralReservation_StatsDecrementsActualBucket(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - items := repoMocks.NewMockIExptItemResultRepo(ctrl) - stats := repoMocks.NewMockIExptStatsRepo(ctrl) - - dispatch.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) - dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil, nil).AnyTimes() - items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: entity.ItemRunState_Fail}}, nil) - items.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - - var gotOp *entity.StatsCntArithOp - stats.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, _, _ int64, op *entity.StatsCntArithOp) error { gotOp = op; return nil }) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: &fakeGuard{confirmResult: true}, dispatchRepo: dispatch, - exptItemResultRepo: items, exptStatsRepo: stats, - } - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) - - require.NotNil(t, gotOp) - assert.Equal(t, -1, gotOp.OpStatusCnt[entity.ItemRunState_Fail], "减的必须是主表实际所在的桶") - assert.Equal(t, 0, gotOp.OpStatusCnt[entity.ItemRunState_Queueing], "不得写死 Queueing") -} - -// TestCentralReservation_NoStatsOpWhenMainTableWriteFails 主表没写进去就不该记账 —— -// 否则计数行跑到主表前面，完成侧的「-1」又会减错。 -func TestCentralReservation_NoStatsOpWhenMainTableWriteFails(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - dispatch := repoMocks.NewMockIExptItemDispatchRepo(ctrl) - items := repoMocks.NewMockIExptItemResultRepo(ctrl) - stats := repoMocks.NewMockIExptStatsRepo(ctrl) - - dispatch.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) - dispatch.EXPECT().MGetDispatchObservations(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil, nil).AnyTimes() - items.EXPECT().BatchGet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptItemResult{{ItemID: 4, Status: entity.ItemRunState_Queueing}}, nil) - items.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(errors.New("db down")) - stats.EXPECT().ArithOperateCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptItemEventEvalServiceImpl{ - centralGuard: &fakeGuard{confirmResult: true}, dispatchRepo: dispatch, - exptItemResultRepo: items, exptStatsRepo: stats, } - handler := svc.HandleCentralReservation(func(context.Context, *entity.ExptItemEvalEvent) error { return nil }) - // 主表是展示投影，写失败不阻断执行 —— 但也不能记账。 - require.NoError(t, handler(centralAdmittedCtx(t), &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, SpaceID: 3, EvalSetItemID: 4})) } // ---------- 每拍对账 ---------- diff --git a/backend/modules/evaluation/domain/service/expt_terminal_runlog_and_retry_release_test.go b/backend/modules/evaluation/domain/service/expt_terminal_runlog_and_retry_release_test.go index 3073d9bc70..4baf674162 100644 --- a/backend/modules/evaluation/domain/service/expt_terminal_runlog_and_retry_release_test.go +++ b/backend/modules/evaluation/domain/service/expt_terminal_runlog_and_retry_release_test.go @@ -292,7 +292,7 @@ func TestLogRetryItemsRun_ReleasesSupersededRunQuota(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker).EXPECT(). BackoffLockWithValue(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(true, "", nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) mgr.exptRepo.(*repoMocks.MockIExperimentRepo).EXPECT(). GetByID(gomock.Any(), exptID, spaceID).Return(&entity.Experiment{ ID: exptID, @@ -337,9 +337,7 @@ func TestLogRetryItemsRun_SkipsReleaseWhenRunAlive(t *testing.T) { BackoffLockWithValue(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(false, "200", nil) mgr.mutex.(*lockMocks.MockILocker).EXPECT().Exists(gomock.Any(), gomock.Any()).Return(false, nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT(). - Get(gomock.Any(), exptID, aliveRunID).Return(&entity.ExptRunLog{ExptID: exptID, ExptRunID: aliveRunID}, nil) - mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo).EXPECT().AppendRunItemIDs(gomock.Any(), exptID, aliveRunID, spaceID, []int64{11}).Return(nil) // 刻意不声明 GetByID / ScanItemRunLogs：真去查就是 unexpected call。 runID, retried, err := mgr.LogRetryItemsRun(context.Background(), exptID, diff --git a/backend/modules/evaluation/domain/service/mocks/expt_manage.go b/backend/modules/evaluation/domain/service/mocks/expt_manage.go index cb9ab8f1c6..51a2272cf4 100644 --- a/backend/modules/evaluation/domain/service/mocks/expt_manage.go +++ b/backend/modules/evaluation/domain/service/mocks/expt_manage.go @@ -21,6 +21,7 @@ import ( type MockIExptManager struct { ctrl *gomock.Controller recorder *MockIExptManagerMockRecorder + isgomock struct{} } // MockIExptManagerMockRecorder is the mock recorder for MockIExptManager. @@ -41,40 +42,40 @@ func (m *MockIExptManager) EXPECT() *MockIExptManagerMockRecorder { } // CheckGroupKey mocks base method. -func (m *MockIExptManager) CheckGroupKey(arg0 context.Context, arg1 string, arg2 int64, arg3 *entity.Session) (bool, error) { +func (m *MockIExptManager) CheckGroupKey(ctx context.Context, groupKey string, spaceID int64, session *entity.Session) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckGroupKey", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "CheckGroupKey", ctx, groupKey, spaceID, session) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // CheckGroupKey indicates an expected call of CheckGroupKey. -func (mr *MockIExptManagerMockRecorder) CheckGroupKey(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) CheckGroupKey(ctx, groupKey, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckGroupKey", reflect.TypeOf((*MockIExptManager)(nil).CheckGroupKey), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckGroupKey", reflect.TypeOf((*MockIExptManager)(nil).CheckGroupKey), ctx, groupKey, spaceID, session) } // CheckName mocks base method. -func (m *MockIExptManager) CheckName(arg0 context.Context, arg1 string, arg2 int64, arg3 *entity.Session) (bool, error) { +func (m *MockIExptManager) CheckName(ctx context.Context, name string, spaceID int64, session *entity.Session) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckName", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "CheckName", ctx, name, spaceID, session) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // CheckName indicates an expected call of CheckName. -func (mr *MockIExptManagerMockRecorder) CheckName(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) CheckName(ctx, name, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckName", reflect.TypeOf((*MockIExptManager)(nil).CheckName), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckName", reflect.TypeOf((*MockIExptManager)(nil).CheckName), ctx, name, spaceID, session) } // CheckRun mocks base method. -func (m *MockIExptManager) CheckRun(arg0 context.Context, arg1 *entity.Experiment, arg2 int64, arg3 *entity.Session, arg4 ...entity.ExptRunCheckOptionFn) error { +func (m *MockIExptManager) CheckRun(ctx context.Context, expt *entity.Experiment, spaceID int64, session *entity.Session, opts ...entity.ExptRunCheckOptionFn) error { m.ctrl.T.Helper() - varargs := []any{arg0, arg1, arg2, arg3} - for _, a := range arg4 { + varargs := []any{ctx, expt, spaceID, session} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CheckRun", varargs...) @@ -83,32 +84,32 @@ func (m *MockIExptManager) CheckRun(arg0 context.Context, arg1 *entity.Experimen } // CheckRun indicates an expected call of CheckRun. -func (mr *MockIExptManagerMockRecorder) CheckRun(arg0, arg1, arg2, arg3 any, arg4 ...any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) CheckRun(ctx, expt, spaceID, session any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1, arg2, arg3}, arg4...) + varargs := append([]any{ctx, expt, spaceID, session}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckRun", reflect.TypeOf((*MockIExptManager)(nil).CheckRun), varargs...) } // Clone mocks base method. -func (m *MockIExptManager) Clone(arg0 context.Context, arg1, arg2 int64, arg3 *entity.Session) (*entity.Experiment, error) { +func (m *MockIExptManager) Clone(ctx context.Context, exptID, spaceID int64, session *entity.Session) (*entity.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Clone", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "Clone", ctx, exptID, spaceID, session) ret0, _ := ret[0].(*entity.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // Clone indicates an expected call of Clone. -func (mr *MockIExptManagerMockRecorder) Clone(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Clone(ctx, exptID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clone", reflect.TypeOf((*MockIExptManager)(nil).Clone), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clone", reflect.TypeOf((*MockIExptManager)(nil).Clone), ctx, exptID, spaceID, session) } // CompleteExpt mocks base method. -func (m *MockIExptManager) CompleteExpt(arg0 context.Context, arg1 int64, arg2 *int64, arg3 int64, arg4 *entity.Session, arg5 ...entity.CompleteExptOptionFn) error { +func (m *MockIExptManager) CompleteExpt(ctx context.Context, exptID int64, exptRunID *int64, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error { m.ctrl.T.Helper() - varargs := []any{arg0, arg1, arg2, arg3, arg4} - for _, a := range arg5 { + varargs := []any{ctx, exptID, exptRunID, spaceID, session} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CompleteExpt", varargs...) @@ -117,17 +118,17 @@ func (m *MockIExptManager) CompleteExpt(arg0 context.Context, arg1 int64, arg2 * } // CompleteExpt indicates an expected call of CompleteExpt. -func (mr *MockIExptManagerMockRecorder) CompleteExpt(arg0, arg1, arg2, arg3, arg4 any, arg5 ...any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) CompleteExpt(ctx, exptID, exptRunID, spaceID, session any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1, arg2, arg3, arg4}, arg5...) + varargs := append([]any{ctx, exptID, exptRunID, spaceID, session}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteExpt", reflect.TypeOf((*MockIExptManager)(nil).CompleteExpt), varargs...) } // CompleteRun mocks base method. -func (m *MockIExptManager) CompleteRun(arg0 context.Context, arg1, arg2, arg3 int64, arg4 *entity.Session, arg5 ...entity.CompleteExptOptionFn) error { +func (m *MockIExptManager) CompleteRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error { m.ctrl.T.Helper() - varargs := []any{arg0, arg1, arg2, arg3, arg4} - for _, a := range arg5 { + varargs := []any{ctx, exptID, exptRunID, spaceID, session} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CompleteRun", varargs...) @@ -136,90 +137,90 @@ func (m *MockIExptManager) CompleteRun(arg0 context.Context, arg1, arg2, arg3 in } // CompleteRun indicates an expected call of CompleteRun. -func (mr *MockIExptManagerMockRecorder) CompleteRun(arg0, arg1, arg2, arg3, arg4 any, arg5 ...any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) CompleteRun(ctx, exptID, exptRunID, spaceID, session any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1, arg2, arg3, arg4}, arg5...) + varargs := append([]any{ctx, exptID, exptRunID, spaceID, session}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteRun", reflect.TypeOf((*MockIExptManager)(nil).CompleteRun), varargs...) } // CreateExpt mocks base method. -func (m *MockIExptManager) CreateExpt(arg0 context.Context, arg1 *entity.CreateExptParam, arg2 *entity.Session) (*entity.Experiment, error) { +func (m *MockIExptManager) CreateExpt(ctx context.Context, req *entity.CreateExptParam, session *entity.Session) (*entity.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateExpt", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "CreateExpt", ctx, req, session) ret0, _ := ret[0].(*entity.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateExpt indicates an expected call of CreateExpt. -func (mr *MockIExptManagerMockRecorder) CreateExpt(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) CreateExpt(ctx, req, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateExpt", reflect.TypeOf((*MockIExptManager)(nil).CreateExpt), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateExpt", reflect.TypeOf((*MockIExptManager)(nil).CreateExpt), ctx, req, session) } // Delete mocks base method. -func (m *MockIExptManager) Delete(arg0 context.Context, arg1, arg2 int64, arg3 *entity.Session) error { +func (m *MockIExptManager) Delete(ctx context.Context, exptID, spaceID int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "Delete", ctx, exptID, spaceID, session) ret0, _ := ret[0].(error) return ret0 } // Delete indicates an expected call of Delete. -func (mr *MockIExptManagerMockRecorder) Delete(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Delete(ctx, exptID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockIExptManager)(nil).Delete), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockIExptManager)(nil).Delete), ctx, exptID, spaceID, session) } // ExistCompletingRunLock mocks base method. -func (m *MockIExptManager) ExistCompletingRunLock(arg0 context.Context, arg1, arg2, arg3 int64) (bool, error) { +func (m *MockIExptManager) ExistCompletingRunLock(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExistCompletingRunLock", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "ExistCompletingRunLock", ctx, exptID, exptRunID, spaceID) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // ExistCompletingRunLock indicates an expected call of ExistCompletingRunLock. -func (mr *MockIExptManagerMockRecorder) ExistCompletingRunLock(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) ExistCompletingRunLock(ctx, exptID, exptRunID, spaceID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExistCompletingRunLock", reflect.TypeOf((*MockIExptManager)(nil).ExistCompletingRunLock), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExistCompletingRunLock", reflect.TypeOf((*MockIExptManager)(nil).ExistCompletingRunLock), ctx, exptID, exptRunID, spaceID) } // Finish mocks base method. -func (m *MockIExptManager) Finish(arg0 context.Context, arg1 *entity.Experiment, arg2 int64, arg3 *entity.Session) error { +func (m *MockIExptManager) Finish(ctx context.Context, exptID *entity.Experiment, exptRunID int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Finish", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "Finish", ctx, exptID, exptRunID, session) ret0, _ := ret[0].(error) return ret0 } // Finish indicates an expected call of Finish. -func (mr *MockIExptManagerMockRecorder) Finish(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Finish(ctx, exptID, exptRunID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockIExptManager)(nil).Finish), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockIExptManager)(nil).Finish), ctx, exptID, exptRunID, session) } // Get mocks base method. -func (m *MockIExptManager) Get(arg0 context.Context, arg1, arg2 int64, arg3 *entity.Session) (*entity.Experiment, error) { +func (m *MockIExptManager) Get(ctx context.Context, exptID, spaceID int64, session *entity.Session) (*entity.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "Get", ctx, exptID, spaceID, session) ret0, _ := ret[0].(*entity.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // Get indicates an expected call of Get. -func (mr *MockIExptManagerMockRecorder) Get(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Get(ctx, exptID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIExptManager)(nil).Get), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIExptManager)(nil).Get), ctx, exptID, spaceID, session) } // GetDetail mocks base method. -func (m *MockIExptManager) GetDetail(arg0 context.Context, arg1, arg2 int64, arg3 *entity.Session, arg4 ...entity.GetExptTupleOptionFn) (*entity.Experiment, error) { +func (m *MockIExptManager) GetDetail(ctx context.Context, exptID, spaceID int64, session *entity.Session, opts ...entity.GetExptTupleOptionFn) (*entity.Experiment, error) { m.ctrl.T.Helper() - varargs := []any{arg0, arg1, arg2, arg3} - for _, a := range arg4 { + varargs := []any{ctx, exptID, spaceID, session} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetDetail", varargs...) @@ -229,16 +230,16 @@ func (m *MockIExptManager) GetDetail(arg0 context.Context, arg1, arg2 int64, arg } // GetDetail indicates an expected call of GetDetail. -func (mr *MockIExptManagerMockRecorder) GetDetail(arg0, arg1, arg2, arg3 any, arg4 ...any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) GetDetail(ctx, exptID, spaceID, session any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1, arg2, arg3}, arg4...) + varargs := append([]any{ctx, exptID, spaceID, session}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDetail", reflect.TypeOf((*MockIExptManager)(nil).GetDetail), varargs...) } // GetIDsByGroupKey mocks base method. -func (m *MockIExptManager) GetIDsByGroupKey(arg0 context.Context, arg1 int64, arg2 string, arg3, arg4 int32, arg5 *entity.Session) ([]int64, int64, error) { +func (m *MockIExptManager) GetIDsByGroupKey(ctx context.Context, spaceID int64, groupKey string, page, pageSize int32, session *entity.Session) ([]int64, int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetIDsByGroupKey", arg0, arg1, arg2, arg3, arg4, arg5) + ret := m.ctrl.Call(m, "GetIDsByGroupKey", ctx, spaceID, groupKey, page, pageSize, session) ret0, _ := ret[0].([]int64) ret1, _ := ret[1].(int64) ret2, _ := ret[2].(error) @@ -246,56 +247,56 @@ func (m *MockIExptManager) GetIDsByGroupKey(arg0 context.Context, arg1 int64, ar } // GetIDsByGroupKey indicates an expected call of GetIDsByGroupKey. -func (mr *MockIExptManagerMockRecorder) GetIDsByGroupKey(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) GetIDsByGroupKey(ctx, spaceID, groupKey, page, pageSize, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDsByGroupKey", reflect.TypeOf((*MockIExptManager)(nil).GetIDsByGroupKey), arg0, arg1, arg2, arg3, arg4, arg5) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDsByGroupKey", reflect.TypeOf((*MockIExptManager)(nil).GetIDsByGroupKey), ctx, spaceID, groupKey, page, pageSize, session) } // GetRunLog mocks base method. -func (m *MockIExptManager) GetRunLog(arg0 context.Context, arg1, arg2, arg3 int64, arg4 *entity.Session) (*entity.ExptRunLog, error) { +func (m *MockIExptManager) GetRunLog(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) (*entity.ExptRunLog, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRunLog", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "GetRunLog", ctx, exptID, exptRunID, spaceID, session) ret0, _ := ret[0].(*entity.ExptRunLog) ret1, _ := ret[1].(error) return ret0, ret1 } // GetRunLog indicates an expected call of GetRunLog. -func (mr *MockIExptManagerMockRecorder) GetRunLog(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) GetRunLog(ctx, exptID, exptRunID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunLog", reflect.TypeOf((*MockIExptManager)(nil).GetRunLog), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunLog", reflect.TypeOf((*MockIExptManager)(nil).GetRunLog), ctx, exptID, exptRunID, spaceID, session) } // InjectExptConfTimeRange mocks base method. -func (m *MockIExptManager) InjectExptConfTimeRange(arg0 context.Context, arg1 int64, arg2 *entity.TaskTimeRangeDO) { +func (m *MockIExptManager) InjectExptConfTimeRange(ctx context.Context, exptID int64, timeRange *entity.TaskTimeRangeDO) { m.ctrl.T.Helper() - m.ctrl.Call(m, "InjectExptConfTimeRange", arg0, arg1, arg2) + m.ctrl.Call(m, "InjectExptConfTimeRange", ctx, exptID, timeRange) } // InjectExptConfTimeRange indicates an expected call of InjectExptConfTimeRange. -func (mr *MockIExptManagerMockRecorder) InjectExptConfTimeRange(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) InjectExptConfTimeRange(ctx, exptID, timeRange any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InjectExptConfTimeRange", reflect.TypeOf((*MockIExptManager)(nil).InjectExptConfTimeRange), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InjectExptConfTimeRange", reflect.TypeOf((*MockIExptManager)(nil).InjectExptConfTimeRange), ctx, exptID, timeRange) } // Invoke mocks base method. -func (m *MockIExptManager) Invoke(arg0 context.Context, arg1 *entity.InvokeExptReq) error { +func (m *MockIExptManager) Invoke(ctx context.Context, invokeExptReq *entity.InvokeExptReq) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Invoke", arg0, arg1) + ret := m.ctrl.Call(m, "Invoke", ctx, invokeExptReq) ret0, _ := ret[0].(error) return ret0 } // Invoke indicates an expected call of Invoke. -func (mr *MockIExptManagerMockRecorder) Invoke(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Invoke(ctx, invokeExptReq any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Invoke", reflect.TypeOf((*MockIExptManager)(nil).Invoke), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Invoke", reflect.TypeOf((*MockIExptManager)(nil).Invoke), ctx, invokeExptReq) } // List mocks base method. -func (m *MockIExptManager) List(arg0 context.Context, arg1, arg2 int32, arg3 int64, arg4 *entity.ExptListFilter, arg5 []*entity.OrderBy, arg6 *entity.Session) ([]*entity.Experiment, int64, error) { +func (m *MockIExptManager) List(ctx context.Context, page, pageSize int32, spaceID int64, filter *entity.ExptListFilter, orders []*entity.OrderBy, session *entity.Session) ([]*entity.Experiment, int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5, arg6) + ret := m.ctrl.Call(m, "List", ctx, page, pageSize, spaceID, filter, orders, session) ret0, _ := ret[0].([]*entity.Experiment) ret1, _ := ret[1].(int64) ret2, _ := ret[2].(error) @@ -303,15 +304,15 @@ func (m *MockIExptManager) List(arg0 context.Context, arg1, arg2 int32, arg3 int } // List indicates an expected call of List. -func (mr *MockIExptManagerMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5, arg6 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) List(ctx, page, pageSize, spaceID, filter, orders, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIExptManager)(nil).List), arg0, arg1, arg2, arg3, arg4, arg5, arg6) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIExptManager)(nil).List), ctx, page, pageSize, spaceID, filter, orders, session) } // ListExptRaw mocks base method. -func (m *MockIExptManager) ListExptRaw(arg0 context.Context, arg1, arg2 int32, arg3 int64, arg4 *entity.ExptListFilter) ([]*entity.Experiment, int64, error) { +func (m *MockIExptManager) ListExptRaw(ctx context.Context, page, pageSize int32, spaceID int64, filter *entity.ExptListFilter) ([]*entity.Experiment, int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListExptRaw", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "ListExptRaw", ctx, page, pageSize, spaceID, filter) ret0, _ := ret[0].([]*entity.Experiment) ret1, _ := ret[1].(int64) ret2, _ := ret[2].(error) @@ -319,29 +320,29 @@ func (m *MockIExptManager) ListExptRaw(arg0 context.Context, arg1, arg2 int32, a } // ListExptRaw indicates an expected call of ListExptRaw. -func (mr *MockIExptManagerMockRecorder) ListExptRaw(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) ListExptRaw(ctx, page, pageSize, spaceID, filter any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListExptRaw", reflect.TypeOf((*MockIExptManager)(nil).ListExptRaw), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListExptRaw", reflect.TypeOf((*MockIExptManager)(nil).ListExptRaw), ctx, page, pageSize, spaceID, filter) } // LockCompletingRun mocks base method. -func (m *MockIExptManager) LockCompletingRun(arg0 context.Context, arg1, arg2, arg3 int64, arg4 *entity.Session) error { +func (m *MockIExptManager) LockCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LockCompletingRun", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "LockCompletingRun", ctx, exptID, exptRunID, spaceID, session) ret0, _ := ret[0].(error) return ret0 } // LockCompletingRun indicates an expected call of LockCompletingRun. -func (mr *MockIExptManagerMockRecorder) LockCompletingRun(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) LockCompletingRun(ctx, exptID, exptRunID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockCompletingRun", reflect.TypeOf((*MockIExptManager)(nil).LockCompletingRun), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockCompletingRun", reflect.TypeOf((*MockIExptManager)(nil).LockCompletingRun), ctx, exptID, exptRunID, spaceID, session) } // LogRetryItemsRun mocks base method. -func (m *MockIExptManager) LogRetryItemsRun(arg0 context.Context, arg1 int64, arg2 entity.ExptRunMode, arg3 int64, arg4 []int64, arg5 *entity.Session) (int64, bool, error) { +func (m *MockIExptManager) LogRetryItemsRun(ctx context.Context, exptID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (int64, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LogRetryItemsRun", arg0, arg1, arg2, arg3, arg4, arg5) + ret := m.ctrl.Call(m, "LogRetryItemsRun", ctx, exptID, mode, spaceID, itemIDs, session) ret0, _ := ret[0].(int64) ret1, _ := ret[1].(bool) ret2, _ := ret[2].(error) @@ -349,192 +350,207 @@ func (m *MockIExptManager) LogRetryItemsRun(arg0 context.Context, arg1 int64, ar } // LogRetryItemsRun indicates an expected call of LogRetryItemsRun. -func (mr *MockIExptManagerMockRecorder) LogRetryItemsRun(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) LogRetryItemsRun(ctx, exptID, mode, spaceID, itemIDs, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRetryItemsRun", reflect.TypeOf((*MockIExptManager)(nil).LogRetryItemsRun), arg0, arg1, arg2, arg3, arg4, arg5) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRetryItemsRun", reflect.TypeOf((*MockIExptManager)(nil).LogRetryItemsRun), ctx, exptID, mode, spaceID, itemIDs, session) } // LogRun mocks base method. -func (m *MockIExptManager) LogRun(arg0 context.Context, arg1, arg2 int64, arg3 entity.ExptRunMode, arg4 int64, arg5 []int64, arg6 *entity.Session) error { +func (m *MockIExptManager) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LogRun", arg0, arg1, arg2, arg3, arg4, arg5, arg6) + ret := m.ctrl.Call(m, "LogRun", ctx, exptID, exptRunID, mode, spaceID, itemIDs, session) ret0, _ := ret[0].(error) return ret0 } // LogRun indicates an expected call of LogRun. -func (mr *MockIExptManagerMockRecorder) LogRun(arg0, arg1, arg2, arg3, arg4, arg5, arg6 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) LogRun(ctx, exptID, exptRunID, mode, spaceID, itemIDs, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRun", reflect.TypeOf((*MockIExptManager)(nil).LogRun), arg0, arg1, arg2, arg3, arg4, arg5, arg6) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRun", reflect.TypeOf((*MockIExptManager)(nil).LogRun), ctx, exptID, exptRunID, mode, spaceID, itemIDs, session) } // MDelete mocks base method. -func (m *MockIExptManager) MDelete(arg0 context.Context, arg1 []int64, arg2 int64, arg3 *entity.Session) error { +func (m *MockIExptManager) MDelete(ctx context.Context, exptIDs []int64, spaceID int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MDelete", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "MDelete", ctx, exptIDs, spaceID, session) ret0, _ := ret[0].(error) return ret0 } // MDelete indicates an expected call of MDelete. -func (mr *MockIExptManagerMockRecorder) MDelete(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) MDelete(ctx, exptIDs, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MDelete", reflect.TypeOf((*MockIExptManager)(nil).MDelete), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MDelete", reflect.TypeOf((*MockIExptManager)(nil).MDelete), ctx, exptIDs, spaceID, session) } // MGet mocks base method. -func (m *MockIExptManager) MGet(arg0 context.Context, arg1 []int64, arg2 int64, arg3 *entity.Session) ([]*entity.Experiment, error) { +func (m *MockIExptManager) MGet(ctx context.Context, exptIDs []int64, spaceID int64, session *entity.Session) ([]*entity.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MGet", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "MGet", ctx, exptIDs, spaceID, session) ret0, _ := ret[0].([]*entity.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // MGet indicates an expected call of MGet. -func (mr *MockIExptManagerMockRecorder) MGet(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) MGet(ctx, exptIDs, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGet", reflect.TypeOf((*MockIExptManager)(nil).MGet), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGet", reflect.TypeOf((*MockIExptManager)(nil).MGet), ctx, exptIDs, spaceID, session) } // MGetBasicByID mocks base method. -func (m *MockIExptManager) MGetBasicByID(arg0 context.Context, arg1 []int64) ([]*entity.Experiment, error) { +func (m *MockIExptManager) MGetBasicByID(ctx context.Context, exptIDs []int64) ([]*entity.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MGetBasicByID", arg0, arg1) + ret := m.ctrl.Call(m, "MGetBasicByID", ctx, exptIDs) ret0, _ := ret[0].([]*entity.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // MGetBasicByID indicates an expected call of MGetBasicByID. -func (mr *MockIExptManagerMockRecorder) MGetBasicByID(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) MGetBasicByID(ctx, exptIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetBasicByID", reflect.TypeOf((*MockIExptManager)(nil).MGetBasicByID), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetBasicByID", reflect.TypeOf((*MockIExptManager)(nil).MGetBasicByID), ctx, exptIDs) } // MGetDetail mocks base method. -func (m *MockIExptManager) MGetDetail(arg0 context.Context, arg1 []int64, arg2 int64, arg3 *entity.Session) ([]*entity.Experiment, error) { +func (m *MockIExptManager) MGetDetail(ctx context.Context, exptIDs []int64, spaceID int64, session *entity.Session) ([]*entity.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MGetDetail", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "MGetDetail", ctx, exptIDs, spaceID, session) ret0, _ := ret[0].([]*entity.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // MGetDetail indicates an expected call of MGetDetail. -func (mr *MockIExptManagerMockRecorder) MGetDetail(arg0, arg1, arg2, arg3 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) MGetDetail(ctx, exptIDs, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetDetail", reflect.TypeOf((*MockIExptManager)(nil).MGetDetail), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetDetail", reflect.TypeOf((*MockIExptManager)(nil).MGetDetail), ctx, exptIDs, spaceID, session) } // RecordExptData mocks base method. -func (m *MockIExptManager) RecordExptData(arg0 context.Context, arg1, arg2, arg3 int64, arg4 *entity.Session) error { +func (m *MockIExptManager) RecordExptData(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecordExptData", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "RecordExptData", ctx, exptID, exptRunID, spaceID, session) ret0, _ := ret[0].(error) return ret0 } // RecordExptData indicates an expected call of RecordExptData. -func (mr *MockIExptManagerMockRecorder) RecordExptData(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) RecordExptData(ctx, exptID, exptRunID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordExptData", reflect.TypeOf((*MockIExptManager)(nil).RecordExptData), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordExptData", reflect.TypeOf((*MockIExptManager)(nil).RecordExptData), ctx, exptID, exptRunID, spaceID, session) +} + +// RetryInCurrentRun mocks base method. +func (m *MockIExptManager) RetryInCurrentRun(arg0 context.Context, arg1 *entity.RetryInCurrentRunRequest) (*entity.ManualRetryReceipt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RetryInCurrentRun", arg0, arg1) + ret0, _ := ret[0].(*entity.ManualRetryReceipt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RetryInCurrentRun indicates an expected call of RetryInCurrentRun. +func (mr *MockIExptManagerMockRecorder) RetryInCurrentRun(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryInCurrentRun", reflect.TypeOf((*MockIExptManager)(nil).RetryInCurrentRun), arg0, arg1) } // RetryItems mocks base method. -func (m *MockIExptManager) RetryItems(arg0 context.Context, arg1, arg2, arg3 int64, arg4 int, arg5 []int64, arg6 *entity.Session, arg7 map[string]string) error { +func (m *MockIExptManager) RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RetryItems", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret := m.ctrl.Call(m, "RetryItems", ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) ret0, _ := ret[0].(error) return ret0 } // RetryItems indicates an expected call of RetryItems. -func (mr *MockIExptManagerMockRecorder) RetryItems(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) RetryItems(ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryItems", reflect.TypeOf((*MockIExptManager)(nil).RetryItems), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryItems", reflect.TypeOf((*MockIExptManager)(nil).RetryItems), ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) } // Run mocks base method. -func (m *MockIExptManager) Run(arg0 context.Context, arg1, arg2, arg3 int64, arg4 int, arg5 *entity.Session, arg6 entity.ExptRunMode, arg7 map[string]string) error { +func (m *MockIExptManager) Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Run", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret := m.ctrl.Call(m, "Run", ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext) ret0, _ := ret[0].(error) return ret0 } // Run indicates an expected call of Run. -func (mr *MockIExptManagerMockRecorder) Run(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Run(ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIExptManager)(nil).Run), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIExptManager)(nil).Run), ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext) } // SetExptTerminating mocks base method. -func (m *MockIExptManager) SetExptTerminating(arg0 context.Context, arg1, arg2, arg3 int64, arg4 *entity.Session) error { +func (m *MockIExptManager) SetExptTerminating(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetExptTerminating", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "SetExptTerminating", ctx, exptID, exptRunID, spaceID, session) ret0, _ := ret[0].(error) return ret0 } // SetExptTerminating indicates an expected call of SetExptTerminating. -func (mr *MockIExptManagerMockRecorder) SetExptTerminating(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) SetExptTerminating(ctx, exptID, exptRunID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetExptTerminating", reflect.TypeOf((*MockIExptManager)(nil).SetExptTerminating), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetExptTerminating", reflect.TypeOf((*MockIExptManager)(nil).SetExptTerminating), ctx, exptID, exptRunID, spaceID, session) } // TerminateItems mocks base method. -func (m *MockIExptManager) TerminateItems(arg0 context.Context, arg1, arg2, arg3 int64, arg4 []int64, arg5 *entity.Session) error { +func (m *MockIExptManager) TerminateItems(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TerminateItems", arg0, arg1, arg2, arg3, arg4, arg5) + ret := m.ctrl.Call(m, "TerminateItems", ctx, exptID, exptRunID, spaceID, itemIDs, session) ret0, _ := ret[0].(error) return ret0 } // TerminateItems indicates an expected call of TerminateItems. -func (mr *MockIExptManagerMockRecorder) TerminateItems(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) TerminateItems(ctx, exptID, exptRunID, spaceID, itemIDs, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateItems", reflect.TypeOf((*MockIExptManager)(nil).TerminateItems), arg0, arg1, arg2, arg3, arg4, arg5) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateItems", reflect.TypeOf((*MockIExptManager)(nil).TerminateItems), ctx, exptID, exptRunID, spaceID, itemIDs, session) } // UnlockCompletingRun mocks base method. -func (m *MockIExptManager) UnlockCompletingRun(arg0 context.Context, arg1, arg2, arg3 int64, arg4 *entity.Session) error { +func (m *MockIExptManager) UnlockCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnlockCompletingRun", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "UnlockCompletingRun", ctx, exptID, exptRunID, spaceID, session) ret0, _ := ret[0].(error) return ret0 } // UnlockCompletingRun indicates an expected call of UnlockCompletingRun. -func (mr *MockIExptManagerMockRecorder) UnlockCompletingRun(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) UnlockCompletingRun(ctx, exptID, exptRunID, spaceID, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockCompletingRun", reflect.TypeOf((*MockIExptManager)(nil).UnlockCompletingRun), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockCompletingRun", reflect.TypeOf((*MockIExptManager)(nil).UnlockCompletingRun), ctx, exptID, exptRunID, spaceID, session) } // Update mocks base method. -func (m *MockIExptManager) Update(arg0 context.Context, arg1 *entity.Experiment, arg2 *entity.Session) error { +func (m *MockIExptManager) Update(ctx context.Context, expt *entity.Experiment, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Update", ctx, expt, session) ret0, _ := ret[0].(error) return ret0 } // Update indicates an expected call of Update. -func (mr *MockIExptManagerMockRecorder) Update(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Update(ctx, expt, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptManager)(nil).Update), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptManager)(nil).Update), ctx, expt, session) } // UpdateRunConf mocks base method. -func (m *MockIExptManager) UpdateRunConf(arg0 context.Context, arg1 *entity.UpdateRunConfParam) error { +func (m *MockIExptManager) UpdateRunConf(ctx context.Context, param *entity.UpdateRunConfParam) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateRunConf", arg0, arg1) + ret := m.ctrl.Call(m, "UpdateRunConf", ctx, param) ret0, _ := ret[0].(error) return ret0 } // UpdateRunConf indicates an expected call of UpdateRunConf. -func (mr *MockIExptManagerMockRecorder) UpdateRunConf(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) UpdateRunConf(ctx, param any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRunConf", reflect.TypeOf((*MockIExptManager)(nil).UpdateRunConf), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRunConf", reflect.TypeOf((*MockIExptManager)(nil).UpdateRunConf), ctx, param) } diff --git a/backend/modules/evaluation/domain/service/mocks/expt_new_run_reset.go b/backend/modules/evaluation/domain/service/mocks/expt_new_run_reset.go new file mode 100644 index 0000000000..678283a844 --- /dev/null +++ b/backend/modules/evaluation/domain/service/mocks/expt_new_run_reset.go @@ -0,0 +1,55 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: IExptNewRunItemResetter) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_new_run_reset.go -package=mocks . IExptNewRunItemResetter +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockIExptNewRunItemResetter is a mock of IExptNewRunItemResetter interface. +type MockIExptNewRunItemResetter struct { + ctrl *gomock.Controller + recorder *MockIExptNewRunItemResetterMockRecorder + isgomock struct{} +} + +// MockIExptNewRunItemResetterMockRecorder is the mock recorder for MockIExptNewRunItemResetter. +type MockIExptNewRunItemResetterMockRecorder struct { + mock *MockIExptNewRunItemResetter +} + +// NewMockIExptNewRunItemResetter creates a new mock instance. +func NewMockIExptNewRunItemResetter(ctrl *gomock.Controller) *MockIExptNewRunItemResetter { + mock := &MockIExptNewRunItemResetter{ctrl: ctrl} + mock.recorder = &MockIExptNewRunItemResetterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptNewRunItemResetter) EXPECT() *MockIExptNewRunItemResetterMockRecorder { + return m.recorder +} + +// ResetItemsForNewRun mocks base method. +func (m *MockIExptNewRunItemResetter) ResetItemsForNewRun(arg0 context.Context, arg1, arg2, arg3 int64, arg4 []int64, arg5 map[string]any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResetItemsForNewRun", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(error) + return ret0 +} + +// ResetItemsForNewRun indicates an expected call of ResetItemsForNewRun. +func (mr *MockIExptNewRunItemResetterMockRecorder) ResetItemsForNewRun(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetItemsForNewRun", reflect.TypeOf((*MockIExptNewRunItemResetter)(nil).ResetItemsForNewRun), arg0, arg1, arg2, arg3, arg4, arg5) +} diff --git a/backend/modules/evaluation/domain/service/mocks/expt_result_filter_sync.go b/backend/modules/evaluation/domain/service/mocks/expt_result_filter_sync.go new file mode 100644 index 0000000000..572f497c4e --- /dev/null +++ b/backend/modules/evaluation/domain/service/mocks/expt_result_filter_sync.go @@ -0,0 +1,56 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: ExptManualRetryFilterService) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_result_filter_sync.go -package=mocks . ExptManualRetryFilterService +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockExptManualRetryFilterService is a mock of ExptManualRetryFilterService interface. +type MockExptManualRetryFilterService struct { + ctrl *gomock.Controller + recorder *MockExptManualRetryFilterServiceMockRecorder + isgomock struct{} +} + +// MockExptManualRetryFilterServiceMockRecorder is the mock recorder for MockExptManualRetryFilterService. +type MockExptManualRetryFilterServiceMockRecorder struct { + mock *MockExptManualRetryFilterService +} + +// NewMockExptManualRetryFilterService creates a new mock instance. +func NewMockExptManualRetryFilterService(ctrl *gomock.Controller) *MockExptManualRetryFilterService { + mock := &MockExptManualRetryFilterService{ctrl: ctrl} + mock.recorder = &MockExptManualRetryFilterServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockExptManualRetryFilterService) EXPECT() *MockExptManualRetryFilterServiceMockRecorder { + return m.recorder +} + +// ReconcileManualRetryFilterBeforeFinish mocks base method. +func (m *MockExptManualRetryFilterService) ReconcileManualRetryFilterBeforeFinish(arg0 context.Context, arg1, arg2, arg3 int64) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconcileManualRetryFilterBeforeFinish", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReconcileManualRetryFilterBeforeFinish indicates an expected call of ReconcileManualRetryFilterBeforeFinish. +func (mr *MockExptManualRetryFilterServiceMockRecorder) ReconcileManualRetryFilterBeforeFinish(arg0, arg1, arg2, arg3 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileManualRetryFilterBeforeFinish", reflect.TypeOf((*MockExptManualRetryFilterService)(nil).ReconcileManualRetryFilterBeforeFinish), arg0, arg1, arg2, arg3) +} diff --git a/backend/modules/evaluation/domain/service/mocks/expt_result_publish.go b/backend/modules/evaluation/domain/service/mocks/expt_result_publish.go new file mode 100644 index 0000000000..9c3a952143 --- /dev/null +++ b/backend/modules/evaluation/domain/service/mocks/expt_result_publish.go @@ -0,0 +1,87 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: ExptResultPublicationService) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_result_publish.go -package=mocks . ExptResultPublicationService +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockExptResultPublicationService is a mock of ExptResultPublicationService interface. +type MockExptResultPublicationService struct { + ctrl *gomock.Controller + recorder *MockExptResultPublicationServiceMockRecorder + isgomock struct{} +} + +// MockExptResultPublicationServiceMockRecorder is the mock recorder for MockExptResultPublicationService. +type MockExptResultPublicationServiceMockRecorder struct { + mock *MockExptResultPublicationService +} + +// NewMockExptResultPublicationService creates a new mock instance. +func NewMockExptResultPublicationService(ctrl *gomock.Controller) *MockExptResultPublicationService { + mock := &MockExptResultPublicationService{ctrl: ctrl} + mock.recorder = &MockExptResultPublicationServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockExptResultPublicationService) EXPECT() *MockExptResultPublicationServiceMockRecorder { + return m.recorder +} + +// GetItemResultPublishSource mocks base method. +func (m *MockExptResultPublicationService) GetItemResultPublishSource(arg0 context.Context, arg1, arg2, arg3, arg4 int64) (*entity.ExptItemResultPublishSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetItemResultPublishSource", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(*entity.ExptItemResultPublishSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetItemResultPublishSource indicates an expected call of GetItemResultPublishSource. +func (mr *MockExptResultPublicationServiceMockRecorder) GetItemResultPublishSource(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetItemResultPublishSource", reflect.TypeOf((*MockExptResultPublicationService)(nil).GetItemResultPublishSource), arg0, arg1, arg2, arg3, arg4) +} + +// PrepareItemResultPublication mocks base method. +func (m *MockExptResultPublicationService) PrepareItemResultPublication(arg0 context.Context, arg1 *entity.ExptItemResultPublishSource, arg2 *entity.Experiment) (*entity.ExptItemResultPublication, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PrepareItemResultPublication", arg0, arg1, arg2) + ret0, _ := ret[0].(*entity.ExptItemResultPublication) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PrepareItemResultPublication indicates an expected call of PrepareItemResultPublication. +func (mr *MockExptResultPublicationServiceMockRecorder) PrepareItemResultPublication(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareItemResultPublication", reflect.TypeOf((*MockExptResultPublicationService)(nil).PrepareItemResultPublication), arg0, arg1, arg2) +} + +// PublishPreparedItemResult mocks base method. +func (m *MockExptResultPublicationService) PublishPreparedItemResult(arg0 context.Context, arg1 *entity.ExptItemResultPublication, arg2 func(context.Context, *entity.ExptItemResultPublishSource) error) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PublishPreparedItemResult", arg0, arg1, arg2) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PublishPreparedItemResult indicates an expected call of PublishPreparedItemResult. +func (mr *MockExptResultPublicationServiceMockRecorder) PublishPreparedItemResult(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublishPreparedItemResult", reflect.TypeOf((*MockExptResultPublicationService)(nil).PublishPreparedItemResult), arg0, arg1, arg2) +} diff --git a/backend/modules/evaluation/domain/service/mocks/expt_run_finish.go b/backend/modules/evaluation/domain/service/mocks/expt_run_finish.go new file mode 100644 index 0000000000..730e5e6ede --- /dev/null +++ b/backend/modules/evaluation/domain/service/mocks/expt_run_finish.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: IExptNaturalRunFinisher) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_run_finish.go -package=mocks . IExptNaturalRunFinisher +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptNaturalRunFinisher is a mock of IExptNaturalRunFinisher interface. +type MockIExptNaturalRunFinisher struct { + ctrl *gomock.Controller + recorder *MockIExptNaturalRunFinisherMockRecorder + isgomock struct{} +} + +// MockIExptNaturalRunFinisherMockRecorder is the mock recorder for MockIExptNaturalRunFinisher. +type MockIExptNaturalRunFinisherMockRecorder struct { + mock *MockIExptNaturalRunFinisher +} + +// NewMockIExptNaturalRunFinisher creates a new mock instance. +func NewMockIExptNaturalRunFinisher(ctrl *gomock.Controller) *MockIExptNaturalRunFinisher { + mock := &MockIExptNaturalRunFinisher{ctrl: ctrl} + mock.recorder = &MockIExptNaturalRunFinisherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptNaturalRunFinisher) EXPECT() *MockIExptNaturalRunFinisherMockRecorder { + return m.recorder +} + +// FinishCurrentRun mocks base method. +func (m *MockIExptNaturalRunFinisher) FinishCurrentRun(arg0 context.Context, arg1 *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinishCurrentRun", arg0, arg1) + ret0, _ := ret[0].(*entity.ExptRunFinishResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FinishCurrentRun indicates an expected call of FinishCurrentRun. +func (mr *MockIExptNaturalRunFinisherMockRecorder) FinishCurrentRun(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishCurrentRun", reflect.TypeOf((*MockIExptNaturalRunFinisher)(nil).FinishCurrentRun), arg0, arg1) +} + +// FinishCurrentRunWithHeldTickLock mocks base method. +func (m *MockIExptNaturalRunFinisher) FinishCurrentRunWithHeldTickLock(arg0 context.Context, arg1 *entity.ExptScheduleEvent) (*entity.ExptRunFinishResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinishCurrentRunWithHeldTickLock", arg0, arg1) + ret0, _ := ret[0].(*entity.ExptRunFinishResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FinishCurrentRunWithHeldTickLock indicates an expected call of FinishCurrentRunWithHeldTickLock. +func (mr *MockIExptNaturalRunFinisherMockRecorder) FinishCurrentRunWithHeldTickLock(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishCurrentRunWithHeldTickLock", reflect.TypeOf((*MockIExptNaturalRunFinisher)(nil).FinishCurrentRunWithHeldTickLock), arg0, arg1) +} diff --git a/backend/modules/evaluation/domain/service/mocks/expt_run_termination.go b/backend/modules/evaluation/domain/service/mocks/expt_run_termination.go new file mode 100644 index 0000000000..c5b2f97f22 --- /dev/null +++ b/backend/modules/evaluation/domain/service/mocks/expt_run_termination.go @@ -0,0 +1,62 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: IExptCurrentRunTerminator) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_run_termination.go -package=mocks . IExptCurrentRunTerminator +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptCurrentRunTerminator is a mock of IExptCurrentRunTerminator interface. +type MockIExptCurrentRunTerminator struct { + ctrl *gomock.Controller + recorder *MockIExptCurrentRunTerminatorMockRecorder + isgomock struct{} +} + +// MockIExptCurrentRunTerminatorMockRecorder is the mock recorder for MockIExptCurrentRunTerminator. +type MockIExptCurrentRunTerminatorMockRecorder struct { + mock *MockIExptCurrentRunTerminator +} + +// NewMockIExptCurrentRunTerminator creates a new mock instance. +func NewMockIExptCurrentRunTerminator(ctrl *gomock.Controller) *MockIExptCurrentRunTerminator { + mock := &MockIExptCurrentRunTerminator{ctrl: ctrl} + mock.recorder = &MockIExptCurrentRunTerminatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptCurrentRunTerminator) EXPECT() *MockIExptCurrentRunTerminatorMockRecorder { + return m.recorder +} + +// TerminateCurrentRun mocks base method. +func (m *MockIExptCurrentRunTerminator) TerminateCurrentRun(arg0 context.Context, arg1, arg2, arg3 int64, arg4 *entity.Session, arg5 *entity.ExptCurrentItemRunSource, arg6 ...entity.CompleteExptOptionFn) (bool, error) { + m.ctrl.T.Helper() + varargs := []any{arg0, arg1, arg2, arg3, arg4, arg5} + for _, a := range arg6 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "TerminateCurrentRun", varargs...) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TerminateCurrentRun indicates an expected call of TerminateCurrentRun. +func (mr *MockIExptCurrentRunTerminatorMockRecorder) TerminateCurrentRun(arg0, arg1, arg2, arg3, arg4, arg5 any, arg6 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0, arg1, arg2, arg3, arg4, arg5}, arg6...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateCurrentRun", reflect.TypeOf((*MockIExptCurrentRunTerminator)(nil).TerminateCurrentRun), varargs...) +} diff --git a/backend/modules/evaluation/domain/service/target_impl_test.go b/backend/modules/evaluation/domain/service/target_impl_test.go index 5bc2817b16..466e4065f2 100755 --- a/backend/modules/evaluation/domain/service/target_impl_test.go +++ b/backend/modules/evaluation/domain/service/target_impl_test.go @@ -1196,7 +1196,7 @@ func TestEvalTargetServiceImpl_ReportInvokeRecords(t *testing.T) { } func TestEvalTargetServiceImpl_ReportInvokeRecords_Trajectory(t *testing.T) { - // do not run in parallel, this test involves time.Sleep + // Keep each asynchronous extraction within its own mock lifetime. ctx := context.Background() spaceID := int64(1) @@ -1241,6 +1241,16 @@ func TestEvalTargetServiceImpl_ReportInvokeRecords_Trajectory(t *testing.T) { deps.configer.EXPECT().BuildEvalExt(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) trajectoryAdapter := trajectorymocks.NewMockITrajectoryAdapter(ctrl) + extractionDone := make(chan struct{}) + waitCtx, cancelWait := context.WithTimeout(ctx, 5*time.Second) + defer cancelWait() + defer func() { + select { + case <-extractionDone: + case <-waitCtx.Done(): + t.Error("trajectory extraction did not finish before mock teardown") + } + }() record := &entity.EvalTargetRecord{ ID: 10, @@ -1291,12 +1301,25 @@ func TestEvalTargetServiceImpl_ReportInvokeRecords_Trajectory(t *testing.T) { }) trajectoryAdapter.EXPECT(). ListTrajectory(gomock.Any(), spaceID, gomock.Any(), gomock.AssignableToTypeOf((*int64)(nil))). - Return(tt.trajectories, tt.err) + DoAndReturn(func(extractCtx context.Context, _ int64, _ []string, _ *int64) ([]*entity.Trajectory, error) { + // Production cancels this context after extraction and persistence. + context.AfterFunc(extractCtx, func() { close(extractionDone) }) + if tt.expectHasField { + // A valid response may arrive after the old 1100ms observation. + timer := time.NewTimer(300 * time.Millisecond) + defer timer.Stop() + select { + case <-timer.C: + case <-extractCtx.Done(): + return nil, extractCtx.Err() + } + } + return tt.trajectories, tt.err + }) // use channel to safely observe async UpdateEvalTargetRecord calls from goroutine updatedCh := make(chan *entity.EvalTargetRecord, 1) - deps.repo.EXPECT().UpdateEvalTargetRecord(gomock.Any(), gomock.Any(), gomock.Any()). - AnyTimes(). + updateCall := deps.repo.EXPECT().UpdateEvalTargetRecord(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, rec *entity.EvalTargetRecord, _ *bool) error { // non-blocking send in case of multiple updates select { @@ -1305,6 +1328,9 @@ func TestEvalTargetServiceImpl_ReportInvokeRecords_Trajectory(t *testing.T) { } return nil }) + if !tt.expectHasField { + updateCall.Times(0) + } svc := &EvalTargetServiceImpl{ evalTargetRepo: deps.repo, @@ -1321,19 +1347,28 @@ func TestEvalTargetServiceImpl_ReportInvokeRecords_Trajectory(t *testing.T) { assert.Equal(t, param.OutputData, saved.EvalTargetOutputData) assert.Equal(t, param.Status, gptr.Indirect(saved.Status)) - // wait for trajectory goroutine to complete - time.Sleep(1100 * time.Millisecond) var updated *entity.EvalTargetRecord - select { - case updated = <-updatedCh: - default: - } - if !tt.expectHasField { + select { + case <-extractionDone: + case <-waitCtx.Done(): + t.Fatal("trajectory extraction did not finish") + } + select { + case updated = <-updatedCh: + default: + } assert.Nil(t, updated) return } + // Observe persistence, rather than guessing when the worker is scheduled. + select { + case updated = <-updatedCh: + case <-waitCtx.Done(): + t.Fatal("trajectory update did not arrive") + } + require.NotNil(t, updated) content, ok := updated.EvalTargetOutputData.OutputFields[consts.EvalTargetOutputFieldKeyTrajectory] assert.True(t, ok) diff --git a/backend/modules/evaluation/domain/service/wire.go b/backend/modules/evaluation/domain/service/wire.go index a129c4ad0e..d144b21ea8 100644 --- a/backend/modules/evaluation/domain/service/wire.go +++ b/backend/modules/evaluation/domain/service/wire.go @@ -29,8 +29,9 @@ import ( // ExperimentDomainServiceSet 提供所有 Experiment 相关的 Domain Service var ExperimentDomainServiceSet = wire.NewSet( - NewExptManager, - NewExptResultService, + NewExptManagerWithRunFinish, + experimentrepo.NewExptRunFinishRepo, + NewExptResultServiceWithPublication, NewExptAggrResultService, NewExptSchedulerSvc, NewExptRecordEvalService, @@ -50,6 +51,7 @@ var ExperimentDomainServiceSet = wire.NewSet( // 中心化调度额度闸：开源侧提供 noop（enforce 消息 fail-closed，legacy 不受影响）。 // 商业版在自己的 wire set 里注入真实账本适配器覆盖它。 component.NewNoopCentralReservationGuard, + component.NewNoopCurrentRunSchedulerWaker, component.NewNoopCentralSchedulerScopeOwner, component.NewNoopCentralSchedulerScopeProvider, component.NewNoopCentralAdmissionPolicy, diff --git a/backend/modules/evaluation/infra/mq/rocket/consumer/expt_record_eval_current_request_test.go b/backend/modules/evaluation/infra/mq/rocket/consumer/expt_record_eval_current_request_test.go new file mode 100644 index 0000000000..87d1eb741e --- /dev/null +++ b/backend/modules/evaluation/infra/mq/rocket/consumer/expt_record_eval_current_request_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package consumer + +import ( + "context" + "encoding/json" + "testing" + + "github.com/coze-dev/coze-loop/backend/infra/mq" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + mocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestCurrentRequestBrokerSeesRetry(t *testing.T) { + for _, request := range []string{"", "rq1"} { + t.Run(request, func(t *testing.T) { + ctrl := gomock.NewController(t) + svc := mocks.NewMockExptItemEvalEvent(ctrl) + want := errno.NewMQRetryErr("item lock busy") + event := &entity.ExptItemEvalEvent{RequestID: request, SpaceID: 3, ExptID: 1, ExptRunID: 2, EvalSetItemID: 11, ExptRunMode: entity.EvaluationModeSubmit, Session: &entity.Session{UserID: "original"}} + body, err := json.Marshal(event) + require.NoError(t, err) + svc.EXPECT().Eval(gomock.Any(), gomock.Eq(event)).Return(want) + handler := NewExptRecordEvalConsumer(svc) + err = handler.HandleMessage(context.Background(), &mq.MessageExt{Message: mq.Message{Body: body}}) + require.Same(t, want, err) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter.go b/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter.go index 84a34399ca..8c6a26637f 100644 --- a/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter.go +++ b/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter.go @@ -112,6 +112,14 @@ type IExptTurnResultFilterDAO interface { GetByExptIDItemIDs(ctx context.Context, spaceID, exptID, createdDate string, itemIDs []string) ([]*compare_model.ExptTurnResultFilter, error) } +// IExptMaterializedFilterDAO is the complete OSS projection read used only by +// final calibration. It leaves the existing asynchronous client contract intact. +// +//go:generate mockgen -destination=mocks/expt_materialized_filter.go -package=mocks . IExptMaterializedFilterDAO +type IExptMaterializedFilterDAO interface { + GetMaterializedByExptIDItemIDs(context.Context, string, string, string, []string) ([]*model.ExptTurnResultFilter, error) +} + type exptTurnResultFilterDAOImpl struct { db ck.Provider // 新增字段，用于存储数据库名 @@ -878,3 +886,14 @@ func (d *exptTurnResultFilterDAOImpl) buildGetByExptIDItemIDsSQL(ctx context.Con logs.CtxInfo(ctx, "GetByExptID sql: %v, args: %v", sql, args) return sql, args } + +// GetMaterializedByExptIDItemIDs reads every stored field and resolves older +// ReplacingMergeTree rows before final calibration. Map key absence is retained. +func (d *exptTurnResultFilterDAOImpl) GetMaterializedByExptIDItemIDs(ctx context.Context, space, expt, date string, ids []string) ([]*model.ExptTurnResultFilter, error) { + if len(ids) == 0 { + return nil, nil + } + var rows []*model.ExptTurnResultFilter + err := d.db.NewSession(ctx).Raw("SELECT etrf.* FROM "+getClickHouseDatabaseName()+".expt_turn_result_filter AS etrf FINAL WHERE etrf.space_id=? AND etrf.expt_id=? AND etrf.created_date=? AND etrf.item_id IN ?", space, expt, date, ids).Scan(&rows).Error + return rows, err +} diff --git a/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter_test.go b/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter_test.go index 070d33f653..29322e0e32 100644 --- a/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter_test.go @@ -9,6 +9,12 @@ import ( "testing" "time" + "github.com/DATA-DOG/go-sqlmock" + ckprovidermocks "github.com/coze-dev/coze-loop/backend/infra/ck/mocks" + "github.com/stretchr/testify/require" + "gorm.io/driver/clickhouse" + "gorm.io/gorm" + "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" @@ -1249,3 +1255,23 @@ func TestExptTurnResultFilterDAOImpl_buildMainTableConditions_EvalSetIDs(t *test assert.Contains(t, whereSQL, " AND etrf.eval_set_version_id = ?") }) } + +// Run the complete materialized SQL through the actual CK DAO; FINAL and map +// preservation are boundary behavior, not helper-source string assertions. +func TestMaterializedFilterDAO_ReadFullProjection(t *testing.T) { + ctrl := gomock.NewController(t) + sqlDB, mock, err := sqlmock.New() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + conn, err := gorm.Open(clickhouse.New(clickhouse.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), &gorm.Config{DisableAutomaticPing: true}) + require.NoError(t, err) + provider := ckprovidermocks.NewMockProvider(ctrl) + provider.EXPECT().NewSession(gomock.Any()).DoAndReturn(func(ctx context.Context) *gorm.DB { return conn.WithContext(ctx) }) + mock.ExpectQuery("SELECT etrf\\.\\* FROM .*expt_turn_result_filter AS etrf FINAL WHERE etrf.space_id=\\? AND etrf.expt_id=\\? AND etrf.created_date=\\? AND etrf.item_id IN \\(\\?\\)").WithArgs("3", "1", "2026-10-20", "7384603622000000011").WillReturnRows(sqlmock.NewRows([]string{"space_id", "expt_id", "item_id", "item_version_id", "turn_id", "status", "eval_set_id", "eval_set_version_id"}).AddRow("3", "1", "7384603622000000011", "7384603622000000088", "0", 2, "55", "66")) + dao := NewExptTurnResultFilterDAO(provider, nil).(*exptTurnResultFilterDAOImpl) + rows, err := dao.GetMaterializedByExptIDItemIDs(context.Background(), "3", "1", "2026-10-20", []string{"7384603622000000011"}) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, "7384603622000000088", rows[0].ItemVersionID) + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/ck/mocks/expt_materialized_filter.go b/backend/modules/evaluation/infra/repo/experiment/ck/mocks/expt_materialized_filter.go new file mode 100644 index 0000000000..88270625da --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/ck/mocks/expt_materialized_filter.go @@ -0,0 +1,57 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/ck (interfaces: IExptMaterializedFilterDAO) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_materialized_filter.go -package=mocks . IExptMaterializedFilterDAO +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + model "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/ck/gorm_gen/model" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptMaterializedFilterDAO is a mock of IExptMaterializedFilterDAO interface. +type MockIExptMaterializedFilterDAO struct { + ctrl *gomock.Controller + recorder *MockIExptMaterializedFilterDAOMockRecorder + isgomock struct{} +} + +// MockIExptMaterializedFilterDAOMockRecorder is the mock recorder for MockIExptMaterializedFilterDAO. +type MockIExptMaterializedFilterDAOMockRecorder struct { + mock *MockIExptMaterializedFilterDAO +} + +// NewMockIExptMaterializedFilterDAO creates a new mock instance. +func NewMockIExptMaterializedFilterDAO(ctrl *gomock.Controller) *MockIExptMaterializedFilterDAO { + mock := &MockIExptMaterializedFilterDAO{ctrl: ctrl} + mock.recorder = &MockIExptMaterializedFilterDAOMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptMaterializedFilterDAO) EXPECT() *MockIExptMaterializedFilterDAOMockRecorder { + return m.recorder +} + +// GetMaterializedByExptIDItemIDs mocks base method. +func (m *MockIExptMaterializedFilterDAO) GetMaterializedByExptIDItemIDs(arg0 context.Context, arg1, arg2, arg3 string, arg4 []string) ([]*model.ExptTurnResultFilter, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaterializedByExptIDItemIDs", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].([]*model.ExptTurnResultFilter) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMaterializedByExptIDItemIDs indicates an expected call of GetMaterializedByExptIDItemIDs. +func (mr *MockIExptMaterializedFilterDAOMockRecorder) GetMaterializedByExptIDItemIDs(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaterializedByExptIDItemIDs", reflect.TypeOf((*MockIExptMaterializedFilterDAO)(nil).GetMaterializedByExptIDItemIDs), arg0, arg1, arg2, arg3, arg4) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl.go index e074b56328..1d6a36ca51 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl.go @@ -20,24 +20,24 @@ func NewExptItemDispatchRepo(dispatchDAO mysql.IExptItemDispatchDAO) repo.IExptI return &ExptItemDispatchRepoImpl{dispatchDAO: dispatchDAO} } -func (e *ExptItemDispatchRepoImpl) ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) { - return e.dispatchDAO.ClaimQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs) +func (e *ExptItemDispatchRepoImpl) ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]repo.ClaimedDispatchItem, error) { + return e.dispatchDAO.ClaimQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) } -func (e *ExptItemDispatchRepoImpl) ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) { - return e.dispatchDAO.ResetQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs) +func (e *ExptItemDispatchRepoImpl) ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]int64, error) { + return e.dispatchDAO.ResetQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) } func (e *ExptItemDispatchRepoImpl) LoadDispatchRuntime(ctx context.Context, spaceID, exptID, exptRunID int64, candidateLimit int) (*repo.ExptDispatchRuntime, error) { return e.dispatchDAO.LoadDispatchRuntime(ctx, spaceID, exptID, exptRunID, candidateLimit) } -func (e *ExptItemDispatchRepoImpl) StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) { - return e.dispatchDAO.StartReservedItem(ctx, spaceID, exptID, exptRunID, itemID) +func (e *ExptItemDispatchRepoImpl) StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (repo.StartReservedItemResult, error) { + return e.dispatchDAO.StartReservedItem(ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) } -func (e *ExptItemDispatchRepoImpl) RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) { - return e.dispatchDAO.RequeueProcessingItem(ctx, spaceID, exptID, exptRunID, itemID) +func (e *ExptItemDispatchRepoImpl) RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (bool, error) { + return e.dispatchDAO.RequeueProcessingItem(ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) } func (e *ExptItemDispatchRepoImpl) MGetDispatchObservations(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]*repo.ExptDispatchObservation, error) { diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl_test.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl_test.go index 7611d4b7c9..9863fd27a3 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_dispatch_repo_impl_test.go @@ -53,25 +53,27 @@ func TestNewExptItemDispatchRepo(t *testing.T) { func TestExptItemDispatchRepo_ClaimQuotaReserved(t *testing.T) { itemIDs := []int64{1, 2, 3} + expected := map[int64]string{1: "", 2: "rq1", 3: "rq2"} + payload := []repo.ClaimedDispatchItem{{ItemID: 1, Mode: entity.EvaluationModeSubmit}, {ItemID: 3, RequestID: "rq2", Mode: entity.EvaluationModeRetryItems, RetryTimes: 3}} t.Run("成功：实参按位置转发，返回值原样上抛", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) dao.EXPECT(). - ClaimQuotaReserved(gomock.Any(), testSpaceID, testExptID, testExptRunID, itemIDs). - Return([]int64{1, 3}, nil) + ClaimQuotaReserved(gomock.Any(), testSpaceID, testExptID, testExptRunID, itemIDs, expected). + Return(payload, nil) - got, err := r.ClaimQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs) + got, err := r.ClaimQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs, expected) require.NoError(t, err) - assert.Equal(t, []int64{1, 3}, got) + assert.Equal(t, payload, got) }) t.Run("失败：错误不被吞", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) wantErr := errors.New("dao down") - dao.EXPECT().ClaimQuotaReserved(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + dao.EXPECT().ClaimQuotaReserved(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, wantErr) - got, err := r.ClaimQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs) + got, err := r.ClaimQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs, expected) require.ErrorIs(t, err, wantErr) assert.Nil(t, got) }) @@ -79,14 +81,15 @@ func TestExptItemDispatchRepo_ClaimQuotaReserved(t *testing.T) { func TestExptItemDispatchRepo_ResetQuotaReserved(t *testing.T) { itemIDs := []int64{7, 8} + expected := map[int64]string{7: "", 8: "rq1"} t.Run("成功", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) dao.EXPECT(). - ResetQuotaReserved(gomock.Any(), testSpaceID, testExptID, testExptRunID, itemIDs). + ResetQuotaReserved(gomock.Any(), testSpaceID, testExptID, testExptRunID, itemIDs, expected). Return(itemIDs, nil) - got, err := r.ResetQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs) + got, err := r.ResetQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs, expected) require.NoError(t, err) assert.Equal(t, itemIDs, got) }) @@ -94,10 +97,10 @@ func TestExptItemDispatchRepo_ResetQuotaReserved(t *testing.T) { t.Run("失败", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) wantErr := errors.New("reset fail") - dao.EXPECT().ResetQuotaReserved(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + dao.EXPECT().ResetQuotaReserved(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, wantErr) - _, err := r.ResetQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs) + _, err := r.ResetQuotaReserved(context.Background(), testSpaceID, testExptID, testExptRunID, itemIDs, expected) require.ErrorIs(t, err, wantErr) }) } @@ -138,31 +141,31 @@ func TestExptItemDispatchRepo_StartReservedItem(t *testing.T) { t.Run("CAS 命中返回 true", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) dao.EXPECT(). - StartReservedItem(gomock.Any(), testSpaceID, testExptID, testExptRunID, testItemID). - Return(true, nil) + StartReservedItem(gomock.Any(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1"). + Return(repo.StartReservedItemResult{CurrentRequest: true, Started: true}, nil) - started, err := r.StartReservedItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID) + started, err := r.StartReservedItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1") require.NoError(t, err) - assert.True(t, started) + assert.True(t, started.Started) }) t.Run("CAS 未命中返回 false 且不报错", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) - dao.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(false, nil) + dao.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(repo.StartReservedItemResult{}, nil) - started, err := r.StartReservedItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID) + started, err := r.StartReservedItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1") require.NoError(t, err) - assert.False(t, started, "CAS 未命中必须回 false —— 改写成 true 会让重复投递的 item 执行两次") + assert.False(t, started.CurrentRequest, "CAS 未命中必须回 false —— 改写成 true 会让重复投递的 item 执行两次") }) t.Run("失败", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) wantErr := errors.New("start fail") - dao.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(false, wantErr) + dao.EXPECT().StartReservedItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(repo.StartReservedItemResult{}, wantErr) - _, err := r.StartReservedItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID) + _, err := r.StartReservedItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1") require.ErrorIs(t, err, wantErr) }) } @@ -171,20 +174,20 @@ func TestExptItemDispatchRepo_RequeueProcessingItem(t *testing.T) { t.Run("退回成功", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) dao.EXPECT(). - RequeueProcessingItem(gomock.Any(), testSpaceID, testExptID, testExptRunID, testItemID). + RequeueProcessingItem(gomock.Any(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1"). Return(true, nil) - requeued, err := r.RequeueProcessingItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID) + requeued, err := r.RequeueProcessingItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1") require.NoError(t, err) assert.True(t, requeued) }) t.Run("CAS 未命中返回 false 且不报错", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) - dao.EXPECT().RequeueProcessingItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + dao.EXPECT().RequeueProcessingItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(false, nil) - requeued, err := r.RequeueProcessingItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID) + requeued, err := r.RequeueProcessingItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1") require.NoError(t, err) assert.False(t, requeued) }) @@ -192,10 +195,10 @@ func TestExptItemDispatchRepo_RequeueProcessingItem(t *testing.T) { t.Run("失败", func(t *testing.T) { r, dao := newDispatchRepoFixture(t) wantErr := errors.New("requeue fail") - dao.EXPECT().RequeueProcessingItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + dao.EXPECT().RequeueProcessingItem(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(false, wantErr) - _, err := r.RequeueProcessingItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID) + _, err := r.RequeueProcessingItem(context.Background(), testSpaceID, testExptID, testExptRunID, testItemID, "rq1") require.ErrorIs(t, err, wantErr) }) } diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_ref.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_ref.go index 773489371d..6b897f7cd7 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_ref.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_ref.go @@ -5,6 +5,8 @@ package experiment import ( "context" + "encoding/json" + "fmt" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" @@ -51,6 +53,12 @@ func (r *ExptItemRefRepoImpl) GetByExptIDAndItemID(ctx context.Context, spaceID, if err != nil { return nil, err } + if po != nil && po.ItemConfig != nil && len(*po.ItemConfig) > 0 { + var cfg *entity.ExptItemConfig + if err := json.Unmarshal(*po.ItemConfig, &cfg); err != nil { + return nil, fmt.Errorf("frozen item config: %w", err) + } + } return convert.NewExptItemRefConvertor().PO2DO(po), nil } diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go index 46b73d622b..427c401737 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go @@ -5,6 +5,9 @@ package experiment import ( "context" + "fmt" + + "github.com/bytedance/gg/gptr" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" @@ -34,7 +37,13 @@ func (e ExptItemResultRepoImpl) BatchGet(ctx context.Context, spaceID, exptID in results := make([]*entity.ExptItemResult, 0) for _, po := range pos { - results = append(results, convert.NewExptItemResultConvertor().PO2DO(po)) + result := convert.NewExptItemResultConvertor().PO2DO(po) + ext, parseErr := entity.ParseExptItemResultExt(gptr.Indirect(po.Ext)) + if parseErr != nil { + return nil, parseErr + } + result.Ext = ext + results = append(results, result) } return results, nil } @@ -140,7 +149,13 @@ func (e ExptItemResultRepoImpl) ScanItemResults(ctx context.Context, exptID, cur return nil, 0, errorx.Wrapf(err, "ScanItemResults fail, exptID=%d, cursor=%d", exptID, cursor) } for _, po := range pos { - results = append(results, convert.NewExptItemResultConvertor().PO2DO(po)) + result := convert.NewExptItemResultConvertor().PO2DO(po) + ext, parseErr := entity.ParseExptItemResultExt(gptr.Indirect(po.Ext)) + if parseErr != nil { + return nil, 0, parseErr + } + result.Ext = ext + results = append(results, result) } return results, ncursor, nil } @@ -152,7 +167,13 @@ func (e ExptItemResultRepoImpl) MGetItemResults(ctx context.Context, exptID int6 } results = make([]*entity.ExptItemResult, 0, len(pos)) for _, po := range pos { - results = append(results, convert.NewExptItemResultConvertor().PO2DO(po)) + result := convert.NewExptItemResultConvertor().PO2DO(po) + ext, parseErr := entity.ParseExptItemResultExt(gptr.Indirect(po.Ext)) + if parseErr != nil { + return nil, parseErr + } + result.Ext = ext + results = append(results, result) } return results, nil } @@ -168,7 +189,13 @@ func (e ExptItemResultRepoImpl) ListItemResultsByExptID(ctx context.Context, exp } results := make([]*entity.ExptItemResult, 0) for _, po := range pos { - results = append(results, convert.NewExptItemResultConvertor().PO2DO(po)) + result := convert.NewExptItemResultConvertor().PO2DO(po) + ext, parseErr := entity.ParseExptItemResultExt(gptr.Indirect(po.Ext)) + if parseErr != nil { + return nil, 0, parseErr + } + result.Ext = ext + results = append(results, result) } return results, total, nil } @@ -213,3 +240,41 @@ func (e ExptItemResultRepoImpl) BatchCreateNXRunLogs(ctx context.Context, itemRe func (e ExptItemResultRepoImpl) GetMaxItemIdxByExptID(ctx context.Context, exptID, spaceID int64) (int32, error) { return e.exptItemResultDAO.GetMaxItemIdxByExptID(ctx, exptID, spaceID) } + +func (e ExptItemResultRepoImpl) ScanCurrentRequests(ctx context.Context, spaceID, exptID, runID int64, retryFirst bool) ([]*entity.ExptItemCurrentRequest, error) { + return e.exptItemResultDAO.ScanCurrentRequests(ctx, spaceID, exptID, runID, retryFirst) +} + +func (e ExptItemResultRepoImpl) GetCurrentRequest(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptItemCurrentRequest, error) { + return e.exptItemResultDAO.GetCurrentRequest(ctx, spaceID, exptID, runID, itemID) +} + +// ScanIncompleteDispatchObservations exposes the existing strict primary JOIN to resource +// cleanup, which must also observe runs that have already entered a terminal state. +func (e ExptItemResultRepoImpl) ScanIncompleteDispatchObservations(ctx context.Context, spaceID, exptID, runID int64) ([]*repo.ExptDispatchObservation, error) { + return e.exptItemResultDAO.ScanIncompleteDispatchObservations(ctx, spaceID, exptID, runID) +} + +func (e ExptItemResultRepoImpl) MarkProcessingIfCurrent(ctx context.Context, spaceID, exptID, runID, itemID int64, requestID string) (bool, error) { + return e.exptItemResultDAO.MarkProcessingIfCurrent(ctx, spaceID, exptID, runID, itemID, requestID) +} + +func (e ExptItemResultRepoImpl) GetCurrentItemRunSource(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptCurrentItemRunSource, error) { + return e.exptItemResultDAO.(repo.IExptCurrentResultRepo).GetCurrentItemRunSource(ctx, spaceID, exptID, runID, itemID) +} + +func (e ExptItemResultRepoImpl) StoreCurrentTurnRunResult(ctx context.Context, source *entity.ExptCurrentItemRunSource, result *entity.ExptTurnResultRunLog) (bool, error) { + return e.exptItemResultDAO.(repo.IExptCurrentResultRepo).StoreCurrentTurnRunResult(ctx, source, result) +} + +func (e ExptItemResultRepoImpl) CompleteCurrentItemRun(ctx context.Context, source *entity.ExptCurrentItemRunSource, write *entity.ExptCurrentItemRunWrite) (bool, error) { + return e.exptItemResultDAO.(repo.IExptCurrentResultRepo).CompleteCurrentItemRun(ctx, source, write) +} + +func (e ExptItemResultRepoImpl) ScanManualRetryFilterItems(ctx context.Context, space, expt, run, cursor int64, limit int) ([]int64, int64, error) { + scanner, ok := e.exptItemResultDAO.(repo.IExptManualRetryFilterRepo) + if !ok { + return nil, 0, fmt.Errorf("manual retry filter DAO enumeration missing") + } + return scanner.ScanManualRetryFilterItems(ctx, space, expt, run, cursor, limit) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go index 05cacdc0f1..e21e55c680 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "gorm.io/gorm/clause" @@ -17,6 +18,20 @@ import ( mocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/mocks" ) +func TestExptItemResultRepoRejectsCorruptExt(t *testing.T) { + for _, raw := range []string{`{`, `{"manual_retry":"{"}`, `{"manual_retry":"{}"}`} { + t.Run(raw, func(t *testing.T) { + ctrl := gomock.NewController(t) + dao := mocks.NewMockIExptItemResultDAO(ctrl) + bytes := []byte(raw) + dao.EXPECT().BatchGet(gomock.Any(), int64(9), int64(7), []int64{11}).Return([]*model.ExptItemResult{{ID: 1, Ext: &bytes}}, nil) + got, err := NewExptItemResultRepo(dao).BatchGet(context.Background(), 9, 7, []int64{11}) + require.Error(t, err) + require.Nil(t, got) + }) + } +} + func TestExptItemResultRepoImpl_BatchGet(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_manual_retry_frozen_config_test.go b/backend/modules/evaluation/infra/repo/experiment/expt_manual_retry_frozen_config_test.go new file mode 100644 index 0000000000..c60da686cb --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/expt_manual_retry_frozen_config_test.go @@ -0,0 +1,24 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package experiment + +import ( + "context" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + daomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/mocks" +) + +func TestManualRetryFrozenConfigRejectsCorruption(t *testing.T) { + dao := daomocks.NewMockIExptItemRefDAO(gomock.NewController(t)) + dao.EXPECT().GetByExptIDAndItemID(gomock.Any(), int64(9), int64(7), int64(11)).Return(&model.ExptItemRef{ItemConfig: gptr.Of([]byte(`{`))}, nil) + r := &ExptItemRefRepoImpl{dao: dao} + _, err := r.GetByExptIDAndItemID(context.Background(), 9, 7, 11) + require.Error(t, err, "corrupt frozen config must abort preparation, not turn into an absent target") +} diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_manual_retry_repo_impl.go b/backend/modules/evaluation/infra/repo/experiment/expt_manual_retry_repo_impl.go new file mode 100644 index 0000000000..3e641e5234 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/expt_manual_retry_repo_impl.go @@ -0,0 +1,34 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package experiment + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/infra/idgen" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" +) + +type ExptManualRetryRepoImpl struct { + dao repo.IExptManualRetryRepo +} + +func NewExptManualRetryRepo(provider db.Provider, generator idgen.IIDGenerator) repo.IExptManualRetryRepo { + return &ExptManualRetryRepoImpl{dao: mysql.NewExptManualRetryDAO(provider, generator)} +} + +func (r *ExptManualRetryRepoImpl) FindReceipt(ctx context.Context, spaceID, exptID, runID int64, requestID, hash string) (*entity.ManualRetryReceipt, error) { + return r.dao.FindReceipt(ctx, spaceID, exptID, runID, requestID, hash) +} + +func (r *ExptManualRetryRepoImpl) RequeueTerminalItems(ctx context.Context, request *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryReceipt, error) { + return r.dao.RequeueTerminalItems(ctx, request) +} + +func (r *ExptManualRetryRepoImpl) PreflightTerminalItems(ctx context.Context, request *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) { + return r.dao.PreflightTerminalItems(ctx, request) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_result_publish_repo_impl.go b/backend/modules/evaluation/infra/repo/experiment/expt_result_publish_repo_impl.go new file mode 100644 index 0000000000..cea71d1ab7 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/expt_result_publish_repo_impl.go @@ -0,0 +1,30 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package experiment + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" +) + +type ExptResultPublishRepoImpl struct{ dao repo.IExptResultPublishRepo } + +func NewExptResultPublishRepo(provider db.Provider) repo.IExptResultPublishRepo { + return &ExptResultPublishRepoImpl{dao: mysql.NewExptResultPublishDAO(provider)} +} + +func (r *ExptResultPublishRepoImpl) GetItemResultPublishSource(ctx context.Context, space, expt, run, item int64) (*entity.ExptItemResultPublishSource, error) { + return r.dao.GetItemResultPublishSource(ctx, space, expt, run, item) +} + +func (r *ExptResultPublishRepoImpl) IsItemResultPublishSourceCurrent(ctx context.Context, source *entity.ExptItemResultPublishSource) (bool, error) { + return r.dao.IsItemResultPublishSourceCurrent(ctx, source) +} + +func (r *ExptResultPublishRepoImpl) CommitItemResult(ctx context.Context, publication *entity.ExptItemResultPublication) (bool, error) { + return r.dao.CommitItemResult(ctx, publication) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_run_finish_repo_impl.go b/backend/modules/evaluation/infra/repo/experiment/expt_run_finish_repo_impl.go new file mode 100644 index 0000000000..631f7c78cf --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/expt_run_finish_repo_impl.go @@ -0,0 +1,30 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package experiment + +import ( + "context" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" +) + +type ExptRunFinishRepoImpl struct{ dao repo.IExptRunFinishRepo } + +func NewExptRunFinishRepo(provider db.Provider) repo.IExptRunFinishRepo { + return &ExptRunFinishRepoImpl{dao: mysql.NewExptRunFinishDAO(provider)} +} + +func (r *ExptRunFinishRepoImpl) FinishCurrentRun(ctx context.Context, request *entity.ExptRunFinishRequest) (*entity.ExptRunFinishResult, error) { + return r.dao.FinishCurrentRun(ctx, request) +} + +func (r *ExptRunFinishRepoImpl) TerminateCurrentRun(ctx context.Context, request *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + return r.dao.(repo.IExptRunTerminationRepo).TerminateCurrentRun(ctx, request) +} + +func (r *ExptRunFinishRepoImpl) GetTerminationItemSource(ctx context.Context, space, expt, run, item int64) (*entity.ExptCurrentItemRunSource, error) { + return r.dao.(repo.IExptRunTerminationRepo).GetTerminationItemSource(ctx, space, expt, run, item) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go index a33f7b314f..dce87ca2cb 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go @@ -23,6 +23,19 @@ type ExptRunLogImpl struct { exptRunLogDAO mysql.IExptRunLogDAO } +func (e *ExptRunLogImpl) AppendRunItemIDs(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64) error { + return e.exptRunLogDAO.AppendRunItemIDs(ctx, exptID, exptRunID, spaceID, itemIDs) +} + +func (e *ExptRunLogImpl) UpdateRunStats(ctx context.Context, exptID, exptRunID, spaceID int64, run *entity.ExptRunLog, stats *entity.ExptStats) error { + fields := map[string]any{ + "status": run.Status, "pending_cnt": run.PendingCnt, "processing_cnt": run.ProcessingCnt, + "success_cnt": run.SuccessCnt, "fail_cnt": run.FailCnt, "terminated_cnt": run.TerminatedCnt, + "credit_cost": run.CreditCost, "token_cost": run.TokenCost, "updated_at": time.Now(), + } + return e.exptRunLogDAO.UpdateRunStats(ctx, exptID, exptRunID, spaceID, fields, convert.NewExptStatsConverter().DO2PO(stats)) +} + func (e *ExptRunLogImpl) Get(ctx context.Context, exptID, exptRunID int64) (*entity.ExptRunLog, error) { po, err := e.exptRunLogDAO.Get(ctx, exptID, exptRunID) if err != nil { diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_turn_result_filter_repo_impl.go b/backend/modules/evaluation/infra/repo/experiment/expt_turn_result_filter_repo_impl.go index 70c2ad46cb..e273f836bb 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_turn_result_filter_repo_impl.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_turn_result_filter_repo_impl.go @@ -5,6 +5,7 @@ package experiment import ( "context" + "fmt" "strconv" "time" @@ -274,3 +275,19 @@ func (e *ExptTurnResultFilterRepoImpl) GetByExptIDItemIDs(ctx context.Context, s } return dos, nil } + +func (e *ExptTurnResultFilterRepoImpl) GetMaterializedByExptIDItemIDs(ctx context.Context, space, expt, date string, ids []string) ([]*entity.ExptTurnResultFilterEntity, error) { + reader, ok := e.exptTurnResultFilterDAO.(ck.IExptMaterializedFilterDAO) + if !ok { + return nil, fmt.Errorf("materialized filter DAO missing") + } + rows, err := reader.GetMaterializedByExptIDItemIDs(ctx, space, expt, date, ids) + if err != nil { + return nil, err + } + result := make([]*entity.ExptTurnResultFilterEntity, 0, len(rows)) + for _, row := range rows { + result = append(result, convertor.ExptTurnResultFilterPO2Entity(row)) + } + return result, nil +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go index 32f40e247b..b046f16f61 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go @@ -34,6 +34,7 @@ func (ExptTurnResultRunLogConvertor) DO2PO(log *entity.ExptTurnResultRunLog) (*m ItemVersionID: log.ItemVersionID, // ★ TurnID: log.TurnID, Status: int32(log.Status), + TraceID: log.TraceID, LogID: log.LogID, TargetResultID: log.TargetResultID, EvaluatorResultIds: gptr.Of(evalResIDs), @@ -77,6 +78,7 @@ func (ExptTurnResultRunLogConvertor) PO2DO(log *model.ExptTurnResultRunLog) (*en ItemVersionID: log.ItemVersionID, // ★ TurnID: log.TurnID, Status: entity.TurnRunState(log.Status), + TraceID: log.TraceID, LogID: log.LogID, TargetResultID: log.TargetResultID, EvaluatorResultIds: evalResIDs, @@ -96,7 +98,7 @@ func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) (*model.ExptRunLog, err if log == nil { return nil, nil } - itemIDsBytes, err := json.Marshal(log.ItemIds) + itemIDsBytes, err := entity.EncodeExptRunLogItems(log.ItemIds) if err != nil { return nil, errorx.Wrapf(err, "ExptRunLogItems list json marshal fail") } diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log_ext_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log_ext_test.go index eb83713eab..de82fd2e6e 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log_ext_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log_ext_test.go @@ -6,6 +6,9 @@ package convert import ( "testing" + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" @@ -55,3 +58,24 @@ func TestExptTurnResultRunLogConvertor_Ext(t *testing.T) { assert.Error(t, err) }) } + +func TestExptRunLogReceiptConvertRoundTrip(t *testing.T) { + raw := []byte(`[{"ItemIDs":[11],"CreateAt":1700000000},{"ItemIDs":null,"CreateAt":1700000001,"manual_retry_receipt":{"request_id":"rq1","payload_hash":"hash","accepted_count":1,"rejected_items":[{"item_id":12,"reason":"busy"}],"snapshot_at_ms":1700000000123},"unknown":[1,2,3]}]`) + converter := NewExptRunLogConvertor() + do, err := converter.PO2DO(&model.ExptRunLog{ID: 7, ExptID: 8, SpaceID: 9, ItemIds: gptr.Of(raw)}) + require.NoError(t, err) + require.Equal(t, []int64{11}, do.GetItemIDs()) + po, err := converter.DO2PO(do) + require.NoError(t, err) + require.JSONEq(t, string(raw), string(*po.ItemIds)) +} + +func TestManualRetryTurnLogTraceSnapshot(t *testing.T) { + converter := NewExptTurnResultRunLogConvertor() + got, err := converter.PO2DO(&model.ExptTurnResultRunLog{TraceID: 55}) + require.NoError(t, err) + require.Equal(t, int64(55), got.TraceID, "retry source must include the stored trace identity") + po, err := converter.DO2PO(got) + require.NoError(t, err) + require.Equal(t, int64(55), po.TraceID) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt.go index 23b2c17738..c5b36865e3 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt.go @@ -30,7 +30,7 @@ type IExptDAO interface { Update(ctx context.Context, expt *model.Experiment) error - UpdateFields(ctx context.Context, id int64, ufields map[string]any) error + UpdateFields(ctx context.Context, id int64, ufields map[string]any, opts ...db.Option) error Delete(ctx context.Context, id int64) error @@ -40,7 +40,7 @@ type IExptDAO interface { GetByName(ctx context.Context, name string, spaceID int64) (*model.Experiment, error) - GetByID(ctx context.Context, id int64) (*model.Experiment, error) + GetByID(ctx context.Context, id int64, opts ...db.Option) (*model.Experiment, error) MGetByID(ctx context.Context, ids []int64) ([]*model.Experiment, error) @@ -72,8 +72,8 @@ type exptDAOImpl struct { query *query.Query } -func (d *exptDAOImpl) UpdateFields(ctx context.Context, id int64, ufields map[string]any) error { - q := query.Use(d.db.NewSession(ctx)).Experiment +func (d *exptDAOImpl) UpdateFields(ctx context.Context, id int64, ufields map[string]any, opts ...db.Option) error { + q := query.Use(d.db.NewSession(ctx, opts...)).Experiment _, err := q.WithContext(ctx). Where(q.ID.Eq(id)). UpdateColumns(ufields) @@ -389,8 +389,8 @@ func (d *exptDAOImpl) GetByName(ctx context.Context, name string, spaceID int64) return found, nil } -func (d *exptDAOImpl) GetByID(ctx context.Context, id int64) (*model.Experiment, error) { - expt := d.query.Experiment +func (d *exptDAOImpl) GetByID(ctx context.Context, id int64, opts ...db.Option) (*model.Experiment, error) { + expt := query.Use(d.db.NewSession(ctx, opts...)).Experiment q := expt.WithContext(ctx) if contexts.CtxWriteDB(ctx) { q = q.WriteDB() diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_current_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_current_result.go new file mode 100644 index 0000000000..36791c8b9b --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_current_result.go @@ -0,0 +1,403 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "errors" + "fmt" + "reflect" + + "github.com/bytedance/gg/gptr" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +func (dao *exptItemResultDAOImpl) GetCurrentItemRunSource(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptCurrentItemRunSource, error) { + var result *entity.ExptCurrentItemRunSource + err := dao.provider.Transaction(ctx, func(tx *gorm.DB) error { + var err error + result, err = loadCurrentRunSource(tx, spaceID, exptID, runID, itemID) + return err + }, db.WithMaster()) + return result, err +} + +func loadCurrentRunSource(tx *gorm.DB, spaceID, exptID, runID, itemID int64) (*entity.ExptCurrentItemRunSource, error) { + return loadItemRunSource(tx, spaceID, exptID, runID, itemID, false) +} + +// Publication uses the same primary row/source validation, with a separate run +// lifecycle rule. Execution remains active-only through loadCurrentRunSource. +func loadItemRunSource(tx *gorm.DB, spaceID, exptID, runID, itemID int64, publication bool) (*entity.ExptCurrentItemRunSource, error) { + return readItemRunSource(tx, spaceID, exptID, runID, itemID, publication, true) +} + +// Scan observations use only consistent nonlocking reads. Execution and +// publication validation/commit keep the canonical-first locking path above. +func readItemRunSource(tx *gorm.DB, spaceID, exptID, runID, itemID int64, publication, lockRows bool) (*entity.ExptCurrentItemRunSource, error) { + var locks []clause.Expression + if lockRows { + locks = append(locks, clause.Locking{Strength: "UPDATE"}) + } + var item model.ExptItemResult + err := tx.Clauses(locks...).Where("space_id=? AND expt_id=? AND item_id=?", spaceID, exptID, itemID).Take(&item).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + if item.ExptRunID != runID { + return nil, nil + } + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + if err != nil { + return nil, err + } + manual, err := entity.ParseManualRetryContext(ext) + if err != nil { + return nil, err + } + request := "" + if manual != nil { + if manual.RunID != runID { + return nil, fmt.Errorf("manual current source run mismatch") + } + request = manual.RequestID + } + var run model.ExptRunLog + if err := tx.Where("space_id=? AND expt_id=? AND expt_run_id=?", spaceID, exptID, runID).Take(&run).Error; err != nil { + return nil, err + } + if run.Status == nil { + return nil, fmt.Errorf("source run status missing") + } + runStatus := entity.ExptStatus(*run.Status) + if !publication { + if entity.IsExptFinished(runStatus) || entity.IsExptFinishing(runStatus) { + return nil, nil + } + if runStatus != entity.ExptStatus_Processing && runStatus != entity.ExptStatus_Pending { + return nil, fmt.Errorf("invalid source run status") + } + } else if runStatus != entity.ExptStatus_Processing && runStatus != entity.ExptStatus_Pending && !entity.IsExptFinished(runStatus) && !entity.IsExptFinishing(runStatus) { + return nil, fmt.Errorf("invalid publication run status") + } + mode := entity.ExptRunMode(gptr.Indirect(run.Mode)) + if manual != nil { + mode = manual.Mode + } + if mode < entity.EvaluationModeSubmit || mode > entity.EvaluationModeTrialRun { + return nil, fmt.Errorf("invalid current source mode") + } + var log model.ExptItemResultRunLog + if err := tx.Clauses(locks...).Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", spaceID, exptID, runID, itemID).Take(&log).Error; err != nil { + return nil, err + } + if item.ItemVersionID != log.ItemVersionID || log.RetryTimes < 0 || !validCurrentItemStatus(entity.ItemRunState(item.Status)) || !validCurrentItemStatus(entity.ItemRunState(log.Status)) || log.ResultState == nil || *log.ResultState < 0 || *log.ResultState > int32(entity.ExptItemResultStateLogged) { + return nil, fmt.Errorf("damaged current item source") + } + source := &entity.ExptCurrentItemRunSource{Current: entity.ExptItemCurrentRequest{SpaceID: spaceID, ExptID: exptID, RunID: runID, ItemID: itemID, ItemVersionID: item.ItemVersionID, RequestID: request, Mode: mode, Status: entity.ItemRunState(log.Status), ResultState: entity.ExptItemResultState(*log.ResultState), RetryTimes: log.RetryTimes, UpdatedAt: gptr.Of(log.UpdatedAt)}, Turns: map[int64]*entity.ExptTurnResultRunLog{}} + if entity.IsItemRunFinished(entity.ItemRunState(item.Status)) && item.Status != log.Status { + source.Current.Status = entity.ItemRunState(item.Status) + } + var canonical []model.ExptTurnResult + if err := tx.Clauses(locks...).Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", spaceID, exptID, runID, itemID).Order("id ASC").Find(&canonical).Error; err != nil { + return nil, err + } + var logs []model.ExptTurnResultRunLog + if err := tx.Clauses(locks...).Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", spaceID, exptID, runID, itemID).Order("id ASC").Find(&logs).Error; err != nil { + return nil, err + } + canonicalIDs := make([]int64, 0, len(canonical)) + byTurn := map[int64]model.ExptTurnResult{} + for _, turn := range canonical { + if turn.ItemVersionID != item.ItemVersionID || turn.TurnID < 0 { + return nil, fmt.Errorf("damaged canonical turn source") + } + byTurn[turn.TurnID] = turn + canonicalIDs = append(canonicalIDs, turn.ID) + source.CanonicalTurns = append(source.CanonicalTurns, convert.NewExptTurnResultConvertor().PO2DO(&turn, nil)) + } + for index := range logs { + log := &logs[index] + turn, ok := byTurn[log.TurnID] + if !ok || log.ItemVersionID != turn.ItemVersionID || log.TurnID < 0 || log.Status < int32(entity.TurnRunState_Queueing) || log.Status > int32(entity.TurnRunState_Terminal) { + return nil, fmt.Errorf("damaged current turn source") + } + converted, err := convert.NewExptTurnResultRunLogConvertor().PO2DO(log) + if err != nil { + return nil, err + } + if err := validateCurrentEvaluatorRefs(converted.EvaluatorResultIds); err != nil { + return nil, err + } + source.Turns[log.TurnID] = converted + } + // A failed first turn need not execute all later turns, but every frozen turn log exists. + if manual != nil && (len(canonical) != len(logs) || len(canonical) == 0) { + return nil, fmt.Errorf("current frozen turn logs incomplete") + } + var refs []model.ExptTurnEvaluatorResultRef + if err := tx.Clauses(locks...).Where("space_id=? AND expt_id=? AND expt_turn_result_id IN ?", spaceID, exptID, canonicalIDs).Order("id ASC").Find(&refs).Error; err != nil { + return nil, err + } + source.CurrentRefs = make([]*entity.ExptTurnEvaluatorResultRef, 0, len(refs)) + for index := range refs { + source.CurrentRefs = append(source.CurrentRefs, convert.NewExptTurnEvaluatorResultRefConvertor().PO2DO(&refs[index])) + } + return source, nil +} + +func validateCurrentEvaluatorRefs(refs *entity.EvaluatorResults) error { + if refs == nil { + return nil + } + seen := map[string]bool{} + for _, ref := range refs.Registered { + if ref == nil || ref.VersionID <= 0 || ref.RecordID <= 0 { + return fmt.Errorf("incomplete registered source reference") + } + key := fmt.Sprintf("r:%d:%s", ref.VersionID, ref.Alias) + if seen[key] { + return fmt.Errorf("duplicate source reference") + } + seen[key] = true + } + for _, ref := range refs.Inline { + if ref == nil || ref.InlineKey == "" || ref.RecordID <= 0 { + return fmt.Errorf("incomplete inline source reference") + } + key := "i:" + ref.InlineKey + if seen[key] { + return fmt.Errorf("duplicate source reference") + } + seen[key] = true + } + for version, record := range refs.EvalVerIDToResID { + if version <= 0 || record <= 0 { + return fmt.Errorf("incomplete legacy source reference") + } + } + return nil +} + +func currentSourceMatches(expected, actual *entity.ExptCurrentItemRunSource) bool { + if expected == nil || actual == nil { + return false + } + a, b := expected.Current, actual.Current + if a.SpaceID != b.SpaceID || a.ExptID != b.ExptID || a.RunID != b.RunID || a.ItemID != b.ItemID || a.ItemVersionID != b.ItemVersionID || a.RequestID != b.RequestID || a.Mode != b.Mode || len(expected.Turns) != len(actual.Turns) || !reflect.DeepEqual(expected.CurrentRefs, actual.CurrentRefs) { + return false + } + if len(expected.CanonicalTurns) != len(actual.CanonicalTurns) { + return false + } + for index, turn := range expected.CanonicalTurns { + other := actual.CanonicalTurns[index] + if turn == nil || other == nil || turn.ID != other.ID || turn.TurnID != other.TurnID || turn.ItemVersionID != other.ItemVersionID || turn.ExptRunID != other.ExptRunID || turn.TargetResultID != other.TargetResultID { + return false + } + } + for id, turn := range expected.Turns { + other := actual.Turns[id] + if turn == nil || other == nil || turn.ID != other.ID || turn.TargetResultID != other.TargetResultID || turn.ItemVersionID != other.ItemVersionID || !entity.SameCurrentEvaluatorRefs(turn.EvaluatorResultIds, other.EvaluatorResultIds) { + return false + } + } + return true +} + +func (dao *exptItemResultDAOImpl) StoreCurrentTurnRunResult(ctx context.Context, source *entity.ExptCurrentItemRunSource, result *entity.ExptTurnResultRunLog) (bool, error) { + if source == nil || result == nil { + return false, fmt.Errorf("execution source or turn result missing") + } + po, err := convert.NewExptTurnResultRunLogConvertor().DO2PO(result) + if err != nil { + return false, err + } + if err := validateCurrentEvaluatorRefs(result.EvaluatorResultIds); err != nil { + return false, err + } + applied := false + err = dao.provider.Transaction(ctx, func(tx *gorm.DB) error { + actual, err := loadCurrentRunSource(tx, source.Current.SpaceID, source.Current.ExptID, source.Current.RunID, source.Current.ItemID) + if err != nil { + return err + } + if !currentSourceMatches(source, actual) || entity.IsItemRunFinished(actual.Current.Status) || actual.Current.ResultState == entity.ExptItemResultStateResulted { + return nil + } + old := actual.Turns[result.TurnID] + if old == nil || result.ID != old.ID || result.SpaceID != source.Current.SpaceID || result.ExptID != source.Current.ExptID || result.ExptRunID != source.Current.RunID || result.ItemID != source.Current.ItemID || result.ItemVersionID != source.Current.ItemVersionID { + return fmt.Errorf("foreign execution turn result") + } + if old.Status == entity.TurnRunState_Success || old.Status == entity.TurnRunState_Terminal { + return nil + } + fields := map[string]any{"status": po.Status, "trace_id": po.TraceID, "log_id": po.LogID, "target_result_id": po.TargetResultID, "evaluator_result_ids": gptr.Indirect(po.EvaluatorResultIds), "err_msg": gptr.Indirect(po.ErrMsg), "ext": po.Ext} + written := tx.Model(&model.ExptTurnResultRunLog{}).Where("id=? AND space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND item_version_id=?", old.ID, source.Current.SpaceID, source.Current.ExptID, source.Current.RunID, source.Current.ItemID, source.Current.ItemVersionID).Updates(fields) + if written.Error != nil { + return written.Error + } + if written.RowsAffected != 1 { + return fmt.Errorf("current execution turn disappeared") + } + applied = true + return ctx.Err() + }, db.WithMaster()) + return applied && err == nil, err +} + +func (dao *exptItemResultDAOImpl) CompleteCurrentItemRun(ctx context.Context, source *entity.ExptCurrentItemRunSource, write *entity.ExptCurrentItemRunWrite) (bool, error) { + if source == nil || write == nil || write.RetryTimes < 0 { + return false, fmt.Errorf("execution source or write missing") + } + applied := false + err := dao.provider.Transaction(ctx, func(tx *gorm.DB) error { + actual, err := loadCurrentRunSource(tx, source.Current.SpaceID, source.Current.ExptID, source.Current.RunID, source.Current.ItemID) + if err != nil { + return err + } + if !currentSourceMatches(source, actual) || actual.Current.ResultState == entity.ExptItemResultStateResulted { + return nil + } + if write.OnlyIfQueueingUnreserved { + // Reuse the canonical-first locked PO reader: the source DTO does not + // carry the canonical status or a populated quota reservation state. + locked, err := lockDispatchItem(tx, actual.Current.SpaceID, actual.Current.ExptID, actual.Current.RunID, actual.Current.ItemID, actual.Current.RequestID) + if err != nil { + return err + } + if locked == nil || locked.item.Status != int32(entity.ItemRunState_Queueing) || locked.log.Status != int32(entity.ItemRunState_Queueing) || locked.log.QuotaReservationState != int32(entity.QuotaReservationStateNone) { + return nil + } + } + c := actual.Current + if entity.IsItemRunFinished(c.Status) && (c.Status != entity.ItemRunState_Terminal || c.ResultState != entity.ExptItemResultStateDefault) { + return nil + } + if !write.RetryOnly && len(actual.Turns) != len(actual.CanonicalTurns) { + if actual.Current.RequestID != "" { + return fmt.Errorf("manual execution turn source incomplete") + } + prepared := map[int64]*entity.ExptTurnResultRunLog{} + for _, log := range write.MissingTurnLogs { + if log == nil || log.ID <= 0 || log.SpaceID != actual.Current.SpaceID || log.ExptID != actual.Current.ExptID || log.ExptRunID != actual.Current.RunID || log.ItemID != actual.Current.ItemID || log.ItemVersionID != actual.Current.ItemVersionID { + return fmt.Errorf("missing raw turn log preparation invalid") + } + prepared[log.TurnID] = log + } + for _, turn := range actual.CanonicalTurns { + if actual.Turns[turn.TurnID] != nil { + continue + } + log := prepared[turn.TurnID] + if log == nil { + return fmt.Errorf("raw turn log preparation missing") + } + po, err := convert.NewExptTurnResultRunLogConvertor().DO2PO(log) + if err != nil { + return err + } + if err := tx.Create(po).Error; err != nil { + return err + } + } + } + + retry := max(c.RetryTimes, write.RetryTimes) + fields := map[string]any{"retry_times": retry} + next := write.Status + if write.RetryOnly { + if retry == c.RetryTimes { + applied = true + return nil + } + } else { + if write.Yield { + next = entity.ItemRunState_Queueing + if c.Status != entity.ItemRunState_Processing { + return nil + } + } else if next != entity.ItemRunState_Success && next != entity.ItemRunState_Fail && next != entity.ItemRunState_Terminal { + return fmt.Errorf("invalid execution completion status") + } + if c.Status == entity.ItemRunState_Terminal { + next = entity.ItemRunState_Terminal + } else { + fields["err_msg"] = write.ErrMsg + } + fields["status"] = int32(next) + result := entity.ExptItemResultStateLogged + if write.Yield { + result = entity.ExptItemResultStateDefault + } + fields["result_state"] = int32(result) + } + scope := "space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND item_version_id=?" + args := []any{c.SpaceID, c.ExptID, c.RunID, c.ItemID, c.ItemVersionID} + if err := tx.Model(&model.ExptItemResultRunLog{}).Where(scope, args...).Updates(fields).Error; err != nil { + return err + } + if !write.RetryOnly { + if write.Yield || write.FailPendingTurns || next == entity.ItemRunState_Terminal { + turnState := entity.TurnRunState_Fail + if write.Yield { + turnState = entity.TurnRunState_Queueing + } + if next == entity.ItemRunState_Terminal { + turnState = entity.TurnRunState_Terminal + } + turnFields := map[string]any{"status": int32(turnState)} + if !write.Yield { + turnFields["err_msg"] = write.TurnError(turnState) + } + if err := tx.Model(&model.ExptTurnResultRunLog{}).Where(scope, args...).Where("status NOT IN ?", []int32{int32(entity.TurnRunState_Success), int32(entity.TurnRunState_Terminal)}).Updates(turnFields).Error; err != nil { + return err + } + } + if write.Yield { + if err := tx.Model(&model.ExptTurnResult{}).Where(scope, args...).Where("status NOT IN ?", []int32{int32(entity.TurnRunState_Success), int32(entity.TurnRunState_Terminal)}).Updates(map[string]any{"status": int32(entity.TurnRunState_Queueing)}).Error; err != nil { + return err + } + } + var canonical model.ExptItemResult + if err := tx.Where("space_id=? AND expt_id=? AND item_id=?", c.SpaceID, c.ExptID, c.ItemID).Take(&canonical).Error; err != nil { + return err + } + prior := entity.ItemRunState(canonical.Status) + canonicalFields := map[string]any{"status": int32(next)} + if next != entity.ItemRunState_Terminal { + canonicalFields["err_msg"] = write.ErrMsg + } + if err := tx.Model(&model.ExptItemResult{}).Where(scope, args...).Updates(canonicalFields).Error; err != nil { + return err + } + if prior != next { + var stats model.ExptStats + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=?", c.SpaceID, c.ExptID).Take(&stats).Error; err != nil { + return err + } + columns := map[entity.ItemRunState]string{entity.ItemRunState_Queueing: "pending_cnt", entity.ItemRunState_Processing: "processing_cnt", entity.ItemRunState_Success: "success_cnt", entity.ItemRunState_Fail: "fail_cnt", entity.ItemRunState_Terminal: "terminated_cnt"} + from, to := columns[prior], columns[next] + if from == "" || to == "" { + return fmt.Errorf("invalid completion stats source") + } + if err := tx.Model(&model.ExptStats{}).Where("id=? AND space_id=? AND expt_id=?", stats.ID, c.SpaceID, c.ExptID).Updates(map[string]any{from: gorm.Expr(from + "-1"), to: gorm.Expr(to + "+1")}).Error; err != nil { + return err + } + } + } + applied = true + return ctx.Err() + }, db.WithMaster()) + return applied && err == nil, err +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_current_result_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_current_result_test.go new file mode 100644 index 0000000000..27560d337d --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_current_result_test.go @@ -0,0 +1,374 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +type currentResultFixture struct { + t *testing.T + provider db.Provider + conn *gorm.DB + port repo.IExptCurrentResultRepo + base, space, expt, run, item, version int64 +} + +func newCurrentResultFixture(t *testing.T, token string) *currentResultFixture { + t.Helper() + dsn := os.Getenv("EXPT_RETRY_MYSQL_DSN") + if dsn == "" { + t.Skip("EXPT_RETRY_MYSQL_DSN missing; REAL MYSQL NOT EXECUTED") + } + provider, err := db.NewDB(mysql.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + conn := provider.NewSession(context.Background(), db.WithMaster()) + var identity struct{ Version, Schema string } + require.NoError(t, conn.Raw("SELECT VERSION() AS version,DATABASE() AS `schema`").Scan(&identity).Error) + require.Equal(t, "expt_running_retry_test", identity.Schema) + t.Logf("REAL_MYSQL_EXECUTED version=%s schema=%s", identity.Version, identity.Schema) + base := int64(7384603622000000000) + time.Now().UnixNano()%1000000000000 + retryFixtureSeq.Add(1)*10000 + f := ¤tResultFixture{t: t, provider: provider, conn: conn, base: base, space: base + 1, expt: base + 2, run: base + 3, item: base + 11, version: base + 31} + t.Cleanup(func() { + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_item_ref", "expt_run_log", "expt_stats"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + } + require.NoError(t, conn.Exec("DELETE FROM experiment WHERE space_id=? AND id=?", f.space, f.expt).Error) + for _, table := range []string{"eval_target_record", "evaluator_record"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id=?", f.space).Error) + } + sqlDB, err := conn.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + }) + require.NoError(t, conn.Exec("INSERT INTO experiment (id,space_id,latest_run_id,status,expt_type) VALUES (?,?,?,?,?)", f.expt, f.space, f.run, entity.ExptStatus_Processing, entity.ExptType_Offline).Error) + require.NoError(t, conn.Create(&model.ExptRunLog{ID: f.run, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit)), ItemIds: gptr.Of([]byte(`[{"ItemIDs":[],"CreateAt":1700000000}]`))}).Error) + require.NoError(t, conn.Create(&model.ExptStats{ID: base + 4, SpaceID: f.space, ExptID: f.expt, ProcessingCnt: 2, PendingCnt: 1}).Error) + for index, status := range []entity.ItemRunState{entity.ItemRunState_Processing, entity.ItemRunState_Processing, entity.ItemRunState_Queueing} { + item := f.item + int64(index) + require.NoError(t, conn.Create(&model.ExptItemResult{ID: base + 100 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: f.version, Status: int32(status), Ext: gptr.Of([]byte(`{"other":"keep"}`))}).Error) + require.NoError(t, conn.Create(&model.ExptItemResultRunLog{ID: base + 200 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: f.version, Status: int32(status), RetryTimes: 1, ResultState: gptr.Of(int32(entity.ExptItemResultStateDefault))}).Error) + require.NoError(t, conn.Create(&model.ExptItemRef{ID: base + 300 + int64(index), SpaceID: f.space, ExptID: f.expt, ItemID: item, ItemVersionID: f.version}).Error) + } + f.token(token) + refs, _ := json.Marshal(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: base + 51, Alias: "alias", RecordID: base + 61}}, Inline: []*entity.InlineEvalResult{{InlineKey: "inline", RecordID: base + 62}}}) + for index, turn := range []int64{0, base + 22} { + require.NoError(t, conn.Create(&model.ExptTurnResult{ID: base + 400 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: turn, Status: int32(entity.TurnRunState_Processing), TargetResultID: base + 41, WeightedScore: gptr.Of(0.0)}).Error) + require.NoError(t, conn.Create(&model.ExptTurnResultRunLog{ID: base + 500 + int64(index), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: f.version, TurnID: turn, Status: int32(entity.TurnRunState_Processing), TargetResultID: base + 41, EvaluatorResultIds: &refs, Ext: []byte(`{"other":"keep"}`)}).Error) + } + require.NoError(t, conn.Create(&model.ExptTurnEvaluatorResultRef{ID: base + 600, SpaceID: f.space, ExptID: f.expt, ExptTurnResultID: base + 400, EvaluatorVersionID: base + 51, Alias_: "alias", EvaluatorResultID: base + 61}).Error) + require.NoError(t, conn.Exec("INSERT INTO eval_target_record (id,space_id,target_id,target_version_id,experiment_run_id,item_id,item_version_id,turn_id,log_id,trace_id,status) VALUES (?,?,?,?,?,?,?,?,?,?,?)", base+41, f.space, base+42, base+43, f.run, f.item, f.version, 0, "target", "trace", 1).Error) + require.NoError(t, conn.Exec("INSERT INTO evaluator_record (id,space_id,evaluator_version_id,experiment_id,experiment_run_id,item_id,item_version_id,turn_id,trace_id,status,target_record_id) VALUES (?,?,?,?,?,?,?,?,?,?,?)", base+61, f.space, base+51, f.expt, f.run, f.item, f.version, 0, "trace", 1, 0).Error) + port, ok := any(NewExptItemResultDAO(provider)).(repo.IExptCurrentResultRepo) + require.True(t, ok, "source conditional execution transaction missing") + f.port = port + return f +} + +func (f *currentResultFixture) token(token string) { + f.t.Helper() + ext := map[string]string{"other": "keep"} + if token != "" { + raw, err := json.Marshal(&entity.ManualRetryContext{RunID: f.run, RequestID: token, Mode: entity.EvaluationModeFailRetry, Operator: "test", AcceptedAtMS: 1}) + require.NoError(f.t, err) + ext[entity.ManualRetryExtKey] = string(raw) + } + raw, err := json.Marshal(ext) + require.NoError(f.t, err) + require.NoError(f.t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Update("ext", raw).Error) +} + +func (f *currentResultFixture) source() *entity.ExptCurrentItemRunSource { + f.t.Helper() + s, err := f.port.GetCurrentItemRunSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(f.t, err) + require.NotNil(f.t, s) + return s +} + +func (f *currentResultFixture) snapshot() map[string]string { + f.t.Helper() + out := map[string]string{} + for _, table := range []string{"experiment", "expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_turn_evaluator_result_ref", "expt_stats", "expt_run_log", "expt_item_ref", "eval_target_record", "evaluator_record"} { + var rows []map[string]any + require.NoError(f.t, f.conn.Table(table).Where("space_id=?", f.space).Order("id").Find(&rows).Error) + raw, err := json.Marshal(rows) + require.NoError(f.t, err) + out[table] = string(raw) + } + return out +} + +func (f *currentResultFixture) result(source *entity.ExptCurrentItemRunSource) *entity.ExptTurnResultRunLog { + r := *source.Turns[0] + r.Status = entity.TurnRunState_Success + r.TargetResultID = f.base + 71 + r.EvaluatorResultIds = &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: f.base + 81}}} + return &r +} + +// Real SQL, not mocked Transaction: late sources must leave all eleven tables byte-for-byte unchanged. +func TestCurrentResultMySQLLateSource(t *testing.T) { + for _, kind := range []string{"rq2", "same_request_invoke", "empty_to_manual", "old_run", "wrong_version", "alias", "inline", "resulted", "cancelled"} { + t.Run(kind, func(t *testing.T) { + token := "rq1" + if kind == "empty_to_manual" { + token = "" + } + f := newCurrentResultFixture(t, token) + source := f.source() + switch kind { + case "rq2", "empty_to_manual": + f.token("rq2") + case "same_request_invoke": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=0", f.space).Update("target_result_id", f.base+42).Error) + case "old_run": + source.Current.RunID++ + case "wrong_version": + source.Current.ItemVersionID++ + case "alias", "inline": + refs := &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "newalias", RecordID: f.base + 61}}, Inline: []*entity.InlineEvalResult{{InlineKey: "newinline", RecordID: f.base + 62}}} + raw, _ := json.Marshal(refs) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=0", f.space).Update("evaluator_result_ids", raw).Error) + case "resulted": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": int32(entity.ExptItemResultStateResulted)}).Error) + } + ctx := context.Background() + if kind == "cancelled" { + c, cancel := context.WithCancel(ctx) + cancel() + ctx = c + } + before := f.snapshot() + ok, err := f.port.StoreCurrentTurnRunResult(ctx, source, f.result(source)) + if kind == "cancelled" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.False(t, ok) + require.Equal(t, before, f.snapshot()) + for _, write := range []*entity.ExptCurrentItemRunWrite{{Status: entity.ItemRunState_Fail, ErrMsg: []byte("late error"), FailPendingTurns: true}, {Yield: true, RetryTimes: 2}, {RetryOnly: true, RetryTimes: 3}} { + ok, err = f.port.CompleteCurrentItemRun(ctx, source, write) + if kind == "cancelled" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.False(t, ok) + require.Equal(t, before, f.snapshot()) + } + }) + } +} + +func TestCurrentResultMySQLCurrentAtomic(t *testing.T) { + for _, kind := range []string{"store", "complete", "yield", "budget", "first_turn_failed", "terminal", "oldmap", "builtin0"} { + t.Run(kind, func(t *testing.T) { + f := newCurrentResultFixture(t, "rq1") + if kind == "terminal" { + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Terminal)).Error) + } + source := f.source() + before := f.snapshot() + var ok bool + var err error + switch kind { + case "store", "oldmap", "builtin0": + result := f.result(source) + if kind == "oldmap" { + result.EvaluatorResultIds = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{f.base + 51: f.base + 81}} + } + if kind == "builtin0" { + result.TargetResultID = 0 + } + ok, err = f.port.StoreCurrentTurnRunResult(context.Background(), source, result) + case "yield": + ok, err = f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Yield: true, RetryTimes: 2, ErrMsg: []byte("retry")}) + case "budget": + ok, err = f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{RetryOnly: true, RetryTimes: 3}) + default: + ok, err = f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, RetryTimes: 1, ErrMsg: []byte("failed"), FailPendingTurns: true}) + } + require.NoError(t, err) + require.True(t, ok) + after := f.snapshot() + require.NotEqual(t, before, after) + for _, table := range []string{"experiment", "expt_item_ref", "expt_run_log", "eval_target_record", "evaluator_record", "expt_turn_evaluator_result_ref"} { + require.Equal(t, before[table], after[table], table) + } + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&log).Error) + if kind == "yield" { + require.Equal(t, int32(2), log.RetryTimes) + require.Equal(t, int32(entity.ItemRunState_Queueing), log.Status) + } + if kind == "budget" { + require.Equal(t, int32(3), log.RetryTimes) + } + if kind == "complete" || kind == "first_turn_failed" || kind == "terminal" { + require.Equal(t, int32(entity.ExptItemResultStateLogged), gptr.Indirect(log.ResultState)) + expected := entity.ItemRunState_Fail + if kind == "terminal" { + expected = entity.ItemRunState_Terminal + } + require.Equal(t, int32(expected), log.Status) + source.Current.Status = expected + before = f.snapshot() + ok, err = f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Status: expected, RetryTimes: 1}) + require.NoError(t, err) + require.False(t, ok) + require.Equal(t, before, f.snapshot()) + } + }) + } +} + +func TestCurrentResultMySQLRollback(t *testing.T) { + for _, kind := range []string{"store", "complete", "yield", "budget"} { + for _, table := range []string{"expt_item_result_run_log", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result", "expt_stats"} { + if kind == "store" && table != "expt_turn_result_run_log" || kind == "budget" && table != "expt_item_result_run_log" || table == "expt_turn_result" && kind != "yield" { + continue + } + t.Run(kind+"/"+table, func(t *testing.T) { + f := newCurrentResultFixture(t, "rq1") + source := f.source() + before := f.snapshot() + injected := errors.New("injected current write fault") + hook := fmt.Sprintf("fault_%s_%s", kind, table) + hits := 0 + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(hook, func(tx *gorm.DB) { + if tx.Statement.Table == table { + hits++ + _ = tx.AddError(injected) + } + })) + var ok bool + var err error + if kind == "store" { + ok, err = f.port.StoreCurrentTurnRunResult(context.Background(), source, f.result(source)) + } else { + ok, err = f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, FailPendingTurns: true, Yield: kind == "yield", RetryOnly: kind == "budget", RetryTimes: 3}) + } + require.NoError(t, f.conn.Callback().Update().Remove(hook)) + require.Positive(t, hits, "actual writepoint not exercised") + require.ErrorIs(t, err, injected) + require.False(t, ok) + require.Equal(t, before, f.snapshot()) + }) + } + } +} + +func TestCurrentResultMySQLConcurrentAdmission(t *testing.T) { + f := newCurrentResultFixture(t, "rq1") + source := f.source() + var wg sync.WaitGroup + wg.Add(1) + ready := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer wg.Done() + require.NoError(t, f.provider.Transaction(context.Background(), func(tx *gorm.DB) error { + var row model.ExptItemResult + if err := tx.Set("gorm:query_option", "FOR UPDATE").Raw("SELECT * FROM expt_item_result WHERE space_id=? AND item_id=? FOR UPDATE", f.space, f.item).Scan(&row).Error; err != nil { + return err + } + ext := []byte(fmt.Sprintf(`{"manual_retry":"{\"run_id\":%d,\"request_id\":\"rq2\",\"mode\":2,\"operator\":\"test\",\"accepted_at_ms\":1}"}`, f.run)) + if err := tx.Model(&row).Update("ext", ext).Error; err != nil { + return err + } + close(ready) + <-done + return nil + }, db.WithMaster())) + }() + <-ready + result := make(chan error, 1) + go func() { + ok, err := f.port.StoreCurrentTurnRunResult(context.Background(), source, f.result(source)) + if ok { + err = errors.New("late store wrote after admission") + } + result <- err + }() + time.Sleep(30 * time.Millisecond) + close(done) + wg.Wait() + require.NoError(t, <-result) +} + +// A raw pre-execution failure still needs a log for every canonical turn to archive. +func TestCurrentResultMySQLRawMissingTurnLogs(t *testing.T) { + f := newCurrentResultFixture(t, "") + require.NoError(t, f.conn.Unscoped().Where("space_id=?", f.space).Delete(&model.ExptTurnResultRunLog{}).Error) + source := f.source() + before := f.snapshot() + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, ErrMsg: []byte("pre-exec error"), FailPendingTurns: true}) + require.Error(t, err, "incomplete raw failure cannot silently create Logged with no turns") + require.False(t, applied) + require.Equal(t, before, f.snapshot()) +} + +func TestCurrentResultMySQLRawPreparedMissingTurns(t *testing.T) { + for _, fault := range []bool{false, true} { + t.Run(fmt.Sprint(fault), func(t *testing.T) { + f := newCurrentResultFixture(t, "") + require.NoError(t, f.conn.Unscoped().Where("space_id=?", f.space).Delete(&model.ExptTurnResultRunLog{}).Error) + source := f.source() + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, ErrMsg: []byte("original pre-exec error"), FailPendingTurns: true} + for index, turn := range source.CanonicalTurns { + log := turn.ToRunLogDO() + log.ID = f.base + 900 + int64(index) + log.ExptRunID = f.run + log.Status = entity.TurnRunState_Fail + write.MissingTurnLogs = append(write.MissingTurnLogs, log) + } + before := f.snapshot() + injected := errors.New("raw missing log write fault") + if fault { + require.NoError(t, f.conn.Callback().Create().Before("gorm:create").Register("missing_raw_log_fault", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_turn_result_run_log" { + _ = tx.AddError(injected) + } + })) + } + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, write) + if fault { + require.NoError(t, f.conn.Callback().Create().Remove("missing_raw_log_fault")) + require.ErrorIs(t, err, injected) + require.False(t, applied) + require.Equal(t, before, f.snapshot()) + return + } + require.NoError(t, err) + require.True(t, applied) + var logs []model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=?", f.space).Find(&logs).Error) + require.Len(t, logs, 2) + for _, log := range logs { + require.Equal(t, f.version, log.ItemVersionID) + require.Equal(t, int32(entity.TurnRunState_Fail), log.Status) + } + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_impossible_state_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_impossible_state_test.go new file mode 100644 index 0000000000..a79c503463 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_impossible_state_test.go @@ -0,0 +1,457 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "database/sql" + "errors" + "sync" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + drivermysql "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +func newImpossibleStateFixture(t *testing.T, token string) *currentResultFixture { + t.Helper() + f := newCurrentResultFixture(t, token) + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Update("status", 0).Error) + } + require.NoError(t, f.conn.Model(&model.ExptStats{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Updates(map[string]any{"pending_cnt": 2, "processing_cnt": 1}).Error) + return f +} + +func impossibleStateWrite() *entity.ExptCurrentItemRunWrite { + return &entity.ExptCurrentItemRunWrite{OnlyIfQueueingUnreserved: true, Status: entity.ItemRunState_Fail, ErrMsg: []byte("impossible quota"), RetryTimes: 1, FailPendingTurns: true} +} + +func requireImpossibleStats(t *testing.T, f *currentResultFixture, pending, processing, failed int32) { + t.Helper() + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Take(&stats).Error) + require.Equal(t, pending, stats.PendingCnt) + require.Equal(t, processing, stats.ProcessingCnt) + require.Equal(t, failed, stats.FailCnt) + require.Zero(t, stats.SuccessCnt) + require.Zero(t, stats.TerminatedCnt) +} + +func prepareImpossibleMissingTurns(t *testing.T, f *currentResultFixture, source *entity.ExptCurrentItemRunSource, write *entity.ExptCurrentItemRunWrite) { + t.Helper() + for index, turn := range source.CanonicalTurns { + log := turn.ToRunLogDO() + log.ID = f.base + 900 + int64(index) + log.ExptRunID = f.run + log.Status = entity.TurnRunState_Fail + write.MissingTurnLogs = append(write.MissingTurnLogs, log) + } +} + +// Guarded Q/None completion keeps the existing Logged, turn and stats contract. +func TestImpossibleStateMySQLQueueingUnreserved(t *testing.T) { + for _, kind := range []string{"raw", "rq1", "raw_zero_ids", "raw_missing_turns", "successful_turn"} { + t.Run(kind, func(t *testing.T) { + token := "" + if kind == "rq1" || kind == "successful_turn" { + token = "rq1" + } + f := newImpossibleStateFixture(t, token) + if kind == "raw_zero_ids" { + f.version = 0 + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_item_ref", "eval_target_record", "evaluator_record"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_version_id", 0).Error) + } + for _, table := range []string{"expt_turn_result", "expt_turn_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, f.item).Update("target_result_id", 0).Error) + } + } + if kind == "raw_missing_turns" { + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Delete(&model.ExptTurnResultRunLog{}).Error) + } + if kind == "successful_turn" { + for _, table := range []string{"expt_turn_result", "expt_turn_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=? AND turn_id=0", f.space, f.item).Update("status", int32(entity.TurnRunState_Success)).Error) + } + } + source := f.source() + require.Equal(t, token, source.Current.RequestID) + if kind == "raw_zero_ids" { + require.Zero(t, source.Current.ItemVersionID) + require.NotNil(t, source.Turns[0]) + require.Zero(t, source.Turns[0].TargetResultID) + } + write := impossibleStateWrite() + if kind == "raw_missing_turns" { + prepareImpossibleMissingTurns(t, f, source, write) + } + before := f.snapshot() + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, write) + require.NoError(t, err) + require.True(t, applied) + var item model.ExptItemResult + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&item).Error) + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&log).Error) + require.Equal(t, int32(entity.ItemRunState_Fail), item.Status) + require.Equal(t, int32(entity.ItemRunState_Fail), log.Status) + require.Equal(t, int32(entity.ExptItemResultStateLogged), gptr.Indirect(log.ResultState)) + require.Zero(t, log.QuotaReservationState) + require.Equal(t, int32(1), log.RetryTimes) + require.Equal(t, write.ErrMsg, gptr.Indirect(item.ErrMsg)) + require.Equal(t, write.ErrMsg, gptr.Indirect(log.ErrMsg)) + var turns []model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Find(&turns).Error) + require.Len(t, turns, 2) + for _, turn := range turns { + want := entity.TurnRunState_Fail + if kind == "successful_turn" && turn.TurnID == 0 { + want = entity.TurnRunState_Success + } + require.Equal(t, int32(want), turn.Status) + require.Equal(t, f.version, turn.ItemVersionID) + } + requireImpossibleStats(t, f, 1, 1, 1) + after := f.snapshot() + for _, table := range []string{"experiment", "expt_turn_result", "expt_turn_evaluator_result_ref", "expt_item_ref", "expt_run_log", "eval_target_record", "evaluator_record"} { + require.Equal(t, before[table], after[table], table) + } + applied, err = f.port.CompleteCurrentItemRun(context.Background(), source, write) + require.NoError(t, err) + require.False(t, applied) + require.Equal(t, after, f.snapshot(), "repeat failure must leave all eleven tables unchanged") + t.Logf("REAL_MYSQL_IMPOSSIBLE_QNONE kind=%s Logged=true stats_once=true snapshots=11", kind) + }) + } +} + +// Every part of Q/None is required even if the observed token and refs match. +func TestImpossibleStateMySQLFreshPredicateZeroWrites(t *testing.T) { + for _, kind := range []string{"canonical_processing", "log_processing", "reserved", "unknown_quota", "wrong_token", "wrong_version", "changed_target", "changed_ref", "canonical_success", "canonical_fail", "canonical_terminal", "log_success", "log_fail", "log_terminal", "raw_missing_claimed"} { + t.Run(kind, func(t *testing.T) { + token := "rq1" + if kind == "raw_missing_claimed" { + token = "" + } + f := newImpossibleStateFixture(t, token) + if kind == "raw_missing_claimed" { + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Delete(&model.ExptTurnResultRunLog{}).Error) + } + source := f.source() + write := impossibleStateWrite() + switch kind { + case "canonical_processing", "canonical_success", "canonical_fail", "canonical_terminal": + states := map[string]entity.ItemRunState{"canonical_processing": entity.ItemRunState_Processing, "canonical_success": entity.ItemRunState_Success, "canonical_fail": entity.ItemRunState_Fail, "canonical_terminal": entity.ItemRunState_Terminal} + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(states[kind])).Error) + case "log_processing", "log_success", "log_fail", "log_terminal": + states := map[string]entity.ItemRunState{"log_processing": entity.ItemRunState_Processing, "log_success": entity.ItemRunState_Success, "log_fail": entity.ItemRunState_Fail, "log_terminal": entity.ItemRunState_Terminal} + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(states[kind])).Error) + case "reserved", "unknown_quota": + quota := int32(entity.QuotaReservationStateReserved) + if kind == "unknown_quota" { + quota = 99 + } + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("quota_reservation_state", quota).Error) + case "wrong_token": + f.token("rq2") + case "wrong_version": + source.Current.ItemVersionID++ + case "changed_target": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=0", f.space).Update("target_result_id", f.base+42).Error) + case "changed_ref": + require.NoError(t, f.conn.Model(&model.ExptTurnEvaluatorResultRef{}).Where("space_id=?", f.space).Update("evaluator_result_id", f.base+62).Error) + case "raw_missing_claimed": + claimed, err := NewExptItemDispatchDAO(f.provider).ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{f.item}, map[int64]string{f.item: token}) + require.NoError(t, err) + require.Len(t, claimed, 1) + // Invalid preparation must not even be inspected after the predicate rejects. + write.MissingTurnLogs = []*entity.ExptTurnResultRunLog{nil} + } + before := f.snapshot() + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, write) + require.NoError(t, err) + require.False(t, applied) + require.Equal(t, before, f.snapshot(), "predicate rejection must make zero writes across all eleven tables") + t.Logf("REAL_MYSQL_IMPOSSIBLE_REJECT kind=%s snapshots=11", kind) + }) + } +} + +// The optional flag must not narrow ordinary Processing completion or retries. +func TestImpossibleStateMySQLDefaultFalseCompatibility(t *testing.T) { + for _, kind := range []string{"success", "fail", "yield", "retry_only", "retry_only_unchanged"} { + t.Run(kind, func(t *testing.T) { + f := newImpossibleStateFixture(t, "rq1") + source := f.source() + dispatch := NewExptItemDispatchDAO(f.provider) + claimed, err := dispatch.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{f.item}, map[int64]string{f.item: "rq1"}) + require.NoError(t, err) + require.Len(t, claimed, 1) + started, err := dispatch.StartReservedItem(context.Background(), f.space, f.expt, f.run, f.item, "rq1") + require.NoError(t, err) + require.True(t, started.Started) + write := &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Fail, RetryTimes: 2, FailPendingTurns: true} + wantStatus, wantResult, wantRetry := entity.ItemRunState_Fail, entity.ExptItemResultStateLogged, int32(2) + switch kind { + case "success": + write.Status, write.FailPendingTurns = entity.ItemRunState_Success, false + wantStatus = entity.ItemRunState_Success + case "yield": + write.Yield = true + wantStatus, wantResult = entity.ItemRunState_Queueing, entity.ExptItemResultStateDefault + case "retry_only", "retry_only_unchanged": + write.RetryOnly = true + wantStatus, wantResult = entity.ItemRunState_Processing, entity.ExptItemResultStateDefault + if kind == "retry_only_unchanged" { + write.RetryTimes, wantRetry = 1, 1 + } + } + before := f.snapshot() + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, write) + require.NoError(t, err) + require.True(t, applied) + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&log).Error) + require.Equal(t, int32(wantStatus), log.Status) + require.Equal(t, int32(wantResult), gptr.Indirect(log.ResultState)) + require.Equal(t, wantRetry, log.RetryTimes) + if kind == "retry_only_unchanged" { + require.Equal(t, before, f.snapshot()) + } + t.Logf("REAL_MYSQL_IMPOSSIBLE_DEFAULT_FALSE kind=%s same_token_started_source=true", kind) + }) + } +} + +func TestImpossibleStateMySQLRollback(t *testing.T) { + for _, table := range []string{"expt_turn_result_run_log_create", "expt_turn_result_run_log_create_second", "expt_item_result_run_log", "expt_turn_result_run_log", "expt_item_result", "expt_stats"} { + t.Run(table, func(t *testing.T) { + f := newImpossibleStateFixture(t, "") + createFault := table == "expt_turn_result_run_log_create" || table == "expt_turn_result_run_log_create_second" + if createFault { + require.NoError(t, f.conn.Unscoped().Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Delete(&model.ExptTurnResultRunLog{}).Error) + } + source := f.source() + write := impossibleStateWrite() + if createFault { + prepareImpossibleMissingTurns(t, f, source, write) + } + before := f.snapshot() + injected := errors.New("impossible completion write fault") + hits := 0 + hook := "impossible_fault_" + table + callback := func(tx *gorm.DB) { + if tx.Statement.Table == table || createFault && tx.Statement.Table == "expt_turn_result_run_log" { + hits++ + _, transactional := tx.Statement.ConnPool.(*sql.Tx) + require.True(t, transactional, "fault must hit the real SQL transaction") + if table != "expt_turn_result_run_log_create_second" || hits == 2 { + _ = tx.AddError(injected) + } + } + } + if createFault { + require.NoError(t, f.conn.Callback().Create().Before("gorm:create").Register(hook, callback)) + } else { + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(hook, callback)) + } + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, write) + if createFault { + require.NoError(t, f.conn.Callback().Create().Remove(hook)) + } else { + require.NoError(t, f.conn.Callback().Update().Remove(hook)) + } + require.Positive(t, hits) + require.ErrorIs(t, err, injected) + require.False(t, applied) + require.Equal(t, before, f.snapshot()) + t.Logf("REAL_MYSQL_IMPOSSIBLE_ROLLBACK point=%s snapshots=11", table) + }) + } +} + +// Hold the canonical lock in the winning operation's actual transaction, then +// run Claim/Start and guarded Fail in two goroutines. Both serializations matter. +func TestImpossibleStateMySQLCanonicalLockRace(t *testing.T) { + for _, token := range []string{"", "rq1"} { + for _, first := range []string{"fail", "claim"} { + t.Run(token+"/"+first, func(t *testing.T) { + f := newImpossibleStateFixture(t, token) + source := f.source() + before := f.snapshot() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ready, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + hook := "impossible_lock_" + first + var once sync.Once + require.NoError(t, f.conn.Callback().Query().After("gorm:query").Register(hook, func(tx *gorm.DB) { + if tx.Statement.Table != "expt_item_result" || tx.Error != nil { + return + } + if _, locked := tx.Statement.Clauses["FOR"]; !locked { + return + } + _, transactional := tx.Statement.ConnPool.(*sql.Tx) + if !transactional { + return + } + once.Do(func() { + close(ready) + select { + case <-release: + case <-ctx.Done(): + _ = tx.AddError(ctx.Err()) + } + }) + })) + defer func() { require.NoError(t, f.conn.Callback().Query().Remove(hook)) }() + type failResult struct { + applied bool + err error + } + type claimResult struct { + claimed []repo.ClaimedDispatchItem + started repo.StartReservedItemResult + err error + } + failed, claimed := make(chan failResult, 1), make(chan claimResult, 1) + fail := func() { + applied, err := f.port.CompleteCurrentItemRun(ctx, source, impossibleStateWrite()) + failed <- failResult{applied, err} + } + claim := func() { + dispatch := NewExptItemDispatchDAO(f.provider) + result := claimResult{} + result.claimed, result.err = dispatch.ClaimQuotaReserved(ctx, f.space, f.expt, f.run, []int64{f.item}, map[int64]string{f.item: token}) + if result.err == nil { + result.started, result.err = dispatch.StartReservedItem(ctx, f.space, f.expt, f.run, f.item, token) + } + claimed <- result + } + if first == "fail" { + go fail() + } else { + go claim() + } + select { + case <-ready: + case <-ctx.Done(): + t.Fatal("winning operation did not lock the canonical MySQL row") + } + // A real NOWAIT read proves the winner currently holds the row. + lockedErr := f.provider.Transaction(ctx, func(tx *gorm.DB) error { + var item model.ExptItemResult + return tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "NOWAIT"}).Where("space_id=? AND item_id=?", f.space, f.item).Take(&item).Error + }, db.WithMaster()) + require.Error(t, lockedErr, "canonical row lock must be held by the first goroutine") + var mysqlErr *drivermysql.MySQLError + require.ErrorAs(t, lockedErr, &mysqlErr) + require.Equal(t, uint16(3572), mysqlErr.Number, "NOWAIT must report the real held row lock") + if first == "fail" { + go claim() + } else { + go fail() + } + releaseOnce.Do(func() { close(release) }) + var failure failResult + var takeover claimResult + select { + case failure = <-failed: + case <-ctx.Done(): + t.Fatal("guarded failure did not finish") + } + select { + case takeover = <-claimed: + case <-ctx.Done(): + t.Fatal("Claim/Start did not finish") + } + require.NoError(t, failure.err) + require.NoError(t, takeover.err) + if first == "fail" { + require.True(t, failure.applied) + require.Empty(t, takeover.claimed) + require.False(t, takeover.started.CurrentRequest) + require.False(t, takeover.started.Started) + requireImpossibleStats(t, f, 1, 1, 1) + } else { + require.False(t, failure.applied) + require.Len(t, takeover.claimed, 1) + require.True(t, takeover.started.CurrentRequest) + require.True(t, takeover.started.Started) + requireImpossibleStats(t, f, 1, 2, 0) + } + after := f.snapshot() + for _, table := range []string{"experiment", "expt_turn_evaluator_result_ref", "expt_item_ref", "expt_run_log", "eval_target_record", "evaluator_record"} { + require.Equal(t, before[table], after[table], table) + } + if first == "fail" { + require.Equal(t, before["expt_turn_result"], after["expt_turn_result"]) + } + var item model.ExptItemResult + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&item).Error) + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&log).Error) + wantStatus, wantResult := entity.ItemRunState_Processing, entity.ExptItemResultStateDefault + if first == "fail" { + wantStatus, wantResult = entity.ItemRunState_Fail, entity.ExptItemResultStateLogged + } + require.Equal(t, int32(wantStatus), item.Status) + require.Equal(t, int32(wantStatus), log.Status) + require.Equal(t, int32(wantResult), gptr.Indirect(log.ResultState)) + require.Zero(t, log.QuotaReservationState) + t.Logf("REAL_MYSQL_IMPOSSIBLE_LOCK_RACE first=%s token=%q canonical_NOWAIT_blocked=true snapshots=11 stats_once=true", first, token) + }) + } + } +} + +// A same-token Claim/Start must supersede an earlier impossible observation. +// Removing the transaction's fresh Q/None predicate must make this test fail. +func TestImpossibleStateMySQLSameTokenTakeover(t *testing.T) { + for _, token := range []string{"", "rq1"} { + for _, start := range []bool{false, true} { + name := "claimed" + if start { + name = "started" + } + t.Run(token+"/"+name, func(t *testing.T) { + f := newImpossibleStateFixture(t, token) + source := f.source() + require.Equal(t, entity.ItemRunState_Queueing, source.Current.Status) + dispatch := NewExptItemDispatchDAO(f.provider) + claimed, err := dispatch.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{f.item}, map[int64]string{f.item: token}) + require.NoError(t, err) + require.Len(t, claimed, 1) + require.Equal(t, token, claimed[0].RequestID) + if start { + started, err := dispatch.StartReservedItem(context.Background(), f.space, f.expt, f.run, f.item, token) + require.NoError(t, err) + require.True(t, started.CurrentRequest) + require.True(t, started.Started) + } + before := f.snapshot() + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{OnlyIfQueueingUnreserved: true, Status: entity.ItemRunState_Fail, ErrMsg: []byte("impossible quota"), RetryTimes: 1, FailPendingTurns: true}) + require.NoError(t, err) + assert.False(t, applied, "same-token execution took over the earlier Q/None observation") + after := f.snapshot() + for table, snapshot := range before { + assert.True(t, snapshot == after[table], "table %s changed after same-token takeover", table) + } + }) + } + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request.go new file mode 100644 index 0000000000..dd6bf967f3 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request.go @@ -0,0 +1,340 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/bytedance/gg/gptr" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +type currentRequestRow struct { + ID, ItemID, ItemVersionID, CanonicalID, CanonicalRunID, CanonicalItemVersionID int64 + Status, CanonicalStatus, RetryTimes int32 + ResultState *int32 + QuotaReservationState int32 + Ext []byte + Mode *int32 + RunStatus *int64 + UpdatedAt time.Time +} + +func (dao *exptItemResultDAOImpl) readCurrentRequests(ctx context.Context, spaceID, exptID, runID int64, itemIDs []int64, retryFirst bool, opts ...db.Option) ([]*entity.ExptItemCurrentRequest, error) { + session := dao.provider.NewSession(ctx, append(opts, db.WithMaster())...).Table("expt_item_result_run_log AS l"). + Select("l.id,l.item_id,l.item_version_id,l.status,l.result_state,l.retry_times,l.quota_reservation_state,l.updated_at,c.id AS canonical_id,c.expt_run_id AS canonical_run_id,c.status AS canonical_status,c.item_version_id AS canonical_item_version_id,c.ext,r.mode,r.status AS run_status"). + Joins("LEFT JOIN expt_item_result AS c ON c.space_id=l.space_id AND c.expt_id=l.expt_id AND c.item_id=l.item_id AND c.deleted_at IS NULL"). + Joins("LEFT JOIN expt_run_log AS r ON r.space_id=l.space_id AND r.expt_id=l.expt_id AND r.expt_run_id=l.expt_run_id AND r.deleted_at IS NULL"). + Where("l.space_id=? AND l.expt_id=? AND l.expt_run_id=? AND l.deleted_at IS NULL", spaceID, exptID, runID) + if itemIDs != nil { + session = session.Where("l.item_id IN ?", itemIDs) + } + if retryFirst { + session = session.Order("l.retry_times ASC") + } + var rows []currentRequestRow + if err := session.Order("l.id ASC").Find(&rows).Error; err != nil { + return nil, err + } + out := make([]*entity.ExptItemCurrentRequest, 0, len(rows)) + for _, row := range rows { + if row.CanonicalID == 0 || row.CanonicalRunID != runID { + continue + } + ext, err := entity.ParseExptItemResultExt(row.Ext) + if err != nil { + return nil, err + } + manual, err := entity.ParseManualRetryContext(ext) + if err != nil { + return nil, err + } + status := entity.ItemRunState(row.Status) + if !validCurrentItemStatus(status) || !validCurrentItemStatus(entity.ItemRunState(row.CanonicalStatus)) || row.RetryTimes < 0 { + return nil, fmt.Errorf("invalid current item state") + } + if row.ItemVersionID != row.CanonicalItemVersionID { + return nil, fmt.Errorf("canonical/run-log item version mismatch") + } + if row.RunStatus == nil { + return nil, fmt.Errorf("source run missing") + } + runStatus := entity.ExptStatus(*row.RunStatus) + if entity.IsExptFinished(runStatus) || entity.IsExptFinishing(runStatus) { + continue + } + if runStatus != entity.ExptStatus_Processing && runStatus != entity.ExptStatus_Pending { + return nil, fmt.Errorf("unknown source run state") + } + mode := entity.ExptRunMode(gptr.Indirect(row.Mode)) + if mode < entity.EvaluationModeSubmit || mode > entity.EvaluationModeTrialRun { + return nil, fmt.Errorf("invalid source run mode") + } + request := "" + if manual != nil { + if manual.RunID != runID { + return nil, fmt.Errorf("manual retry run mismatch") + } + mode = manual.Mode + request = manual.RequestID + } + if mode < entity.EvaluationModeSubmit || mode > entity.EvaluationModeTrialRun { + return nil, fmt.Errorf("invalid current run mode") + } + result := gptr.Indirect(row.ResultState) + if result < 0 || result > int32(entity.ExptItemResultStateLogged) { + return nil, fmt.Errorf("invalid item result state") + } + // The run log leads completion/yield/start; the canonical projection is synchronized later. + // A finished canonical item remains absorbing even if a late run-log write disagrees. + canonicalStatus := entity.ItemRunState(row.CanonicalStatus) + if entity.IsItemRunFinished(canonicalStatus) && canonicalStatus != status { + status = canonicalStatus + result = int32(entity.ExptItemResultStateDefault) + } + + out = append(out, &entity.ExptItemCurrentRequest{SpaceID: spaceID, ExptID: exptID, RunID: runID, ItemID: row.ItemID, ItemVersionID: row.ItemVersionID, RequestID: request, Mode: mode, Status: status, ResultState: entity.ExptItemResultState(result), RetryTimes: row.RetryTimes, UpdatedAt: gptr.Of(row.UpdatedAt), QuotaReservationState: entity.QuotaReservationState(row.QuotaReservationState)}) + } + return out, nil +} + +func validCurrentItemStatus(status entity.ItemRunState) bool { + return status == entity.ItemRunState_Queueing || status == entity.ItemRunState_Processing || entity.IsItemRunFinished(status) +} + +func (dao *exptItemResultDAOImpl) ScanCurrentRequests(ctx context.Context, spaceID, exptID, runID int64, retryFirst bool) ([]*entity.ExptItemCurrentRequest, error) { + var rows []*entity.ExptItemCurrentRequest + // NDB accepts ordinary Begin, without per-transaction isolation/readOnly SQL. + // Complete raw source verification also makes default RC reads fail closed. + err := dao.provider.NewSession(ctx, db.WithMaster()).Transaction(func(tx *gorm.DB) error { + before, err := collectItemReadObservation(tx, spaceID, exptID, runID, nil) + if err != nil { + return err + } + if len(before.Experiment) != 1 || before.Experiment[0].LatestRunID != runID { + return nil + } + rows, err = dao.readCurrentRequests(ctx, spaceID, exptID, runID, nil, retryFirst, db.WithTransaction(tx)) + if err != nil { + return err + } + for _, row := range rows { + if row.ResultState == entity.ExptItemResultStateLogged { + row.PublicationObserved = true + publication, err := readItemResultPublishSourceWithLocks(tx, spaceID, exptID, runID, row.ItemID, false) + if err != nil { + return err + } + if publication != nil && publication.Source.Current.RequestID == row.RequestID && publication.ItemLog.Status == int32(row.Status) && publication.ItemLog.ResultState == int32(row.ResultState) && publication.ItemLog.RetryTimes == row.RetryTimes && publication.ItemLog.UpdatedAt != nil && row.UpdatedAt != nil && publication.ItemLog.UpdatedAt.Equal(*row.UpdatedAt) { + row.Publication = publication + } + } + // Both source forms are inside the same verified observation window; + // mutation paths still recheck under their own row locks. + if row.Status != entity.ItemRunState_Processing { + continue + } + source, err := readItemRunSource(tx, spaceID, exptID, runID, row.ItemID, false, false) + if err != nil { + return err + } + if source == nil || !source.Current.Matches(runID, row.RequestID) || source.Current.ItemVersionID != row.ItemVersionID || source.Current.Status != row.Status || source.Current.RetryTimes != row.RetryTimes || source.Current.UpdatedAt == nil || row.UpdatedAt == nil || !source.Current.UpdatedAt.Equal(*row.UpdatedAt) { + continue + } + row.Source = source + } + after, err := collectItemReadObservation(tx, spaceID, exptID, runID, before) + if err != nil { + return err + } + if err := verifyItemReadObservation(before, after); err != nil { + return err + } + return ctx.Err() + }) + if err != nil { + return nil, err + } + return rows, err +} + +func (dao *exptItemResultDAOImpl) GetCurrentRequest(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptItemCurrentRequest, error) { + rows, err := dao.readCurrentRequests(ctx, spaceID, exptID, runID, []int64{itemID}, false) + if err != nil { + return nil, err + } + if len(rows) == 0 { + current, err := dao.BatchGet(ctx, spaceID, exptID, []int64{itemID}, db.WithMaster()) + if err != nil { + return nil, err + } + if len(current) > 0 && current[0].ExptRunID == runID && !entity.IsItemRunFinished(entity.ItemRunState(current[0].Status)) { + log, err := dao.GetItemRunLog(ctx, exptID, runID, itemID, spaceID, db.WithMaster()) + if errors.Is(err, gorm.ErrRecordNotFound) || log == nil && err == nil { + return nil, fmt.Errorf("current item source log missing") + } + if err != nil { + return nil, err + } + } + return nil, nil + } + return rows[0], nil +} + +func (dao *exptItemResultDAOImpl) MarkProcessingIfCurrent(ctx context.Context, spaceID, exptID, runID, itemID int64, requestID string) (bool, error) { + changed := false + err := dao.provider.Transaction(ctx, func(tx *gorm.DB) error { + var item model.ExptItemResult + lock := clause.Locking{Strength: "UPDATE"} + err := tx.Clauses(lock).Where("space_id=? AND expt_id=? AND item_id=?", spaceID, exptID, itemID).Take(&item).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + if err != nil { + return err + } + if item.ExptRunID != runID || entity.IsItemRunFinished(entity.ItemRunState(item.Status)) { + return nil + } + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + if err != nil { + return err + } + manual, err := entity.ParseManualRetryContext(ext) + if err != nil { + return err + } + currentRequest := "" + if manual != nil { + if manual.RunID != runID { + return fmt.Errorf("manual retry run mismatch") + } + currentRequest = manual.RequestID + } + if currentRequest != requestID { + return nil + } + var log model.ExptItemResultRunLog + err = tx.Clauses(lock).Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", spaceID, exptID, runID, itemID).Take(&log).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + if err != nil { + return err + } + if entity.IsItemRunFinished(entity.ItemRunState(log.Status)) { + return nil + } + if log.RetryTimes < 0 || gptr.Indirect(log.ResultState) < 0 || gptr.Indirect(log.ResultState) > int32(entity.ExptItemResultStateLogged) { + return fmt.Errorf("invalid item run counters or result state") + } + if !validCurrentItemStatus(entity.ItemRunState(item.Status)) || !validCurrentItemStatus(entity.ItemRunState(log.Status)) { + return fmt.Errorf("invalid current item state") + } + if item.Status == int32(entity.ItemRunState_Processing) && log.Status == item.Status { + return nil + } + if item.Status != int32(entity.ItemRunState_Queueing) || log.Status != item.Status { + return fmt.Errorf("canonical/run-log status mismatch") + } + var run model.ExptRunLog + if err := tx.Where("space_id=? AND expt_id=? AND expt_run_id=?", spaceID, exptID, runID).Take(&run).Error; err != nil { + return err + } + if mode := entity.ExptRunMode(gptr.Indirect(run.Mode)); mode < entity.EvaluationModeSubmit || mode > entity.EvaluationModeTrialRun { + return fmt.Errorf("invalid source run mode") + } + if entity.IsExptFinished(entity.ExptStatus(gptr.Indirect(run.Status))) || entity.IsExptFinishing(entity.ExptStatus(gptr.Indirect(run.Status))) { + return nil + } + var turns []model.ExptTurnResult + if err := tx.Clauses(lock).Where("space_id=? AND expt_id=? AND item_id=? AND expt_run_id=?", spaceID, exptID, itemID, runID).Order("id ASC").Find(&turns).Error; err != nil { + return err + } + var logs []model.ExptTurnResultRunLog + if err := tx.Clauses(lock).Where("space_id=? AND expt_id=? AND item_id=? AND expt_run_id=?", spaceID, exptID, itemID, runID).Order("id ASC").Find(&logs).Error; err != nil { + return err + } + byTurn := make(map[int64]model.ExptTurnResultRunLog, len(logs)) + for _, l := range logs { + if _, err := convert.NewExptTurnResultRunLogConvertor().PO2DO(&l); err != nil { + return err + } + if err := validateManualRetryTurnExt(l.Ext); err != nil { + return err + } + byTurn[l.TurnID] = l + } + pendingTurns, pendingLogs := []int64{}, []int64{} + if manual != nil && len(turns) == 0 { + return fmt.Errorf("manual frozen turns missing") + } + for _, turn := range turns { + l, exists := byTurn[turn.TurnID] + if manual != nil && !exists { + return fmt.Errorf("manual frozen turn log missing") + } + if turn.Status == int32(entity.TurnRunState_Success) { + if manual != nil && l.Status != int32(entity.TurnRunState_Success) { + return fmt.Errorf("frozen success turn mismatch") + } + continue + } + if manual != nil && (turn.Status != int32(entity.TurnRunState_Queueing) || l.Status != int32(entity.TurnRunState_Queueing)) { + return fmt.Errorf("invalid frozen pending turn") + } + if turn.Status == int32(entity.TurnRunState_Queueing) { + pendingTurns = append(pendingTurns, turn.ID) + } + if exists && l.Status == int32(entity.TurnRunState_Queueing) { + pendingLogs = append(pendingLogs, l.ID) + } + } + var stats model.ExptStats + if err := tx.Clauses(lock).Where("space_id=? AND expt_id=?", spaceID, exptID).Take(&stats).Error; err != nil { + return err + } + if stats.PendingCnt <= 0 { + return fmt.Errorf("queueing stats underflow") + } + for _, row := range []struct { + value any + id int64 + }{{&model.ExptItemResult{}, item.ID}, {&model.ExptItemResultRunLog{}, log.ID}} { + res := tx.Model(row.value).Where("id=? AND status=?", row.id, int32(entity.ItemRunState_Queueing)).Update("status", int32(entity.ItemRunState_Processing)) + if res.Error != nil { + return res.Error + } + if res.RowsAffected != 1 { + return fmt.Errorf("processing CAS lost") + } + } + if len(pendingTurns) > 0 { + if err := tx.Model(&model.ExptTurnResult{}).Where("id IN ?", pendingTurns).Update("status", int32(entity.TurnRunState_Processing)).Error; err != nil { + return err + } + } + if len(pendingLogs) > 0 { + if err := tx.Model(&model.ExptTurnResultRunLog{}).Where("id IN ?", pendingLogs).Update("status", int32(entity.TurnRunState_Processing)).Error; err != nil { + return err + } + } + if err := tx.Model(&model.ExptStats{}).Where("id=?", stats.ID).Updates(map[string]any{"pending_cnt": gorm.Expr("pending_cnt - 1"), "processing_cnt": gorm.Expr("processing_cnt + 1")}).Error; err != nil { + return err + } + changed = true + return nil + }, db.WithMaster()) + return changed && err == nil, err +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request_scan_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request_scan_test.go new file mode 100644 index 0000000000..18f9de587b --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request_scan_test.go @@ -0,0 +1,275 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "database/sql" + "errors" + "sync" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestCurrentRequestMySQLRollback(t *testing.T) { + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_stats"} { + t.Run(table, func(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + before := f.snapshot() + hits := 0 + name := "processing_fault_" + table + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(name, func(tx *gorm.DB) { + if tx.Statement.Table == table { + _, same := tx.Statement.ConnPool.(*sql.Tx) + require.True(t, same) + hits++ + _ = tx.AddError(errors.New("injected processing failure")) + } + })) + defer func() { require.NoError(t, f.conn.Callback().Update().Remove(name)) }() + changed, err := NewExptItemResultDAO(f.provider).MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.Error(t, err) + require.False(t, changed) + require.Positive(t, hits) + require.Equal(t, before, f.snapshot()) + t.Logf("REAL_MYSQL_PROCESSING_ROLLBACK table=%s snapshots=11", table) + }) + } +} + +func TestCurrentRequestMySQLScanMix(t *testing.T) { + f := newManualRetryFixture(t) + // Convert the failed item to queueing through the actual admission transaction. + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + ext, err := entity.MergeManualRetryContext(nil, &entity.ManualRetryContext{RunID: f.run, RequestID: "rq2", Mode: entity.EvaluationModeRetryItems}) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 13).Update("ext", ext).Error) + rows, err := NewExptItemResultDAO(f.provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + require.NoError(t, err) + require.Len(t, rows, 3) + got := map[int64]*entity.ExptItemCurrentRequest{} + for _, r := range rows { + got[r.ItemID] = r + } + require.Equal(t, "rq1", got[11].RequestID) + require.Equal(t, entity.EvaluationModeFailRetry, got[11].Mode) + require.Equal(t, int32(3), got[11].RetryTimes) + require.Empty(t, got[12].RequestID) + require.Equal(t, entity.EvaluationModeSubmit, got[12].Mode) + require.Equal(t, "rq2", got[13].RequestID) + require.Equal(t, entity.EvaluationModeRetryItems, got[13].Mode) + require.Equal(t, entity.ItemRunState_Processing, got[12].Status) + t.Log("REAL_MYSQL_CURRENT_SOURCE_MIX empty/rq1/rq2 modes and retry_times verified") +} + +func TestCurrentRequestMySQLZeroWrite(t *testing.T) { + for _, kind := range []string{"old_token", "empty_token", "old_run", "missing", "soft_item", "soft_log", "terminal", "null_context", "bad_json", "foreign_run", "missing_turn", "unknown_item", "unknown_log", "bad_turn", "missing_stats"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + run, token := f.run, "rq1" + wantErr := false + q := f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11) + switch kind { + case "old_token": + token = "rq0" + case "empty_token": + token = "" + case "old_run": + run++ + case "missing": + require.NoError(t, q.Unscoped().Delete(&model.ExptItemResult{}).Error) + case "soft_item": + require.NoError(t, q.Delete(&model.ExptItemResult{}).Error) + case "soft_log": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Delete(&model.ExptItemResultRunLog{}).Error) + case "terminal": + require.NoError(t, q.Update("status", int32(entity.ItemRunState_Success)).Error) + case "null_context": + require.NoError(t, q.Update("ext", []byte(`{"manual_retry":"null"}`)).Error) + wantErr = true + case "bad_json": + require.NoError(t, q.Update("ext", []byte(`{"manual_retry":`)).Error) + wantErr = true + case "foreign_run": + require.NoError(t, q.Update("expt_run_id", f.run+1).Error) + case "missing_turn": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=? AND turn_id=?", f.space, f.expt, 11, 21).Delete(&model.ExptTurnResultRunLog{}).Error) + wantErr = true + case "unknown_item": + require.NoError(t, q.Update("status", 9).Error) + wantErr = true + case "unknown_log": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("status", 9).Error) + wantErr = true + case "bad_turn": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND expt_id=? AND item_id=? AND turn_id=?", f.space, f.expt, 11, 21).Update("evaluator_result_ids", []byte(`{"registered":`)).Error) + wantErr = true + case "missing_stats": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Delete(&model.ExptStats{}).Error) + wantErr = true + } + before := f.snapshot() + changed, err := NewExptItemResultDAO(f.provider).MarkProcessingIfCurrent(context.Background(), f.space, f.expt, run, 11, token) + if wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.False(t, changed) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestCurrentRequestMySQLSuccessZeroScore(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + var before model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND turn_id=?", f.space, f.expt, 22).Take(&before).Error) + require.Equal(t, 0.0, gptr.Indirect(before.WeightedScore)) + _, err = NewExptItemResultDAO(f.provider).MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + var after model.ExptTurnResult + require.NoError(t, f.conn.Where("id=?", before.ID).Take(&after).Error) + require.Equal(t, before, after) +} + +func TestCurrentRequestMySQLConcurrentAndLateCAS(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + port := NewExptItemResultDAO(f.provider) + var wg sync.WaitGroup + changes := make(chan bool, 2) + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + changed, err := port.MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 11, "rq1") + changes <- changed + errs <- err + }() + } + wg.Wait() + close(changes) + close(errs) + count := 0 + for changed := range changes { + if changed { + count++ + } + } + for err := range errs { + require.NoError(t, err) + } + require.Equal(t, 1, count) + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Take(&stats).Error) + require.Equal(t, int32(1), stats.PendingCnt) + require.Equal(t, int32(2), stats.ProcessingCnt) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("status", int32(entity.ItemRunState_Success)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": int32(entity.ExptItemResultStateResulted)}).Error) + before := f.snapshot() + changed, err := port.MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + require.False(t, changed) + require.Equal(t, before, f.snapshot()) + t.Log("REAL_MYSQL_CONCURRENT_CAS stats moved once; late producer after terminal zero writes") +} + +func TestCurrentRequestMySQLOriginalEmptyToken(t *testing.T) { + f := newManualRetryFixture(t) + port := NewExptItemResultDAO(f.provider) + before := f.snapshot() + changed, err := port.MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 13, "rq1") + require.NoError(t, err) + require.False(t, changed) + require.Equal(t, before, f.snapshot()) + changed, err = port.MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 13, "") + require.NoError(t, err) + require.True(t, changed) + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 13).Take(&log).Error) + require.Equal(t, int32(3), log.RetryTimes) +} + +func TestCurrentRequestMySQLScanRejectsBadSource(t *testing.T) { + for _, kind := range []string{"bad_json", "null_context", "status", "mode", "run_deleted", "foreign_manual_run"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + q := f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 13) + switch kind { + case "bad_json": + require.NoError(t, q.Update("ext", []byte(`{`)).Error) + case "null_context": + require.NoError(t, q.Update("ext", []byte(`{"manual_retry":"null"}`)).Error) + case "status": + require.NoError(t, q.Update("status", 9).Error) + case "mode": + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Update("mode", 0).Error) + case "run_deleted": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Delete(&model.ExptRunLog{}).Error) + case "foreign_manual_run": + ext, err := entity.MergeManualRetryContext(nil, &entity.ManualRetryContext{RunID: f.run + 1, RequestID: "rq2", Mode: entity.EvaluationModeRetryItems}) + require.NoError(t, err) + require.NoError(t, q.Update("ext", ext).Error) + } + before := f.snapshot() + rows, err := NewExptItemResultDAO(f.provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + require.Error(t, err) + require.Nil(t, rows) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestCurrentRequestMySQLCASRejectsCorruptState(t *testing.T) { + for _, kind := range []string{"retry_times", "result_state", "success_turn_mismatch", "unknown_turn", "manual_run_missing"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + switch kind { + case "retry_times": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("retry_times", int64(2147483648)).Error) + case "result_state": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("result_state", 9).Error) + case "success_turn_mismatch": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND expt_id=? AND turn_id=?", f.space, f.expt, 22).Update("status", int32(entity.TurnRunState_Queueing)).Error) + case "unknown_turn": + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=? AND expt_id=? AND turn_id=?", f.space, f.expt, 21).Update("status", 9).Error) + case "manual_run_missing": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Delete(&model.ExptRunLog{}).Error) + } + before := f.snapshot() + changed, err := NewExptItemResultDAO(f.provider).MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.Error(t, err) + require.False(t, changed) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestCurrentRequestMySQLMissingLogIsStorageError(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Delete(&model.ExptItemResultRunLog{}).Error) + before := f.snapshot() + source, err := NewExptItemResultDAO(f.provider).GetCurrentRequest(context.Background(), f.space, f.expt, f.run, 11) + require.Error(t, err) + require.Nil(t, source) + require.Equal(t, before, f.snapshot()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request_test.go new file mode 100644 index 0000000000..fd01915d84 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_current_request_test.go @@ -0,0 +1,37 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "testing" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/stretchr/testify/require" +) + +type processingCASPort interface { + MarkProcessingIfCurrent(context.Context, int64, int64, int64, int64, string) (bool, error) +} + +func TestCurrentRequestMySQLProcessingCAS(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + dao := NewExptItemResultDAO(f.provider) + port, ok := any(dao).(processingCASPort) + require.True(t, ok, "current request CAS transaction is missing") + before := f.snapshot() + changed, err := port.MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + require.True(t, changed) + after := f.snapshot() + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_stats"} { + require.NotEqual(t, before[table], after[table], table) + } + changed, err = port.MarkProcessingIfCurrent(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + require.False(t, changed) + require.Equal(t, after, f.snapshot()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch.go index 63ee556206..84aeef3b54 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch.go @@ -5,141 +5,279 @@ package mysql import ( "context" + "errors" + "fmt" + + "github.com/bytedance/gg/gptr" + "gorm.io/gorm" + "gorm.io/gorm/clause" "github.com/coze-dev/coze-loop/backend/infra/db" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" - "github.com/coze-dev/coze-loop/backend/pkg/errorx" ) //go:generate mockgen -destination=mocks/expt_item_dispatch.go -package=mocks . IExptItemDispatchDAO -// IExptItemDispatchDAO run log 派发投影的数据访问。 type IExptItemDispatchDAO interface { - ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) - ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) + ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]repo.ClaimedDispatchItem, error) + ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]int64, error) LoadDispatchRuntime(ctx context.Context, spaceID, exptID, exptRunID int64, candidateLimit int) (*repo.ExptDispatchRuntime, error) - StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) - RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) + StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (repo.StartReservedItemResult, error) + RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (bool, error) MGetDispatchObservations(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]*repo.ExptDispatchObservation, error) } -func NewExptItemDispatchDAO(db db.Provider) IExptItemDispatchDAO { - return &exptItemDispatchDAOImpl{db: db} +func NewExptItemDispatchDAO(provider db.Provider) IExptItemDispatchDAO { + return &exptItemDispatchDAOImpl{db: provider} } -type exptItemDispatchDAOImpl struct { - db db.Provider +type exptItemDispatchDAOImpl struct{ db db.Provider } + +func dispatchRequest(ext []byte, runID int64) (string, *entity.ManualRetryContext, error) { + parsed, err := entity.ParseExptItemResultExt(ext) + if err != nil { + return "", nil, err + } + manual, err := entity.ParseManualRetryContext(parsed) + if err != nil { + return "", nil, err + } + if manual == nil { + return "", nil, nil + } + if manual.RunID != runID { + return "", nil, fmt.Errorf("dispatch manual retry run mismatch") + } + return manual.RequestID, manual, nil } -// claimBatchSize 单次 CAS 的 item 上限。 -// 不做无界 IN 查询：单拍授予数本身有预算上限，超大 IN 会让执行计划退化。 -const claimBatchSize = 500 +// The grant map is required even for original empty-token items. Validate the +// whole input first so a missing entry cannot silently become an empty wildcard. +func dispatchIDs(itemIDs []int64, expected map[int64]string) ([]int64, error) { + seen := make(map[int64]bool, len(itemIDs)) + ids := make([]int64, 0, len(itemIDs)) + for _, id := range itemIDs { + if _, ok := expected[id]; !ok { + return nil, fmt.Errorf("dispatch expected request ID missing for item %d", id) + } + if !seen[id] { + ids = append(ids, id) + seen[id] = true + } + } + return ids, nil +} -func (d *exptItemDispatchDAOImpl) ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) { - if len(itemIDs) == 0 { +type lockedDispatchItem struct { + item model.ExptItemResult + log model.ExptItemResultRunLog + manual *entity.ManualRetryContext + request string + mode entity.ExptRunMode +} + +func lockDispatchItem(tx *gorm.DB, spaceID, exptID, runID, itemID int64, expected string) (*lockedDispatchItem, error) { + s := &lockedDispatchItem{} + lock := clause.Locking{Strength: "UPDATE"} + err := tx.Clauses(lock).Where("space_id=? AND expt_id=? AND item_id=? AND deleted_at IS NULL", spaceID, exptID, itemID).Take(&s.item).Error + if errors.Is(err, gorm.ErrRecordNotFound) { return nil, nil } + if err != nil { + return nil, err + } + if s.item.ExptRunID != runID { + return nil, nil + } + s.request, s.manual, err = dispatchRequest(gptr.Indirect(s.item.Ext), runID) + if err != nil { + return nil, err + } + if s.request != expected { + return nil, nil + } + // Admission locks experiment/run before canonical. Never lock those rows + // after canonical here: current-source control reads follow the existing + // execution path; run-tick lifecycle serialization is owned by the caller. + var run model.ExptRunLog + if err = tx.Where("space_id=? AND expt_id=? AND expt_run_id=? AND deleted_at IS NULL", spaceID, exptID, runID).Take(&run).Error; err != nil { + return nil, err + } + if run.Status == nil { + return nil, fmt.Errorf("dispatch source run status missing") + } + status := entity.ExptStatus(*run.Status) + if entity.IsExptFinished(status) || entity.IsExptFinishing(status) { + return nil, nil + } + if status != entity.ExptStatus_Pending && status != entity.ExptStatus_Processing { + return nil, fmt.Errorf("dispatch invalid run status") + } + var expt model.Experiment + if err = tx.Where("space_id=? AND id=? AND deleted_at IS NULL", spaceID, exptID).Take(&expt).Error; err != nil { + return nil, err + } + if expt.LatestRunID != runID || entity.IsExptFinished(entity.ExptStatus(expt.Status)) || entity.IsExptFinishing(entity.ExptStatus(expt.Status)) { + return nil, nil + } + s.mode = entity.ExptRunMode(gptr.Indirect(run.Mode)) + if s.mode < entity.EvaluationModeSubmit || s.mode > entity.EvaluationModeTrialRun { + return nil, fmt.Errorf("dispatch invalid source run mode") + } + if s.manual != nil { + s.mode = s.manual.Mode + } + if err = tx.Clauses(lock).Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND deleted_at IS NULL", spaceID, exptID, runID, itemID).Take(&s.log).Error; err != nil { + return nil, err + } + if s.item.ItemVersionID != s.log.ItemVersionID || s.log.RetryTimes < 0 || !validCurrentItemStatus(entity.ItemRunState(s.item.Status)) || !validCurrentItemStatus(entity.ItemRunState(s.log.Status)) || gptr.Indirect(s.log.ResultState) < 0 || gptr.Indirect(s.log.ResultState) > int32(entity.ExptItemResultStateLogged) { + return nil, fmt.Errorf("dispatch damaged item binding/state") + } + if entity.IsItemRunFinished(entity.ItemRunState(s.item.Status)) || entity.IsItemRunFinished(entity.ItemRunState(s.log.Status)) { + return nil, nil + } + return s, nil +} - claimed := make([]int64, 0, len(itemIDs)) - for _, batch := range chunkInt64(itemIDs, claimBatchSize) { - // 逐个 CAS 而非一条批量 UPDATE：批量 UPDATE 只能拿到总 RowsAffected， - // 无法知道**哪些** item 成功。而调用方必须精确知道，才能只发布成功项、 - // 并释放失败项的 Redis reservation —— 发布了未 claim 成功的 item 会造成双驱动。 - for _, itemID := range batch { - res := d.db.NewSession(ctx).Model(&model.ExptItemResultRunLog{}). - Where("space_id = ? AND expt_id = ? AND expt_run_id = ? AND item_id = ?", - spaceID, exptID, exptRunID, itemID). - Where("status = ?", int32(entity.ItemRunState_Queueing)). - Where("quota_reservation_state = ?", int32(entity.QuotaReservationStateNone)). - Update("quota_reservation_state", int32(entity.QuotaReservationStateReserved)) - if res.Error != nil { - return claimed, errorx.Wrapf(res.Error, "claim quota reserved fail, expt_run_id: %v, item_id: %v", exptRunID, itemID) +func (d *exptItemDispatchDAOImpl) ClaimQuotaReserved(ctx context.Context, spaceID, exptID, runID int64, itemIDs []int64, expected map[int64]string) ([]repo.ClaimedDispatchItem, error) { + ids, err := dispatchIDs(itemIDs, expected) + if err != nil { + return nil, err + } + claimed := make([]repo.ClaimedDispatchItem, 0, len(ids)) + for _, id := range ids { + var payload *repo.ClaimedDispatchItem + err = d.db.Transaction(ctx, func(tx *gorm.DB) error { + s, err := lockDispatchItem(tx, spaceID, exptID, runID, id, expected[id]) + if err != nil || s == nil { + return err + } + if s.log.Status != int32(entity.ItemRunState_Queueing) || s.log.QuotaReservationState != int32(entity.QuotaReservationStateNone) { + return nil } - if res.RowsAffected > 0 { - claimed = append(claimed, itemID) + result := tx.Model(&model.ExptItemResultRunLog{}).Where("id=? AND status=? AND quota_reservation_state=?", s.log.ID, int32(entity.ItemRunState_Queueing), int32(entity.QuotaReservationStateNone)).Update("quota_reservation_state", int32(entity.QuotaReservationStateReserved)) + if result.Error != nil { + return result.Error } + if result.RowsAffected == 1 { + payload = &repo.ClaimedDispatchItem{ItemID: id, RequestID: s.request, Mode: s.mode, RetryTimes: s.log.RetryTimes} + } + return nil + }, db.WithMaster()) + if err != nil { + return claimed, err + } + if payload != nil { + claimed = append(claimed, *payload) } } return claimed, nil } -func (d *exptItemDispatchDAOImpl) ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) { - if len(itemIDs) == 0 { - return nil, nil +func (d *exptItemDispatchDAOImpl) ResetQuotaReserved(ctx context.Context, spaceID, exptID, runID int64, itemIDs []int64, expected map[int64]string) ([]int64, error) { + ids, err := dispatchIDs(itemIDs, expected) + if err != nil { + return nil, err } - - reset := make([]int64, 0, len(itemIDs)) - for _, batch := range chunkInt64(itemIDs, claimBatchSize) { - for _, itemID := range batch { - // 条件里带 status=Queueing：若 item 已被 consumer 推进到 Processing， - // 说明消息其实已投递成功，退回投影会让它被重复授予。 - res := d.db.NewSession(ctx).Model(&model.ExptItemResultRunLog{}). - Where("space_id = ? AND expt_id = ? AND expt_run_id = ? AND item_id = ?", - spaceID, exptID, exptRunID, itemID). - Where("status = ?", int32(entity.ItemRunState_Queueing)). - Where("quota_reservation_state = ?", int32(entity.QuotaReservationStateReserved)). - Update("quota_reservation_state", int32(entity.QuotaReservationStateNone)) - if res.Error != nil { - return reset, errorx.Wrapf(res.Error, "reset quota reserved fail, expt_run_id: %v, item_id: %v", exptRunID, itemID) + reset := make([]int64, 0, len(ids)) + for _, id := range ids { + changed := false + err = d.db.Transaction(ctx, func(tx *gorm.DB) error { + s, err := lockDispatchItem(tx, spaceID, exptID, runID, id, expected[id]) + if err != nil || s == nil { + return err } - if res.RowsAffected > 0 { - reset = append(reset, itemID) + if s.log.Status != int32(entity.ItemRunState_Queueing) || s.log.QuotaReservationState != int32(entity.QuotaReservationStateReserved) { + return nil } + result := tx.Model(&model.ExptItemResultRunLog{}).Where("id=? AND status=? AND quota_reservation_state=?", s.log.ID, int32(entity.ItemRunState_Queueing), int32(entity.QuotaReservationStateReserved)).Update("quota_reservation_state", int32(entity.QuotaReservationStateNone)) + changed = result.RowsAffected == 1 + return result.Error + }, db.WithMaster()) + if err != nil { + return reset, err + } + if changed { + reset = append(reset, id) } } return reset, nil } -func (d *exptItemDispatchDAOImpl) LoadDispatchRuntime(ctx context.Context, spaceID, exptID, exptRunID int64, candidateLimit int) (*repo.ExptDispatchRuntime, error) { - if candidateLimit <= 0 { - candidateLimit = defaultLimit +func (d *exptItemDispatchDAOImpl) dispatchObservations(ctx context.Context, spaceID, exptID, runID int64, itemIDs []int64, activeOnly bool) ([]*repo.ExptDispatchObservation, error) { + // Filter canonical soft deletion in ON, not WHERE: a missing/deleted right + // row must remain visible as an incomplete binding, never a fake empty run. + session := d.db.NewSession(ctx, db.WithMaster()).Table("expt_item_result_run_log AS l"). + Select("l.id,l.item_id,l.item_version_id,l.status,l.result_state,l.retry_times,l.quota_reservation_state,c.id AS canonical_id,c.expt_run_id AS canonical_run_id,c.status AS canonical_status,c.item_version_id AS canonical_item_version_id,c.ext,r.mode,r.status AS run_status"). + Joins("LEFT JOIN expt_item_result AS c ON c.space_id=l.space_id AND c.expt_id=l.expt_id AND c.item_id=l.item_id AND c.deleted_at IS NULL"). + Joins("LEFT JOIN expt_run_log AS r ON r.space_id=l.space_id AND r.expt_id=l.expt_id AND r.expt_run_id=l.expt_run_id AND r.deleted_at IS NULL"). + Where("l.space_id=? AND l.expt_id=? AND l.expt_run_id=? AND l.deleted_at IS NULL", spaceID, exptID, runID) + if itemIDs != nil { + session = session.Where("l.item_id IN ?", itemIDs) } - - // 一次查询取回两类记录，而非分两次查 Processing 和 Queueing： - // 分两次的话，两次查询之间 item 可能从 Queueing 变成 Processing， - // 于是它在两个结果里都不出现（第一次查时还是 Queueing，第二次查时已是 Processing 但 - // 第二次只查 Queueing），导致并发占用少算、超发。 - type row struct { - ItemID int64 `gorm:"column:item_id"` - Status int32 `gorm:"column:status"` - QuotaReservationState int32 `gorm:"column:quota_reservation_state"` - } - var rows []row - - err := d.db.NewSession(ctx).Model(&model.ExptItemResultRunLog{}). - Select("item_id, status, quota_reservation_state"). - Where("space_id = ? AND expt_id = ? AND expt_run_id = ?", spaceID, exptID, exptRunID). - Where("status IN ?", []int32{ - int32(entity.ItemRunState_Queueing), - int32(entity.ItemRunState_Processing), - }). - Order("id ASC"). - Find(&rows).Error - if err != nil { - return nil, errorx.Wrapf(err, "load dispatch runtime fail, expt_run_id: %v", exptRunID) + if activeOnly { + session = session.Where("l.status IN ?", []int32{int32(entity.ItemRunState_Queueing), int32(entity.ItemRunState_Processing)}) } - - observations := make([]*repo.ExptDispatchObservation, 0, len(rows)) + var rows []currentRequestRow + if err := session.Order("l.id ASC").Find(&rows).Error; err != nil { + return nil, err + } + out := make([]*repo.ExptDispatchObservation, 0, len(rows)) for _, r := range rows { - observations = append(observations, &repo.ExptDispatchObservation{ - ItemID: r.ItemID, - Status: r.Status, - QuotaReservationState: entity.QuotaReservationState(r.QuotaReservationState), - }) + if r.CanonicalID == 0 || r.CanonicalRunID != runID || r.CanonicalItemVersionID != r.ItemVersionID { + return nil, fmt.Errorf("dispatch incomplete canonical binding for item %d", r.ItemID) + } + request, _, err := dispatchRequest(r.Ext, runID) + if err != nil { + return nil, err + } + if r.RunStatus == nil || r.Mode == nil || entity.ExptRunMode(*r.Mode) < entity.EvaluationModeSubmit || entity.ExptRunMode(*r.Mode) > entity.EvaluationModeTrialRun { + return nil, fmt.Errorf("dispatch incomplete run binding") + } + if !validCurrentItemStatus(entity.ItemRunState(r.Status)) || !validCurrentItemStatus(entity.ItemRunState(r.CanonicalStatus)) || r.RetryTimes < 0 || gptr.Indirect(r.ResultState) < 0 || gptr.Indirect(r.ResultState) > int32(entity.ExptItemResultStateLogged) { + return nil, fmt.Errorf("dispatch damaged observation") + } + runStatus := entity.ExptStatus(*r.RunStatus) + if runStatus != entity.ExptStatus_Pending && runStatus != entity.ExptStatus_Processing && !entity.IsExptFinished(runStatus) && !entity.IsExptFinishing(runStatus) { + return nil, fmt.Errorf("dispatch invalid source run status") + } + status := r.Status + if entity.IsItemRunFinished(entity.ItemRunState(r.CanonicalStatus)) { + status = r.CanonicalStatus + } + out = append(out, &repo.ExptDispatchObservation{ItemID: r.ItemID, Status: status, QuotaReservationState: entity.QuotaReservationState(r.QuotaReservationState), RequestID: request}) } + if itemIDs != nil { + wanted := make(map[int64]bool, len(itemIDs)) + for _, id := range itemIDs { + wanted[id] = true + } + for _, row := range rows { + delete(wanted, row.ItemID) + } + if len(wanted) != 0 { + return nil, fmt.Errorf("dispatch item log observation incomplete") + } + } + return out, nil +} +func (d *exptItemDispatchDAOImpl) LoadDispatchRuntime(ctx context.Context, spaceID, exptID, runID int64, candidateLimit int) (*repo.ExptDispatchRuntime, error) { + if candidateLimit <= 0 { + candidateLimit = defaultLimit + } + observations, err := d.dispatchObservations(ctx, spaceID, exptID, runID, nil, true) + if err != nil { + return nil, err + } return classifyDispatchRuntime(observations, candidateLimit), nil } -// classifyDispatchRuntime 把 run log 观测按 (status, 预占态) 分成「占用并发」与「可授予候选」。 -// -// 抽成纯函数是为了让这段分类规则可直接单测 —— 它是本次改动的正确性核心： -// 漏把 Queueing/reserved 计入占用会导致 deficit 高估、同一批 item 被反复授予； -// 误把它算进候选则会重复派发。 func classifyDispatchRuntime(observations []*repo.ExptDispatchObservation, candidateLimit int) *repo.ExptDispatchRuntime { - runtime := &repo.ExptDispatchRuntime{} + runtime := &repo.ExptDispatchRuntime{ItemRequestIDs: make(map[int64]string)} for _, o := range observations { if o == nil { continue @@ -147,96 +285,199 @@ func classifyDispatchRuntime(observations []*repo.ExptDispatchObservation, candi switch { case o.Status == int32(entity.ItemRunState_Processing): runtime.OccupiedItemIDs = append(runtime.OccupiedItemIDs, o.ItemID) + case o.Status != int32(entity.ItemRunState_Queueing): + continue case o.QuotaReservationState.IsQuotaReserved(): - // Queueing + reserved：已预占未消费，计入占用且不得再次授予 runtime.OccupiedItemIDs = append(runtime.OccupiedItemIDs, o.ItemID) default: - // Queueing + none：唯一的可授予候选 - if candidateLimit <= 0 || len(runtime.CandidateItemIDs) < candidateLimit { - runtime.CandidateItemIDs = append(runtime.CandidateItemIDs, o.ItemID) + if candidateLimit > 0 && len(runtime.CandidateItemIDs) >= candidateLimit { + continue } + runtime.CandidateItemIDs = append(runtime.CandidateItemIDs, o.ItemID) } + runtime.ItemRequestIDs[o.ItemID] = o.RequestID } return runtime } -func (d *exptItemDispatchDAOImpl) StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) { - // Queueing/reserved → Processing/none：预占兑现为实际执行。 - // 同时清掉预占标记，因为 Processing 本身已经代表占用，留着 reserved 会让对账看到 - // 一个自相矛盾的状态（Processing 且 reserved）。 - res := d.db.NewSession(ctx).Model(&model.ExptItemResultRunLog{}). - Where("space_id = ? AND expt_id = ? AND expt_run_id = ? AND item_id = ?", - spaceID, exptID, exptRunID, itemID). - Where("status = ?", int32(entity.ItemRunState_Queueing)). - Where("quota_reservation_state = ?", int32(entity.QuotaReservationStateReserved)). - Updates(map[string]any{ - "status": int32(entity.ItemRunState_Processing), - "quota_reservation_state": int32(entity.QuotaReservationStateNone), - }) - if res.Error != nil { - return false, errorx.Wrapf(res.Error, "start reserved item fail, expt_run_id: %v, item_id: %v", exptRunID, itemID) - } - return res.RowsAffected > 0, nil -} - -func (d *exptItemDispatchDAOImpl) RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) { - // Processing/none → Queueing/none：孤儿 item 退回授予候选。 - // - // 条件里同时钉住 status=Processing 与 quota_reservation_state=none，是为了让这次退回 - // 只作用于「已兑现执行但额度已消失」这一种形状： - // - status 若已是终态，说明消息只是迟到，item 早就跑完了，不能退回（会重复执行）； - // - qrs 若是 reserved，说明它还没被 StartReservedItem 兑现，该走 ResetQuotaReserved。 - // ★ 用 UpdateColumn 而非 Update：**刻意不刷新 `updated_at`**。 - // - // 僵尸兜底的判据是 `Processing 且 time.Since(updated_at) > zombieSecond`。若这里刷新时间戳， - // 而"reservation 消失"是个持续性成因（账本损坏、reap 竞态反复触发），item 就会在 - // Queueing ↔ Processing 之间无限往返、每次都把僵尸时钟拨回零，实验永不收敛 —— - // 那是把「3 小时后必定收敛」的有界故障换成了无界故障。 - // 保留原始 updated_at 让僵尸时钟继续走：一次性成因下 item 正常重跑， - // 持续性成因下它仍会在原定的 3 小时被判 Fail，最坏情况不劣于修复前。 - res := d.db.NewSession(ctx).Model(&model.ExptItemResultRunLog{}). - Where("space_id = ? AND expt_id = ? AND expt_run_id = ? AND item_id = ?", - spaceID, exptID, exptRunID, itemID). - Where("status = ?", int32(entity.ItemRunState_Processing)). - Where("quota_reservation_state = ?", int32(entity.QuotaReservationStateNone)). - UpdateColumn("status", int32(entity.ItemRunState_Queueing)) - if res.Error != nil { - return false, errorx.Wrapf(res.Error, "requeue processing item fail, expt_run_id: %v, item_id: %v", exptRunID, itemID) - } - return res.RowsAffected > 0, nil -} - -func (d *exptItemDispatchDAOImpl) MGetDispatchObservations(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]*repo.ExptDispatchObservation, error) { - if len(itemIDs) == 0 { - return nil, nil +// Both item/log and pending canonical/log turns move before the one stats +// delta. Success turns and all invocation refs/scores remain untouched. +func transitionDispatchItem(tx *gorm.DB, s *lockedDispatchItem, next entity.ItemRunState) error { + var turns []model.ExptTurnResult + var logs []model.ExptTurnResultRunLog + lock := clause.Locking{Strength: "UPDATE"} + where := "space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND deleted_at IS NULL" + args := []any{s.item.SpaceID, s.item.ExptID, s.item.ExptRunID, s.item.ItemID} + if err := tx.Clauses(lock).Where(where, args...).Order("id ASC").Find(&turns).Error; err != nil { + return err + } + if err := tx.Clauses(lock).Where(where, args...).Order("id ASC").Find(&logs).Error; err != nil { + return err + } + byTurn := make(map[int64]model.ExptTurnResultRunLog, len(logs)) + for _, log := range logs { + if _, duplicate := byTurn[log.TurnID]; duplicate { + return fmt.Errorf("dispatch duplicate turn log") + } + if _, err := convert.NewExptTurnResultRunLogConvertor().PO2DO(&log); err != nil { + return err + } + if err := validateManualRetryTurnExt(log.Ext); err != nil { + return err + } + byTurn[log.TurnID] = log + } + if s.manual != nil && (len(turns) == 0 || len(turns) != len(logs)) { + return fmt.Errorf("dispatch frozen turns incomplete") + } + pendingTurns, pendingLogs := []int64{}, []int64{} + matchedLogs := 0 + fromTurn, toTurn := int32(entity.TurnRunState_Queueing), int32(entity.TurnRunState_Processing) + if next == entity.ItemRunState_Queueing { + fromTurn, toTurn = toTurn, fromTurn + } + for _, turn := range turns { + log, exists := byTurn[turn.TurnID] + if exists { + matchedLogs++ + } + if turn.ItemVersionID != s.item.ItemVersionID || turn.TurnID < 0 || exists && log.ItemVersionID != turn.ItemVersionID { + return fmt.Errorf("dispatch turn binding mismatch") + } + if s.manual != nil && !exists { + return fmt.Errorf("dispatch frozen turn log missing") + } + if turn.Status == int32(entity.TurnRunState_Success) { + if exists && log.Status != turn.Status { + return fmt.Errorf("dispatch success turn mismatch") + } + continue + } + if turn.Status < 0 || turn.Status > int32(entity.TurnRunState_Terminal) || exists && (log.Status < 0 || log.Status > int32(entity.TurnRunState_Terminal)) { + return fmt.Errorf("dispatch invalid turn state") + } + if s.manual != nil && (turn.Status != fromTurn && turn.Status != toTurn || log.Status != fromTurn && log.Status != toTurn) { + return fmt.Errorf("dispatch invalid pending turn state") + } + if turn.Status == fromTurn { + pendingTurns = append(pendingTurns, turn.ID) + } + if exists && log.Status == fromTurn { + pendingLogs = append(pendingLogs, log.ID) + } + } + if matchedLogs != len(logs) { + return fmt.Errorf("dispatch orphaned turn log") + } + var stats model.ExptStats + if err := tx.Clauses(lock).Where("space_id=? AND expt_id=? AND deleted_at IS NULL", s.item.SpaceID, s.item.ExptID).Take(&stats).Error; err != nil { + return err } + old := entity.ItemRunState(s.item.Status) + if old != next && (old == entity.ItemRunState_Queueing && stats.PendingCnt <= 0 || old == entity.ItemRunState_Processing && stats.ProcessingCnt <= 0) { + return fmt.Errorf("dispatch stats underflow") + } + // UpdateColumns skips GORM timestamp hooks; explicit original timestamps + // also prevent MySQL ON UPDATE from restarting the zombie clock on requeue. + update := func(value any, ids []int64, fields map[string]any) error { + if len(ids) == 0 { + return nil + } + if next == entity.ItemRunState_Queueing { + fields["updated_at"] = gorm.Expr("updated_at") + } + query := tx.Model(value).Where("id IN ?", ids) + var result *gorm.DB + if next == entity.ItemRunState_Queueing { + result = query.UpdateColumns(fields) + } else { + result = query.Updates(fields) + } + if result.Error != nil { + return result.Error + } + if result.RowsAffected != int64(len(ids)) { + return fmt.Errorf("dispatch transition write lost") + } + return nil + } + if old != next { + if err := update(&model.ExptItemResult{}, []int64{s.item.ID}, map[string]any{"status": int32(next)}); err != nil { + return err + } + } + if err := update(&model.ExptItemResultRunLog{}, []int64{s.log.ID}, map[string]any{"status": int32(next), "quota_reservation_state": int32(entity.QuotaReservationStateNone)}); err != nil { + return err + } + if err := update(&model.ExptTurnResult{}, pendingTurns, map[string]any{"status": toTurn}); err != nil { + return err + } + if err := update(&model.ExptTurnResultRunLog{}, pendingLogs, map[string]any{"status": toTurn}); err != nil { + return err + } + if old != next { + delta := -1 + if next == entity.ItemRunState_Queueing { + delta = 1 + } + result := tx.Model(&model.ExptStats{}).Where("id=?", stats.ID).Updates(map[string]any{"pending_cnt": gorm.Expr("pending_cnt + ?", delta), "processing_cnt": gorm.Expr("processing_cnt - ?", delta)}) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("dispatch stats update lost") + } + } + return nil +} - type row struct { - ItemID int64 `gorm:"column:item_id"` - Status int32 `gorm:"column:status"` - QuotaReservationState int32 `gorm:"column:quota_reservation_state"` +func (d *exptItemDispatchDAOImpl) StartReservedItem(ctx context.Context, spaceID, exptID, runID, itemID int64, expected string) (repo.StartReservedItemResult, error) { + result := repo.StartReservedItemResult{} + err := d.db.Transaction(ctx, func(tx *gorm.DB) error { + s, err := lockDispatchItem(tx, spaceID, exptID, runID, itemID, expected) + if err != nil || s == nil { + return err + } + result.CurrentRequest = true + if s.log.Status != int32(entity.ItemRunState_Queueing) || s.log.QuotaReservationState != int32(entity.QuotaReservationStateReserved) { + return nil + } + if err = transitionDispatchItem(tx, s, entity.ItemRunState_Processing); err != nil { + return err + } + result.Started = true + return nil + }, db.WithMaster()) + if err != nil { + return repo.StartReservedItemResult{}, err } + return result, nil +} - out := make([]*repo.ExptDispatchObservation, 0, len(itemIDs)) - for _, batch := range chunkInt64(itemIDs, claimBatchSize) { - var rows []row - err := d.db.NewSession(ctx).Model(&model.ExptItemResultRunLog{}). - Select("item_id, status, quota_reservation_state"). - Where("space_id = ? AND expt_id = ? AND expt_run_id = ?", spaceID, exptID, exptRunID). - Where("item_id IN ?", batch). - Find(&rows).Error - if err != nil { - return nil, errorx.Wrapf(err, "mget dispatch observations fail, expt_run_id: %v", exptRunID) +func (d *exptItemDispatchDAOImpl) RequeueProcessingItem(ctx context.Context, spaceID, exptID, runID, itemID int64, expected string) (bool, error) { + changed := false + err := d.db.Transaction(ctx, func(tx *gorm.DB) error { + s, err := lockDispatchItem(tx, spaceID, exptID, runID, itemID, expected) + if err != nil || s == nil { + return err + } + if s.log.Status != int32(entity.ItemRunState_Processing) || s.log.QuotaReservationState != int32(entity.QuotaReservationStateNone) { + return nil } - for _, r := range rows { - out = append(out, &repo.ExptDispatchObservation{ - ItemID: r.ItemID, - Status: r.Status, - QuotaReservationState: entity.QuotaReservationState(r.QuotaReservationState), - }) + if err = transitionDispatchItem(tx, s, entity.ItemRunState_Queueing); err != nil { + return err } + changed = true + return nil + }, db.WithMaster()) + return changed && err == nil, err +} + +func (d *exptItemDispatchDAOImpl) MGetDispatchObservations(ctx context.Context, spaceID, exptID, runID int64, itemIDs []int64) ([]*repo.ExptDispatchObservation, error) { + if len(itemIDs) == 0 { + return nil, nil } - return out, nil + return d.dispatchObservations(ctx, spaceID, exptID, runID, itemIDs, false) } func chunkInt64(in []int64, size int) [][]int64 { diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch_test.go index 26a648b744..a69e4565e1 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_dispatch_test.go @@ -4,14 +4,101 @@ package mysql import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" "testing" + "time" + "github.com/DATA-DOG/go-sqlmock" + "github.com/coze-dev/coze-loop/backend/infra/db" + dbmock "github.com/coze-dev/coze-loop/backend/infra/db/mocks" + "go.uber.org/mock/gomock" + "gorm.io/gorm" + "gorm.io/plugin/dbresolver" + + "github.com/bytedance/gg/gptr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" ) +func TestDispatchMySQLRejectsDamagedObservation(t *testing.T) { + for _, damage := range []string{"missing_canonical", "soft_canonical", "bad_run", "bad_outer_json", "bad_inner_json", "missing_run", "unknown_run_status"} { + t.Run(damage, func(t *testing.T) { + f := newManualRetryFixture(t) + switch damage { + case "missing_canonical": + require.NoError(t, f.conn.Exec("DELETE FROM expt_item_result WHERE space_id=? AND expt_id=? AND item_id=13", f.space, f.expt).Error) + case "soft_canonical": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=13", f.space, f.expt).Delete(&model.ExptItemResult{}).Error) + case "bad_run": + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET expt_run_id=? WHERE space_id=? AND expt_id=? AND item_id=13", f.run+1, f.space, f.expt).Error) + case "bad_outer_json", "bad_inner_json": + raw := []byte(`{`) + if damage == "bad_inner_json" { + raw = []byte(`{"manual_retry":"null"}`) + } + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET ext=? WHERE space_id=? AND expt_id=? AND item_id=13", raw, f.space, f.expt).Error) + case "unknown_run_status": + require.NoError(t, f.conn.Exec("UPDATE expt_run_log SET status=99 WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + case "missing_run": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Delete(&model.ExptRunLog{}).Error) + } + before := f.snapshot() + dao := NewExptItemDispatchDAO(f.provider) + runtime, err := dao.LoadDispatchRuntime(context.Background(), f.space, f.expt, f.run, 1) + require.Error(t, err, "incomplete primary snapshot must not become usable runtime") + require.Nil(t, runtime) + observations, err := dao.MGetDispatchObservations(context.Background(), f.space, f.expt, f.run, []int64{13}) + require.Error(t, err) + require.Nil(t, observations) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestDispatchMySQLStartAtomic(t *testing.T) { + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Update("quota_reservation_state", 1).Error) + before := f.snapshot() + started, err := NewExptItemDispatchDAO(f.provider).StartReservedItem(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + require.True(t, started.Started) + var item model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Take(&item).Error) + require.Equal(t, int32(entity.ItemRunState_Processing), item.Status, "Start must atomically advance canonical") + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Take(&stats).Error) + require.Equal(t, int32(1), stats.PendingCnt) + require.Equal(t, int32(2), stats.ProcessingCnt) + var success model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND turn_id=22", f.space, f.expt).Take(&success).Error) + require.Equal(t, int32(entity.TurnRunState_Success), success.Status) + require.Equal(t, 0.0, gptr.Indirect(success.WeightedScore)) + require.Equal(t, before["expt_turn_evaluator_result_ref"], f.snapshot()["expt_turn_evaluator_result_ref"]) +} + +func TestDispatchMySQLCanonicalTerminalAbsorbing(t *testing.T) { + f := newManualRetryFixture(t) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET status=3 WHERE space_id=? AND expt_id=? AND item_id=13", f.space, f.expt).Error) + before := f.snapshot() + claimed, err := NewExptItemDispatchDAO(f.provider).ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{13}, map[int64]string{13: ""}) + require.NoError(t, err) + require.Empty(t, claimed, "canonical terminal must absorb stale queueing log") + require.Equal(t, before, f.snapshot()) +} + func obs(itemID int64, status entity.ItemRunState, quota entity.QuotaReservationState) *repo.ExptDispatchObservation { return &repo.ExptDispatchObservation{ ItemID: itemID, @@ -171,3 +258,548 @@ func TestQuotaReservationState_IsQuotaReserved(t *testing.T) { // 未知值按未预占处理：安全侧是让它进候选被重新预占，而非当成已占用而永不派发 assert.False(t, entity.QuotaReservationState(99).IsQuotaReserved()) } + +func callDispatchForSource(t *testing.T, dao IExptItemDispatchDAO, name string, f *manualRetryFixture, item int64, token string) (any, error) { + t.Helper() + switch name { + case "ClaimQuotaReserved": + return dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{item}, map[int64]string{item: token}) + case "ResetQuotaReserved": + return dao.ResetQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{item}, map[int64]string{item: token}) + case "StartReservedItem": + return dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, item, token) + case "RequeueProcessingItem": + return dao.RequeueProcessingItem(context.Background(), f.space, f.expt, f.run, item, token) + default: + t.Fatalf("unknown dispatch method %s", name) + return nil, nil + } +} + +func TestDispatchMySQLOldSourceZeroWrites(t *testing.T) { + for _, action := range []string{"ClaimQuotaReserved", "ResetQuotaReserved", "StartReservedItem", "RequeueProcessingItem"} { + for _, token := range []string{"", "rq1"} { + t.Run(action+"/old_"+token, func(t *testing.T) { + f := newManualRetryFixture(t) + req := f.request(entity.EvaluationModeFailRetry) + req.RequestID = "rq2" + req.PayloadHash = "hash2" + _, err := f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + state, quota := 0, 0 + if action == "ResetQuotaReserved" || action == "StartReservedItem" { + quota = 1 + } + if action == "RequeueProcessingItem" { + state = 1 + } + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET status=? WHERE space_id=? AND expt_id=? AND item_id=11", state, f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET status=?,quota_reservation_state=? WHERE space_id=? AND expt_id=? AND item_id=11", state, quota, f.space, f.expt).Error) + before := f.snapshot() + _, err = callDispatchForSource(t, NewExptItemDispatchDAO(f.provider), action, f, 11, token) + require.NoError(t, err) + require.Equal(t, before, f.snapshot(), "old token may not alter any status, count, reference or timestamp") + }) + } + } +} + +func prepareDispatchMySQL(t *testing.T) (*manualRetryFixture, IExptItemDispatchDAO) { + t.Helper() + f := newManualRetryFixture(t) + _, err := f.port.RequeueTerminalItems(context.Background(), f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + return f, NewExptItemDispatchDAO(f.provider) +} + +func TestDispatchMySQLMixedPayloadAndSingleJoin(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + high := int64(9223372036854775806) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET item_id=? WHERE space_id=? AND expt_id=? AND item_id=13", high, f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET item_id=? WHERE space_id=? AND expt_id=? AND item_id=13", high, f.space, f.expt).Error) + raw, err := entity.MergeManualRetryContext(nil, &entity.ManualRetryContext{RunID: f.run, RequestID: "rq2", Mode: entity.EvaluationModeRetryItems}) + require.NoError(t, err) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET ext=? WHERE space_id=? AND expt_id=? AND item_id=?", raw, f.space, f.expt, high).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_run_log SET mode=? WHERE space_id=? AND expt_id=?", entity.EvaluationModeAppend, f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET retry_times=7 WHERE space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, high).Error) + queries := []string{} + name := "dispatch_single_join" + require.NoError(t, f.conn.Callback().Query().After("gorm:query").Register(name, func(tx *gorm.DB) { queries = append(queries, tx.Statement.SQL.String()) })) + defer func() { require.NoError(t, f.conn.Callback().Query().Remove(name)) }() + runtime, err := dao.LoadDispatchRuntime(context.Background(), f.space, f.expt, f.run, 1) + require.NoError(t, err) + require.Len(t, queries, 1) + require.Contains(t, queries[0], "LEFT JOIN expt_item_result AS c") + require.Contains(t, queries[0], "c.deleted_at IS NULL") + require.Contains(t, queries[0], "l.deleted_at IS NULL") + require.Equal(t, []int64{12}, runtime.OccupiedItemIDs) + require.Equal(t, []int64{11}, runtime.CandidateItemIDs) + require.Equal(t, map[int64]string{11: "rq1", 12: ""}, runtime.ItemRequestIDs) + queries = nil + observations, err := dao.MGetDispatchObservations(context.Background(), f.space, f.expt, f.run, []int64{11, 12, high}) + require.NoError(t, err) + require.Len(t, queries, 1) + require.Equal(t, "rq1", observations[0].RequestID) + require.Equal(t, "", observations[1].RequestID) + require.Equal(t, high, observations[2].ItemID) + require.Equal(t, "rq2", observations[2].RequestID) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET quota_reservation_state=1 WHERE space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, high).Error) + runtime, err = dao.LoadDispatchRuntime(context.Background(), f.space, f.expt, f.run, 1) + require.NoError(t, err) + require.Equal(t, []int64{12, high}, runtime.OccupiedItemIDs) + require.Equal(t, map[int64]string{11: "rq1", 12: "", high: "rq2"}, runtime.ItemRequestIDs) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET quota_reservation_state=0 WHERE space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, high).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET status=0 WHERE space_id=? AND expt_id=? AND item_id=12", f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET status=0 WHERE space_id=? AND expt_id=? AND item_id=12", f.space, f.expt).Error) + require.NoError(t, f.conn.Callback().Query().Remove(name)) + queries = nil + expected := map[int64]string{11: "rq1", 12: "", high: "rq2"} + claimed, err := dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{12, 11, high, 11}, expected) + require.NoError(t, err) + require.Equal(t, []repo.ClaimedDispatchItem{{ItemID: 12, Mode: entity.EvaluationModeAppend, RetryTimes: 3}, {ItemID: 11, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry, RetryTimes: 3}, {ItemID: high, RequestID: "rq2", Mode: entity.EvaluationModeRetryItems, RetryTimes: 7}}, claimed) + before := f.snapshot() + claimed, err = dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11, 12, high}, expected) + require.NoError(t, err) + require.Empty(t, claimed) + require.Equal(t, before, f.snapshot()) + reset, err := dao.ResetQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11, 12, high, 11}, expected) + require.NoError(t, err) + require.Equal(t, []int64{11, 12, high}, reset) + t.Log("REAL_MYSQL_DISPATCH_MIXED_JOIN_PAYLOAD tokens=empty/RQ1/RQ2 modes=Append/FailRetry/RetryItems high_i64_preserved occupied_uncapped=true") +} + +func TestDispatchMySQLRequiredMapAndPartialSuccess(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + for _, action := range []string{"claim", "reset"} { + before := f.snapshot() + var err error + if action == "claim" { + _, err = dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11, 13}, map[int64]string{11: "rq1"}) + } else { + _, err = dao.ResetQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11, 13}, map[int64]string{11: "rq1"}) + } + require.ErrorContains(t, err, "missing") + require.Equal(t, before, f.snapshot()) + } + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET ext=? WHERE space_id=? AND expt_id=? AND item_id=13", []byte(`{`), f.space, f.expt).Error) + claimed, err := dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11, 13}, map[int64]string{11: "rq1", 13: ""}) + require.Error(t, err) + require.Equal(t, []repo.ClaimedDispatchItem{{ItemID: 11, RequestID: "rq1", Mode: entity.EvaluationModeFailRetry, RetryTimes: 3}}, claimed) + reset, err := dao.ResetQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11, 13}, map[int64]string{11: "rq1", 13: ""}) + require.Error(t, err) + require.Equal(t, []int64{11}, reset) +} + +func TestDispatchMySQLStartCurrentCombinationsAndRequeue(t *testing.T) { + for _, original := range []bool{false, true} { + t.Run(fmt.Sprintf("original_%t", original), func(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + item, token := int64(11), "rq1" + if original { + item, token = 13, "" + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET item_version_id=0 WHERE space_id=? AND expt_id=? AND item_id=13", f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET item_version_id=0 WHERE space_id=? AND expt_id=? AND item_id=13", f.space, f.expt).Error) + } + expected := map[int64]string{item: token} + before := f.snapshot() + current, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, item, token) + require.NoError(t, err) + require.Equal(t, repo.StartReservedItemResult{CurrentRequest: true}, current) + require.Equal(t, before, f.snapshot()) + _, err = dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{item}, expected) + require.NoError(t, err) + started, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, item, token) + require.NoError(t, err) + require.Equal(t, repo.StartReservedItemResult{CurrentRequest: true, Started: true}, started) + before = f.snapshot() + duplicate, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, item, token) + require.NoError(t, err) + require.Equal(t, repo.StartReservedItemResult{CurrentRequest: true}, duplicate) + require.Equal(t, before, f.snapshot()) + timestamps := map[string][]time.Time{} + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log"} { + require.NoError(t, f.conn.Exec("UPDATE "+table+" SET updated_at='2020-01-02 03:04:05' WHERE space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, item).Error) + var rows []time.Time + require.NoError(t, f.conn.Table(table).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, item).Order("id").Pluck("updated_at", &rows).Error) + timestamps[table] = rows + } + before = f.snapshot() + requeued, err := dao.RequeueProcessingItem(context.Background(), f.space, f.expt, f.run, item, token) + require.NoError(t, err) + require.True(t, requeued) + for table, want := range timestamps { + var got []time.Time + require.NoError(t, f.conn.Table(table).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, item).Order("id").Pluck("updated_at", &got).Error) + require.Equal(t, want, got, table) + } + after := f.snapshot() + require.Equal(t, before["expt_turn_evaluator_result_ref"], after["expt_turn_evaluator_result_ref"]) + require.Equal(t, before["eval_target_record"], after["eval_target_record"]) + require.Equal(t, before["evaluator_record"], after["evaluator_record"]) + requeued, err = dao.RequeueProcessingItem(context.Background(), f.space, f.expt, f.run, item, token) + require.NoError(t, err) + require.False(t, requeued) + require.Equal(t, after, f.snapshot()) + t.Log("REAL_MYSQL_DISPATCH_START_REQUEUE duplicate_current=true second_stats=false all_zombie_timestamps_preserved=true original_version_zero_valid=true") + }) + } +} + +func TestDispatchMySQLTerminalAndControlZeroWrites(t *testing.T) { + for _, kind := range []string{"canonical_success", "canonical_fail", "canonical_terminal", "log_success", "log_fail", "log_terminal", "run_finished", "latest_changed", "soft_canonical", "soft_log"} { + t.Run(kind, func(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET quota_reservation_state=1 WHERE space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Error) + table, status := "expt_item_result", int32(entity.ItemRunState_Success) + switch kind { + case "canonical_fail": + status = int32(entity.ItemRunState_Fail) + case "canonical_terminal": + status = int32(entity.ItemRunState_Terminal) + case "log_success": + table = "expt_item_result_run_log" + case "log_fail": + table = "expt_item_result_run_log" + status = int32(entity.ItemRunState_Fail) + case "log_terminal": + table = "expt_item_result_run_log" + status = int32(entity.ItemRunState_Terminal) + case "run_finished": + table = "expt_run_log" + status = int32(entity.ExptStatus_Success) + case "latest_changed": + require.NoError(t, f.conn.Exec("UPDATE experiment SET latest_run_id=? WHERE space_id=? AND id=?", f.run+1, f.space, f.expt).Error) + case "soft_canonical": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Delete(&model.ExptItemResult{}).Error) + case "soft_log": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Delete(&model.ExptItemResultRunLog{}).Error) + } + if !strings.HasPrefix(kind, "soft_") && kind != "latest_changed" { + query := "UPDATE " + table + " SET status=? WHERE space_id=? AND expt_id=?" + if table != "expt_run_log" { + query += " AND item_id=11" + } + require.NoError(t, f.conn.Exec(query, status, f.space, f.expt).Error) + } + before := f.snapshot() + result, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, 11, "rq1") + if kind == "soft_log" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, repo.StartReservedItemResult{}, result) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestDispatchMySQLAdmissionLockRejectsOldOperations(t *testing.T) { + for _, action := range []string{"ClaimQuotaReserved", "ResetQuotaReserved", "StartReservedItem", "RequeueProcessingItem"} { + t.Run(action, func(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + observed, err := dao.MGetDispatchObservations(context.Background(), f.space, f.expt, f.run, []int64{11}) + require.NoError(t, err) + require.Equal(t, "rq1", observed[0].RequestID) + // Complete RQ1 before preparing RQ2 using the actual admission port. + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET status=3 WHERE space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET status=3,result_state=1 WHERE space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_stats SET pending_cnt=pending_cnt-1,fail_cnt=fail_cnt+1 WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + req := f.request(entity.EvaluationModeFailRetry) + req.RequestID = "rq2" + req.PayloadHash = "hash2" + held, resume, done := make(chan struct{}), make(chan struct{}), make(chan error, 1) + var admittedSnapshot map[string]string + var once sync.Once + release := func() { once.Do(func() { close(resume) }) } + defer release() + var canonicalLocked atomic.Bool + name := "dispatch_admission_lock" + require.NoError(t, f.conn.Callback().Query().After("gorm:query").Register(name, func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_result" { + if _, ok := tx.Statement.Clauses["FOR"]; ok { + canonicalLocked.Store(true) + } + } + })) + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(name, func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_result" && canonicalLocked.Load() { + close(held) + select { + case <-resume: + case <-tx.Statement.Context.Done(): + _ = tx.AddError(tx.Statement.Context.Err()) + } + } + })) + require.NoError(t, f.conn.Callback().Update().After("gorm:update").Register(name+"_snapshot", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_run_log" { + capture := *f + capture.conn = tx.Session(&gorm.Session{NewDB: true}) + admittedSnapshot = capture.snapshot() + } + })) + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + go func() { var err error; defer func() { done <- err }(); _, err = f.port.RequeueTerminalItems(ctx, req) }() + select { + case <-held: + case err := <-done: + t.Fatalf("admission did not hold canonical: %v", err) + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + oldDone := make(chan error, 1) + go func() { _, err := callDispatchForSource(t, dao, action, f, 11, observed[0].RequestID); oldDone <- err }() + select { + case err := <-oldDone: + t.Fatalf("old operation bypassed admission canonical lock: %v", err) + case <-time.After(150 * time.Millisecond): + } + release() + require.NoError(t, <-done) + require.NoError(t, <-oldDone) + require.NoError(t, f.conn.Callback().Update().Remove(name)) + require.NoError(t, f.conn.Callback().Query().Remove(name)) + after := f.snapshot() + require.NotNil(t, admittedSnapshot) + require.Equal(t, admittedSnapshot, after, "old waiter must not mutate admission write set") + require.NoError(t, f.conn.Callback().Update().Remove(name+"_snapshot")) + current, err := dao.MGetDispatchObservations(context.Background(), f.space, f.expt, f.run, []int64{11}) + require.NoError(t, err) + require.Equal(t, "rq2", current[0].RequestID) + require.Equal(t, int32(entity.ItemRunState_Queueing), current[0].Status) + require.Zero(t, current[0].QuotaReservationState) + for _, old := range []string{"", "rq1"} { + _, err := callDispatchForSource(t, dao, action, f, 11, old) + require.NoError(t, err) + require.Equal(t, after, f.snapshot()) + } + t.Logf("REAL_MYSQL_DISPATCH_LOCK_BLOCKING action=%s observed=RQ1 accepted=RQ2 blocked_before_release=true old_source_zero=true", action) + }) + } +} + +// Wrap a real MySQL transaction; commit failure rolls the actual connection +// back before reporting failure. This is not a sqlmock transaction substitute. +type dispatchCommitFailurePool struct{ base *sql.DB } + +func (p *dispatchCommitFailurePool) PrepareContext(ctx context.Context, q string) (*sql.Stmt, error) { + return p.base.PrepareContext(ctx, q) +} + +func (p *dispatchCommitFailurePool) ExecContext(ctx context.Context, q string, args ...any) (sql.Result, error) { + return p.base.ExecContext(ctx, q, args...) +} + +func (p *dispatchCommitFailurePool) QueryContext(ctx context.Context, q string, args ...any) (*sql.Rows, error) { + return p.base.QueryContext(ctx, q, args...) +} + +func (p *dispatchCommitFailurePool) QueryRowContext(ctx context.Context, q string, args ...any) *sql.Row { + return p.base.QueryRowContext(ctx, q, args...) +} + +func (p *dispatchCommitFailurePool) BeginTx(ctx context.Context, options *sql.TxOptions) (gorm.ConnPool, error) { + tx, err := p.base.BeginTx(ctx, options) + if err != nil { + return nil, err + } + return &dispatchCommitFailureTx{Tx: tx}, nil +} + +type dispatchCommitFailureTx struct{ *sql.Tx } + +func (t *dispatchCommitFailureTx) Commit() error { + if err := t.Rollback(); err != nil { + return err + } + return errors.New("injected MySQL commit failure") +} + +type dispatchCommitFailureProvider struct{ base db.Provider } + +func (p *dispatchCommitFailureProvider) NewSession(ctx context.Context, options ...db.Option) *gorm.DB { + session := p.base.NewSession(ctx, options...) + if pool, ok := session.Statement.ConnPool.(*sql.DB); ok { + session.Statement.ConnPool = &dispatchCommitFailurePool{base: pool} + } + return session +} + +func (p *dispatchCommitFailureProvider) Transaction(ctx context.Context, callback func(*gorm.DB) error, options ...db.Option) error { + return p.NewSession(ctx, options...).Transaction(callback) +} + +func TestDispatchMySQLTransitionRollback(t *testing.T) { + for _, action := range []string{"StartReservedItem", "RequeueProcessingItem"} { + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_stats", "commit"} { + t.Run(action+"/"+table, func(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + _, err := dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11}, map[int64]string{11: "rq1"}) + require.NoError(t, err) + if action == "RequeueProcessingItem" { + result, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + require.True(t, result.Started) + } + before := f.snapshot() + hits := 0 + if table == "commit" { + dao = NewExptItemDispatchDAO(&dispatchCommitFailureProvider{base: f.provider}) + } else { + name := "dispatch_failure_" + table + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(name, func(tx *gorm.DB) { + if tx.Statement.Table == table { + _, same := tx.Statement.ConnPool.(*sql.Tx) + require.True(t, same) + hits++ + _ = tx.AddError(errors.New("injected MySQL write failure")) + } + })) + defer func() { require.NoError(t, f.conn.Callback().Update().Remove(name)) }() + } + value, err := callDispatchForSource(t, dao, action, f, 11, "rq1") + require.ErrorContains(t, err, "injected MySQL") + if action == "StartReservedItem" { + require.Equal(t, repo.StartReservedItemResult{}, value) + } else { + require.Equal(t, false, value) + } + if table != "commit" { + require.Equal(t, 1, hits) + } + require.Equal(t, before, f.snapshot(), "rollback includes all states/counts/timestamps/refs/scores/receipt") + t.Logf("REAL_MYSQL_DISPATCH_ROLLBACK action=%s point=%s snapshots=11", action, table) + }) + } + } +} + +func TestDispatchMySQLPendingTurnDamageRollback(t *testing.T) { + for _, damage := range []string{"missing_turn_log", "wrong_version", "bad_turn_json", "missing_stats", "unknown_turn"} { + t.Run(damage, func(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + _, err := dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11}, map[int64]string{11: "rq1"}) + require.NoError(t, err) + switch damage { + case "missing_turn_log": + require.NoError(t, f.conn.Exec("DELETE FROM expt_turn_result_run_log WHERE space_id=? AND expt_id=? AND turn_id=21", f.space, f.expt).Error) + case "wrong_version": + require.NoError(t, f.conn.Exec("UPDATE expt_turn_result_run_log SET item_version_id=0 WHERE space_id=? AND expt_id=? AND turn_id=21", f.space, f.expt).Error) + case "bad_turn_json": + require.NoError(t, f.conn.Exec("UPDATE expt_turn_result_run_log SET evaluator_result_ids=? WHERE space_id=? AND expt_id=? AND turn_id=21", []byte(`{`), f.space, f.expt).Error) + case "missing_stats": + require.NoError(t, f.conn.Exec("DELETE FROM expt_stats WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + case "unknown_turn": + require.NoError(t, f.conn.Exec("UPDATE expt_turn_result SET status=9 WHERE space_id=? AND expt_id=? AND turn_id=21", f.space, f.expt).Error) + } + before := f.snapshot() + result, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.Error(t, err) + require.Equal(t, repo.StartReservedItemResult{}, result) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestDispatchMySQLSuccessTurnPayloadPreserved(t *testing.T) { + for _, score := range []*float64{nil, gptr.Of(0.0)} { + name := "nil" + if score != nil { + name = "zero" + } + t.Run(name, func(t *testing.T) { + f, dao := prepareDispatchMySQL(t) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=? AND expt_id=? AND turn_id=22", f.space, f.expt).UpdateColumn("weighted_score", score).Error) + snapshot := func() map[string]any { + out := map[string]any{} + for _, table := range []string{"expt_turn_result", "expt_turn_result_run_log"} { + var row map[string]any + require.NoError(t, f.conn.Table(table).Where("space_id=? AND expt_id=? AND turn_id=22", f.space, f.expt).Take(&row).Error) + out[table] = row + } + var refs []model.ExptTurnEvaluatorResultRef + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Order("id").Find(&refs).Error) + out["refs"] = refs + return out + } + before := snapshot() + _, err := dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{11}, map[int64]string{11: "rq1"}) + require.NoError(t, err) + started, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + require.True(t, started.Started) + require.Equal(t, before, snapshot()) + requeued, err := dao.RequeueProcessingItem(context.Background(), f.space, f.expt, f.run, 11, "rq1") + require.NoError(t, err) + require.True(t, requeued) + require.Equal(t, before, snapshot()) + }) + } +} + +func TestDispatchReadForcesPrimary(t *testing.T) { + for _, kind := range []string{"runtime", "observations"} { + t.Run(kind, func(t *testing.T) { + conn, primary, replica := newRunLogResolverTestDB(t) + provider := dbmock.NewMockProvider(gomock.NewController(t)) + provider.EXPECT().NewSession(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, opts ...db.Option) *gorm.DB { + require.True(t, db.ContainWithMasterOpt(opts)) + return conn.Clauses(dbresolver.Write).WithContext(ctx) + }) + primary.ExpectQuery("SELECT .*l.retry_times.*LEFT JOIN expt_item_result AS c.*c.deleted_at IS NULL.*l.deleted_at IS NULL").WillReturnRows(sqlmock.NewRows([]string{"item_id", "item_version_id", "canonical_item_version_id", "canonical_id", "canonical_run_id", "status", "canonical_status", "retry_times", "quota_reservation_state", "mode", "run_status", "ext"}).AddRow(int64(11), int64(0), int64(0), int64(21), int64(8), int32(0), int32(0), int32(3), int32(0), int32(entity.EvaluationModeSubmit), int64(entity.ExptStatus_Processing), []byte(`{}`))) + dao := NewExptItemDispatchDAO(provider) + if kind == "runtime" { + result, err := dao.LoadDispatchRuntime(context.Background(), 9, 7, 8, 1) + require.NoError(t, err) + require.Equal(t, map[int64]string{11: ""}, result.ItemRequestIDs) + } else { + result, err := dao.MGetDispatchObservations(context.Background(), 9, 7, 8, []int64{11}) + require.NoError(t, err) + require.Len(t, result, 1) + require.Empty(t, result[0].RequestID) + } + require.NoError(t, primary.ExpectationsWereMet()) + require.NoError(t, replica.ExpectationsWereMet()) + }) + } +} + +func TestDispatchMySQLOrphanTurnLogRollsBack(t *testing.T) { + f := newManualRetryFixture(t) + dao := NewExptItemDispatchDAO(f.provider) + // Original execution with canonical frozen turns, but one log bound to an + // unrelated turn. Equal row counts must not make this a complete binding. + require.NoError(t, f.conn.Exec("UPDATE expt_item_result SET status=0 WHERE space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET status=0,result_state=0,quota_reservation_state=1 WHERE space_id=? AND expt_id=? AND item_id=11", f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_turn_result SET status=0 WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + require.NoError(t, f.conn.Exec("UPDATE expt_turn_result_run_log SET status=0,turn_id=99 WHERE space_id=? AND expt_id=? AND turn_id=21", f.space, f.expt).Error) + before := f.snapshot() + result, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, 11, "") + require.Error(t, err) + require.Equal(t, repo.StartReservedItemResult{}, result) + require.Equal(t, before, f.snapshot()) +} + +func TestDispatchMySQLCanonicalProcessingQueueingLog(t *testing.T) { + f := newManualRetryFixture(t) + dao := NewExptItemDispatchDAO(f.provider) + require.NoError(t, f.conn.Exec("UPDATE expt_item_result_run_log SET status=0 WHERE space_id=? AND expt_id=? AND item_id=12", f.space, f.expt).Error) + expected := map[int64]string{12: ""} + before := f.snapshot() + claimed, err := dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{12}, expected) + require.NoError(t, err) + require.Len(t, claimed, 1, "run-log queueing remains candidate with a nonterminal canonical") + reset, err := dao.ResetQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{12}, expected) + require.NoError(t, err) + require.Equal(t, []int64{12}, reset) + _, err = dao.ClaimQuotaReserved(context.Background(), f.space, f.expt, f.run, []int64{12}, expected) + require.NoError(t, err) + started, err := dao.StartReservedItem(context.Background(), f.space, f.expt, f.run, 12, "") + require.NoError(t, err) + require.True(t, started.Started) + after := f.snapshot() + require.Equal(t, before["expt_stats"], after["expt_stats"], "canonical already Processing must not be counted twice") + require.Equal(t, before["expt_item_result"], after["expt_item_result"]) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_ref.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_ref.go index 183fcf6f2b..fe0aab555f 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_ref.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_ref.go @@ -8,10 +8,12 @@ import ( "gorm.io/gen/field" "gorm.io/gorm/clause" + "gorm.io/plugin/dbresolver" "github.com/coze-dev/coze-loop/backend/infra/db" "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/query" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" "github.com/coze-dev/coze-loop/backend/pkg/errorx" ) @@ -76,6 +78,9 @@ func (dao *exptItemRefDAOImpl) ListByExptID(ctx context.Context, spaceID, exptID func (dao *exptItemRefDAOImpl) GetByExptIDAndItemID(ctx context.Context, spaceID, exptID, itemID int64, opts ...db.Option) (*model.ExptItemRef, error) { dbConn := dao.provider.NewSession(ctx, opts...) + if contexts.CtxWriteDB(ctx) { + dbConn = dbConn.Clauses(dbresolver.Write) + } q := query.Use(dbConn).ExptItemRef item, err := q.WithContext(ctx). Where(q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ItemID.Eq(itemID), q.DeletedAt.IsNull()). @@ -91,6 +96,9 @@ func (dao *exptItemRefDAOImpl) MGetByExptIDAndItemIDs(ctx context.Context, space return nil, nil } dbConn := dao.provider.NewSession(ctx, opts...) + if contexts.CtxWriteDB(ctx) { + dbConn = dbConn.Clauses(dbresolver.Write) + } q := query.Use(dbConn).ExptItemRef items, err := q.WithContext(ctx). Where(q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ItemID.In(itemIDs...), q.DeletedAt.IsNull()). diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go index 8e1018bf32..f534508bdb 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go @@ -8,7 +8,9 @@ import ( "database/sql" "fmt" + "github.com/bytedance/gg/gptr" "gorm.io/gen" + "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/hints" "gorm.io/plugin/dbresolver" @@ -16,6 +18,7 @@ import ( "github.com/coze-dev/coze-loop/backend/infra/db" "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/query" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" @@ -26,6 +29,10 @@ import ( //go:generate mockgen -destination=mocks/expt_item_result.go -package mocks . IExptItemResultDAO type IExptItemResultDAO interface { + ScanIncompleteDispatchObservations(ctx context.Context, spaceID, exptID, runID int64) ([]*repo.ExptDispatchObservation, error) + ScanCurrentRequests(ctx context.Context, spaceID, exptID, runID int64, retryFirst bool) ([]*entity.ExptItemCurrentRequest, error) + GetCurrentRequest(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptItemCurrentRequest, error) + MarkProcessingIfCurrent(ctx context.Context, spaceID, exptID, runID, itemID int64, requestID string) (bool, error) BatchGet(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptItemResult, error) BatchCreateNX(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64, opts ...db.Option) (results []*model.ExptItemResult, ncursor int64, err error) @@ -67,6 +74,12 @@ func NewExptItemResultDAO(db db.Provider) IExptItemResultDAO { } } +// Cleanup shares dispatch's canonical/run-log observation, never a later +// canonical token paired with an earlier run-log status. +func (dao *exptItemResultDAOImpl) ScanIncompleteDispatchObservations(ctx context.Context, spaceID, exptID, runID int64) ([]*repo.ExptDispatchObservation, error) { + return (&exptItemDispatchDAOImpl{db: dao.provider}).dispatchObservations(ctx, spaceID, exptID, runID, nil, true) +} + func (dao *exptItemResultDAOImpl) BatchGet(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptItemResult, error) { db := dao.provider.NewSession(ctx, opts...) if contexts.CtxWriteDB(ctx) { @@ -81,6 +94,38 @@ func (dao *exptItemResultDAOImpl) BatchGet(ctx context.Context, spaceID, exptID } func (dao *exptItemResultDAOImpl) UpdateItemsResult(ctx context.Context, spaceID, exptID int64, itemIDs []int64, ufields map[string]any, opts ...db.Option) error { + if next, ok := ufields["expt_run_id"].(int64); ok && next > 0 { + return dao.provider.Transaction(ctx, func(tx *gorm.DB) error { + var items []model.ExptItemResult + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=? AND item_id IN ?", spaceID, exptID, itemIDs).Order("item_id").Find(&items).Error; err != nil { + return err + } + for _, item := range items { + fields := make(map[string]any, len(ufields)+1) + for key, value := range ufields { + fields[key] = value + } + if item.ExptRunID != next { + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + if err != nil { + return err + } + if _, exists := ext[entity.ManualRetryExtKey]; exists { + delete(ext, entity.ManualRetryExtKey) + raw, err := json.Marshal(ext) + if err != nil { + return err + } + fields["ext"] = raw + } + } + if err := tx.Model(&model.ExptItemResult{}).Where("id=? AND space_id=? AND expt_id=?", item.ID, spaceID, exptID).Updates(fields).Error; err != nil { + return err + } + } + return ctx.Err() + }, opts...) + } db := dao.provider.NewSession(ctx, opts...) q := query.Use(db).ExptItemResult _, err := q.WithContext(ctx). @@ -236,6 +281,9 @@ func (dao *exptItemResultDAOImpl) GetItemRunLog(ctx context.Context, exptID, exp func (dao *exptItemResultDAOImpl) MGetItemRunLog(ctx context.Context, exptID, exptRunID int64, itemIDs []int64, spaceID int64, opts ...db.Option) ([]*model.ExptItemResultRunLog, error) { db := dao.provider.NewSession(ctx, opts...) + if contexts.CtxWriteDB(ctx) { + db = db.Clauses(dbresolver.Write) + } q := query.Use(db).ExptItemResultRunLog found, err := q.WithContext(ctx). Where(q.SpaceID.Eq(spaceID), @@ -351,6 +399,9 @@ func (dao *exptItemResultDAOImpl) MGetItemResults(ctx context.Context, spaceID, func (dao *exptItemResultDAOImpl) ListItemResultsByExptID(ctx context.Context, exptID, spaceID int64, page entity.Page, desc bool) ([]*model.ExptItemResult, int64, error) { db := dao.provider.NewSession(ctx) + if contexts.CtxWriteDB(ctx) { + db = db.Clauses(dbresolver.Write) + } q := query.Use(db).ExptItemResult conds := []gen.Condition{ q.SpaceID.Eq(spaceID), @@ -416,6 +467,9 @@ func (dao *exptItemResultDAOImpl) ScanItemRunLogs(ctx context.Context, exptID, e filter = &entity.ExptItemRunLogFilter{} } session := dao.provider.NewSession(ctx, opts...) + if contexts.CtxWriteDB(ctx) { + session = session.Clauses(dbresolver.Write) + } // RawFilter: use raw gorm.DB Where(sql, vars...) to avoid gen clause conversion / unknown clause issues. if filter.RawFilter && filter.RawCond.SQL != "" { diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry.go new file mode 100644 index 0000000000..6030af5af2 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry.go @@ -0,0 +1,737 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "sort" + "time" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "github.com/coze-dev/coze-loop/backend/pkg/errorx" + + "github.com/bytedance/gg/gptr" + "gorm.io/gorm" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/infra/idgen" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +type ExptManualRetryDAOImpl struct { + provider db.Provider + idgen idgen.IIDGenerator +} + +var _ repo.IExptManualRetryRepo = (*ExptManualRetryDAOImpl)(nil) + +func NewExptManualRetryDAO(provider db.Provider, generator idgen.IIDGenerator) repo.IExptManualRetryRepo { + return &ExptManualRetryDAOImpl{provider: provider, idgen: generator} +} + +func (d *ExptManualRetryDAOImpl) FindReceipt(ctx context.Context, spaceID, exptID, runID int64, requestID, hash string) (*entity.ManualRetryReceipt, error) { + run, err := NewExptRunLogDAO(d.provider).Get(ctx, exptID, runID, db.WithMaster()) + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + if run.SpaceID != spaceID { + return nil, errorx.NewByCode(errno.ManualRetryRunConflictCode) + } + chunks, err := decodeManualRetryRunItems(run) + if err != nil { + return nil, err + } + return findManualRetryReceipt(chunks, requestID, hash) +} + +func decodeManualRetryRunItems(run *model.ExptRunLog) ([]entity.ExptRunLogItems, error) { + var chunks []entity.ExptRunLogItems + if raw := gptr.Indirect(run.ItemIds); len(raw) > 0 { + if err := json.Unmarshal(raw, &chunks); err != nil { + return nil, fmt.Errorf("invalid run item metadata: %w", err) + } + } + return chunks, nil +} + +func findManualRetryReceipt(chunks []entity.ExptRunLogItems, requestID, hash string) (*entity.ManualRetryReceipt, error) { + var found *entity.ManualRetryReceipt + for _, chunk := range chunks { + receipt := chunk.ManualRetryReceipt + if receipt == nil || receipt.RequestID != requestID { + continue + } + if receipt.PayloadHash != hash { + return nil, errorx.NewByCode(errno.ManualRetryRequestConflictCode) + } + if found != nil { + return nil, fmt.Errorf("duplicate manual retry receipt") + } + found = receipt + } + return found, nil +} + +type manualRetryLogIDs struct { + itemID int64 + turnIDs map[int64]int64 +} + +func (d *ExptManualRetryDAOImpl) allocateRetryLogIDs(ctx context.Context, items []*entity.PreparedRetryItem) (map[*entity.PreparedRetryItem]*manualRetryLogIDs, error) { + result := make(map[*entity.PreparedRetryItem]*manualRetryLogIDs, len(items)) + for _, item := range items { + if item == nil || item.SourceItem == nil { + return nil, fmt.Errorf("invalid prepared retry item") + } + ids := &manualRetryLogIDs{turnIDs: make(map[int64]int64)} + generate := func() (int64, error) { + if d.idgen == nil { + return 0, fmt.Errorf("manual retry IDGen missing") + } + id, err := d.idgen.GenID(ctx) + if err != nil { + return 0, err + } + if id <= 0 { + return 0, fmt.Errorf("invalid manual retry log id") + } + return id, nil + } + var err error + if item.CurrentItemLog == nil { + ids.itemID, err = generate() + if err != nil { + return nil, err + } + } + for _, turn := range item.Turns { + if turn == nil || turn.SourceTurn == nil { + return nil, fmt.Errorf("invalid prepared retry turn") + } + if turn.CurrentRunLog == nil { + id, err := generate() + if err != nil { + return nil, err + } + ids.turnIDs[turn.SourceTurn.TurnID] = id + } + } + result[item] = ids + } + return result, nil +} + +type lockedManualRetryItem struct { + prepared *entity.PreparedRetryItem + item *model.ExptItemResult + log *model.ExptItemResultRunLog + ext []byte + turns []*lockedManualRetryTurn +} + +type lockedManualRetryTurn struct { + prepared *entity.PreparedRetryTurn + turn *model.ExptTurnResult + log *model.ExptTurnResultRunLog + sourceLog *model.ExptTurnResultRunLog +} + +func (d *ExptManualRetryDAOImpl) RequeueTerminalItems(ctx context.Context, request *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryReceipt, error) { + if request == nil || request.SpaceID <= 0 || request.ExptID <= 0 || request.RunID <= 0 || request.RequestID == "" || request.PayloadHash == "" || (request.Mode != entity.EvaluationModeFailRetry && request.Mode != entity.EvaluationModeRetryItems) { + return nil, fmt.Errorf("invalid manual retry request") + } + receipt, err := d.FindReceipt(ctx, request.SpaceID, request.ExptID, request.RunID, request.RequestID, request.PayloadHash) + if err != nil || receipt != nil { + return receipt, err + } + logIDs, err := d.allocateRetryLogIDs(ctx, request.Items) + if err != nil { + return nil, err + } + err = d.provider.Transaction(ctx, func(tx *gorm.DB) error { + opts := []db.Option{db.WithTransaction(tx), db.WithMaster(), db.WithSelectForUpdate()} + expt, err := NewExptDAO(d.provider).GetByID(ctx, request.ExptID, opts...) + if err != nil { + return err + } + run, err := NewExptRunLogDAO(d.provider).Get(ctx, request.ExptID, request.RunID, opts...) + if err != nil { + return err + } + if expt.SpaceID != request.SpaceID || run.SpaceID != request.SpaceID { + return errorx.NewByCode(errno.ManualRetryRunConflictCode) + } + chunks, err := decodeManualRetryRunItems(run) + if err != nil { + return err + } + receipt, err = findManualRetryReceipt(chunks, request.RequestID, request.PayloadHash) + if err != nil || receipt != nil { + return err + } + if expt.LatestRunID != request.RunID { + return errorx.NewByCode(errno.ManualRetryRunConflictCode) + } + if expt.Status != int32(entity.ExptStatus_Processing) || (gptr.Indirect(run.Status) != int64(entity.ExptStatus_Pending) && gptr.Indirect(run.Status) != int64(entity.ExptStatus_Processing)) || expt.ExptType != int32(entity.ExptType_Offline) { + return errorx.NewByCode(errno.ManualRetryRunFinishedCode) + } + receipt = &entity.ManualRetryReceipt{RequestID: request.RequestID, PayloadHash: request.PayloadHash, SnapshotAtMS: request.SnapshotAtMS, RejectedItems: append([]*entity.ManualRetryRejectedItem(nil), request.RejectedItems...)} + prepared := append([]*entity.PreparedRetryItem(nil), request.Items...) + for _, item := range prepared { + if item == nil || item.SourceItem == nil { + return fmt.Errorf("invalid prepared retry item") + } + } + sort.Slice(prepared, func(i, j int) bool { return prepared[i].SourceItem.ItemID < prepared[j].SourceItem.ItemID }) + locked := make([]*lockedManualRetryItem, 0, len(prepared)) + for _, rejected := range receipt.RejectedItems { + if rejected == nil || rejected.ItemID <= 0 { + return fmt.Errorf("invalid rejected retry item") + } + } + for index, item := range prepared { + if index > 0 && item.SourceItem.ItemID == prepared[index-1].SourceItem.ItemID { + return fmt.Errorf("duplicate prepared retry item") + } + current, reason, err := d.lockManualRetryItem(ctx, request, item, opts) + if err != nil { + return err + } + if reason != "" { + receipt.RejectedItems = append(receipt.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.SourceItem.ItemID, Reason: reason}) + continue + } + locked = append(locked, current) + } + accepted := make([]*lockedManualRetryItem, 0, len(locked)) + for _, item := range locked { + reason, err := d.lockManualRetryTurns(ctx, request, item, expt, opts) + if err != nil { + return err + } + if reason != "" { + receipt.RejectedItems = append(receipt.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.item.ItemID, Reason: reason}) + continue + } + accepted = append(accepted, item) + } + receipt.AcceptedCount = int32(len(accepted)) + sort.Slice(receipt.RejectedItems, func(i, j int) bool { return receipt.RejectedItems[i].ItemID < receipt.RejectedItems[j].ItemID }) + business := (&entity.ExptRunLog{ItemIds: chunks}).GetItemIDs() + contained := make(map[int64]bool, len(business)) + for _, id := range business { + contained[id] = true + } + added := make([]int64, 0) + for _, item := range accepted { + if item.log == nil && !contained[item.item.ItemID] { + contained[item.item.ItemID] = true + added = append(added, item.item.ItemID) + } + } + if len(added) > 0 { + chunks = append(chunks, entity.ExptRunLogItems{ItemIDs: added, CreateAt: gptr.Of(time.Now().Unix())}) + } + chunks = append(chunks, entity.ExptRunLogItems{CreateAt: gptr.Of(time.Now().Unix()), ManualRetryReceipt: receipt}) + encoded, err := entity.EncodeExptRunLogItems(chunks) + if errors.Is(err, entity.ErrExptRunLogItemsCapacity) { + return errorx.WrapByCode(err, errno.ManualRetryCapacityCode, errorx.WithExtraMsg(err.Error())) + } + if err != nil { + return err + } + var stats *model.ExptStats + if len(accepted) > 0 { + stats, err = NewExptStatsDAO(d.provider).Get(ctx, request.ExptID, request.SpaceID, opts...) + if err != nil { + return err + } + if stats == nil { + return fmt.Errorf("manual retry stats missing") + } + } + deltas := make(map[entity.ItemRunState]int) + for _, item := range accepted { + deltas[entity.ItemRunState(item.item.Status)]-- + deltas[entity.ItemRunState_Queueing]++ + } + if stats != nil { + for status, delta := range deltas { + count := map[entity.ItemRunState]int32{entity.ItemRunState_Queueing: stats.PendingCnt, entity.ItemRunState_Success: stats.SuccessCnt, entity.ItemRunState_Fail: stats.FailCnt, entity.ItemRunState_Terminal: stats.TerminatedCnt}[status] + if int(count)+delta < 0 { + return fmt.Errorf("manual retry stats underflow") + } + } + } + for _, item := range accepted { + if err := d.writeManualRetryItem(ctx, request, item, logIDs[item.prepared], opts); err != nil { + return err + } + } + if len(accepted) > 0 { + if err := NewExptStatsDAO(d.provider).ArithOperateCount(ctx, request.ExptID, request.SpaceID, &entity.StatsCntArithOp{OpStatusCnt: deltas}, opts...); err != nil { + return err + } + } + fields := map[string]any{"item_ids": encoded} + if len(accepted) > 0 { + type statusCount struct { + Status int32 + Count int32 + } + var counts []statusCount + if err := d.provider.NewSession(ctx, opts...).Model(&model.ExptItemResultRunLog{}).Select("status, COUNT(*) AS count").Where("space_id = ? AND expt_id = ? AND expt_run_id = ?", request.SpaceID, request.ExptID, request.RunID).Group("status").Scan(&counts).Error; err != nil { + return err + } + for _, field := range []string{"pending_cnt", "processing_cnt", "success_cnt", "fail_cnt", "terminated_cnt"} { + fields[field] = int32(0) + } + for _, count := range counts { + if field := ItemRunStateStatsField(entity.ItemRunState(count.Status)); field != "" { + fields[field] = count.Count + } + } + } + return NewExptRunLogDAO(d.provider).Update(ctx, request.ExptID, request.RunID, request.SpaceID, fields, opts...) + }, db.WithMaster()) + if err != nil { + return nil, err + } + return receipt, nil +} + +func (d *ExptManualRetryDAOImpl) lockManualRetryItem(ctx context.Context, r *entity.RequeueTerminalItemsRequest, p *entity.PreparedRetryItem, opts []db.Option) (*lockedManualRetryItem, string, error) { + rows, err := NewExptItemResultDAO(d.provider).BatchGet(ctx, r.SpaceID, r.ExptID, []int64{p.SourceItem.ItemID}, opts...) + if err != nil { + return nil, "", err + } + if len(rows) == 0 { + return nil, "not_found", nil + } + current := rows[0] + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(current.Ext)) + if err != nil { + return nil, "", err + } + retry, err := entity.ParseManualRetryContext(ext) + if err != nil { + return nil, "", err + } + previousRequest := "" + if retry != nil { + if retry.RunID != current.ExptRunID { + return nil, "", fmt.Errorf("manual retry source run mismatch") + } + previousRequest = retry.RequestID + } + if reason := retryTerminalReason(entity.ItemRunState(current.Status)); reason != "" { + return nil, reason, nil + } + source := convert.NewExptItemResultConvertor().PO2DO(current) + source.Ext = ext + if r.Mode == entity.EvaluationModeFailRetry && current.Status != int32(entity.ItemRunState_Fail) { + return nil, "source_changed", nil + } + if current.ItemVersionID != p.ItemVersionID { + return nil, "source_changed", nil + } + if current.ExptRunID != p.PreviousRunID || previousRequest != p.PreviousRequestID || !reflect.DeepEqual(source, p.SourceItem) { + return nil, "source_changed", nil + } + log, err := NewExptItemResultDAO(d.provider).GetItemRunLog(ctx, r.ExptID, r.RunID, current.ItemID, r.SpaceID, opts...) + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, "", err + } + if errors.Is(err, gorm.ErrRecordNotFound) { + log = nil + } + if log == nil { + if p.CurrentItemLog != nil || current.ExptRunID == r.RunID { + return nil, "source_changed", nil + } + } else { + if reason := retryTerminalReason(entity.ItemRunState(log.Status)); reason != "" { + return nil, reason, nil + } + if gptr.Indirect(log.ResultState) != int32(entity.ExptItemResultStateResulted) { + return nil, "result_not_ready", nil + } + if !reflect.DeepEqual(convert.NewExptItemResultRunLogConverter().PO2DO(log), p.CurrentItemLog) { + return nil, "source_changed", nil + } + } + merged, err := entity.MergeManualRetryContext(gptr.Indirect(current.Ext), &entity.ManualRetryContext{RunID: r.RunID, RequestID: r.RequestID, Mode: r.Mode, Operator: r.Operator, AcceptedAtMS: r.SnapshotAtMS}) + if err != nil { + return nil, "", err + } + return &lockedManualRetryItem{prepared: p, item: current, log: log, ext: merged}, "", nil +} + +func retryTerminalReason(status entity.ItemRunState) string { + switch status { + case entity.ItemRunState_Success, entity.ItemRunState_Fail, entity.ItemRunState_Terminal: + return "" + case entity.ItemRunState_Queueing: + return "queueing" + case entity.ItemRunState_Processing: + return "processing" + default: + return "not_terminal" + } +} + +func (d *ExptManualRetryDAOImpl) lockManualRetryTurns(ctx context.Context, r *entity.RequeueTerminalItemsRequest, item *lockedManualRetryItem, expt *model.Experiment, opts []db.Option) (string, error) { + if reason, err := d.lockManualRetrySource(ctx, r, item, expt, opts); err != nil || reason != "" { + return reason, err + } + turns, err := NewExptTurnResultDAO(d.provider).GetItemTurnResults(ctx, r.ExptID, item.item.ItemID, r.SpaceID, opts...) + if err != nil { + return "", err + } + if len(turns) != len(item.prepared.Turns) || len(turns) == 0 { + return "source_changed", nil + } + expected := make(map[int64]*entity.PreparedRetryTurn) + for _, turn := range item.prepared.Turns { + if turn == nil || turn.SourceTurn == nil || expected[turn.SourceTurn.TurnID] != nil { + return "", fmt.Errorf("invalid prepared retry turn") + } + expected[turn.SourceTurn.TurnID] = turn + } + sort.Slice(turns, func(i, j int) bool { return turns[i].ID < turns[j].ID }) + for _, turn := range turns { + p := expected[turn.TurnID] + if p == nil || turn.ExptRunID != item.prepared.PreviousRunID || turn.ItemVersionID != item.prepared.ItemVersionID || !reflect.DeepEqual(convert.NewExptTurnResultConvertor().PO2DO(turn, p.SourceTurn.EvaluatorResults), p.SourceTurn) { + return "source_changed", nil + } + var log model.ExptTurnResultRunLog + err := d.provider.NewSession(ctx, opts...).Where("space_id = ? AND expt_id = ? AND expt_run_id = ? AND item_id = ? AND turn_id = ?", r.SpaceID, r.ExptID, r.RunID, item.item.ItemID, turn.TurnID).First(&log).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return "", err + } + var currentLog *model.ExptTurnResultRunLog + if err == nil { + currentLog = &log + } else if p.CurrentRunLog != nil || item.log != nil { + return "source_changed", nil + } + var logDO *entity.ExptTurnResultRunLog + if currentLog != nil { + if currentLog.ItemVersionID != item.prepared.ItemVersionID { + return "source_changed", nil + } + if err := validateManualRetryTurnExt(currentLog.Ext); err != nil { + return "", err + } + logDO, err = convert.NewExptTurnResultRunLogConvertor().PO2DO(currentLog) + } else { + err = nil + } + if err != nil { + return "", err + } + if !reflect.DeepEqual(logDO, p.CurrentRunLog) { + return "source_changed", nil + } + refs, err := NewExptTurnEvaluatorResultRefDAO(d.provider).BatchGet(ctx, r.SpaceID, []int64{turn.ID}, opts...) + if err != nil { + return "", err + } + actual := make(map[int64]*entity.ExptTurnEvaluatorResultRef) + for _, ref := range refs { + actual[ref.ID] = convert.NewExptTurnEvaluatorResultRefConvertor().PO2DO(ref) + } + if len(actual) != len(p.SourceRefs) { + return "source_changed", nil + } + for _, ref := range p.SourceRefs { + if ref == nil || !reflect.DeepEqual(actual[ref.ID], ref) { + return "source_changed", nil + } + } + if r.Mode == entity.EvaluationModeRetryItems && (p.PreserveSuccess || p.TargetResultID != 0 || p.EvaluatorResults != nil || len(p.KeepRefIDs) > 0) { + return "", fmt.Errorf("invalid whole-row retry plan") + } + if p.PreserveSuccess && (turn.Status != int32(entity.TurnRunState_Success) || p.TargetResultID != turn.TargetResultID || len(p.KeepRefIDs) != len(actual)) { + return "", fmt.Errorf("invalid successful retry turn plan") + } + for _, id := range p.KeepRefIDs { + if actual[id] == nil { + return "", fmt.Errorf("invalid retained retry reference") + } + } + item.turns = append(item.turns, &lockedManualRetryTurn{prepared: p, turn: turn, log: currentLog}) + } + for _, source := range item.prepared.SourceTurnLogs { + if source == nil || source.SpaceID != r.SpaceID || source.ExptID != r.ExptID || source.ItemID != item.item.ItemID { + return "", fmt.Errorf("invalid frozen retry source log") + } + var row model.ExptTurnResultRunLog + if err := d.provider.NewSession(ctx, opts...).Where("space_id=? AND expt_id=? AND expt_run_id=? AND id=?", r.SpaceID, r.ExptID, source.ExptRunID, source.ID).First(&row).Error; err != nil { + return "", err + } + if err := validateManualRetryTurnExt(row.Ext); err != nil { + return "", err + } + got, err := convert.NewExptTurnResultRunLogConvertor().PO2DO(&row) + if err != nil { + return "", err + } + if !reflect.DeepEqual(got, source) { + return "source_changed", nil + } + } + if item.log == nil && item.prepared.PreviousRunID != r.RunID { + for _, turn := range item.turns { + if turn.log != nil { + continue + } + var source model.ExptTurnResultRunLog + err := d.provider.NewSession(ctx, opts...).Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND turn_id=?", r.SpaceID, r.ExptID, item.prepared.PreviousRunID, item.item.ItemID, turn.turn.TurnID).First(&source).Error + if err != nil { + return "", err + } + if source.ItemVersionID != item.prepared.ItemVersionID { + return "source_changed", nil + } + if err := validateManualRetryTurnExt(source.Ext); err != nil { + return "", err + } + if _, err := convert.NewExptTurnResultRunLogConvertor().PO2DO(&source); err != nil { + return "", err + } + turn.sourceLog = &source + } + } + return "", nil +} + +func (d *ExptManualRetryDAOImpl) writeManualRetryItem(ctx context.Context, r *entity.RequeueTerminalItemsRequest, item *lockedManualRetryItem, ids *manualRetryLogIDs, opts []db.Option) error { + conn := func() *gorm.DB { return d.provider.NewSession(ctx, opts...) } + if err := conn().Model(&model.ExptItemResult{}).Where("space_id = ? AND expt_id = ? AND id = ?", r.SpaceID, r.ExptID, item.item.ID).Updates(map[string]any{"status": int32(entity.ItemRunState_Queueing), "expt_run_id": r.RunID, "err_msg": nil, "log_id": "", "ext": item.ext}).Error; err != nil { + return err + } + if item.log == nil { + id := ids.itemID + if err := conn().Create(&model.ExptItemResultRunLog{ID: id, SpaceID: r.SpaceID, ExptID: r.ExptID, ExptRunID: r.RunID, ItemID: item.item.ItemID, ItemVersionID: item.prepared.ItemVersionID, Status: int32(entity.ItemRunState_Queueing), ResultState: gptr.Of(int32(entity.ExptItemResultStateDefault)), QuotaReservationState: int32(entity.QuotaReservationStateNone)}).Error; err != nil { + return err + } + } else { + if err := conn().Model(&model.ExptItemResultRunLog{}).Where("space_id = ? AND expt_id = ? AND expt_run_id = ? AND id = ?", r.SpaceID, r.ExptID, r.RunID, item.log.ID).Updates(map[string]any{"status": int32(entity.ItemRunState_Queueing), "result_state": int32(entity.ExptItemResultStateDefault), "quota_reservation_state": int32(entity.QuotaReservationStateNone), "err_msg": nil, "log_id": ""}).Error; err != nil { + return err + } + } + + for _, turn := range item.turns { + p := turn.prepared + canonicalFields := map[string]any{"expt_run_id": r.RunID} + logFields := map[string]any{"expt_run_id": r.RunID} + if !p.PreserveSuccess { + canonicalFields["status"] = int32(entity.TurnRunState_Queueing) + canonicalFields["target_result_id"] = p.TargetResultID + canonicalFields["weighted_score"] = nil + canonicalFields["err_msg"] = nil + canonicalFields["log_id"] = "" + canonicalFields["trace_id"] = int64(0) + results, err := p.EvaluatorResults.Serialize() + if err != nil { + return err + } + for field, value := range canonicalFields { + if field != "weighted_score" { + logFields[field] = value + } + } + logFields["evaluator_result_ids"] = results + } + if err := conn().Model(&model.ExptTurnResult{}).Where("space_id = ? AND expt_id = ? AND id = ?", r.SpaceID, r.ExptID, turn.turn.ID).Updates(canonicalFields).Error; err != nil { + return err + } + if turn.log == nil { + id := ids.turnIDs[turn.turn.TurnID] + results, err := p.EvaluatorResults.Serialize() + if err != nil { + return err + } + row := &model.ExptTurnResultRunLog{ID: id, SpaceID: r.SpaceID, ExptID: r.ExptID, ExptRunID: r.RunID, ItemID: item.item.ItemID, ItemVersionID: item.prepared.ItemVersionID, TurnID: turn.turn.TurnID, Status: int32(entity.TurnRunState_Queueing), TargetResultID: p.TargetResultID, EvaluatorResultIds: &results} + if turn.sourceLog != nil { + row.Ext = turn.sourceLog.Ext + } + if p.PreserveSuccess { + row.Status = turn.turn.Status + row.TraceID = turn.turn.TraceID + row.LogID = turn.turn.LogID + row.ErrMsg = turn.turn.ErrMsg + } + if err := conn().Create(row).Error; err != nil { + return err + } + } else { + if err := conn().Model(&model.ExptTurnResultRunLog{}).Where("space_id = ? AND expt_id = ? AND expt_run_id = ? AND id = ?", r.SpaceID, r.ExptID, r.RunID, turn.log.ID).Updates(logFields).Error; err != nil { + return err + } + } + + if !p.PreserveSuccess { + if err := NewExptTurnEvaluatorResultRefDAO(d.provider).DeleteByTurnResultIDs(ctx, r.SpaceID, r.ExptID, []int64{turn.turn.ID}, p.KeepRefIDs, opts...); err != nil { + return err + } + } + } + return nil +} + +// Preflight reuses final source validation without allocating IDs or writing rows. +func (d *ExptManualRetryDAOImpl) PreflightTerminalItems(ctx context.Context, request *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) { + if request == nil { + return nil, fmt.Errorf("invalid manual retry request") + } + plan := &entity.ManualRetryPreparation{SnapshotAtMS: request.SnapshotAtMS, RejectedItems: append([]*entity.ManualRetryRejectedItem(nil), request.RejectedItems...)} + err := d.provider.Transaction(ctx, func(tx *gorm.DB) error { + opts := []db.Option{db.WithTransaction(tx), db.WithMaster(), db.WithSelectForUpdate()} + expt, err := NewExptDAO(d.provider).GetByID(ctx, request.ExptID, opts...) + if err != nil { + return err + } + run, err := NewExptRunLogDAO(d.provider).Get(ctx, request.ExptID, request.RunID, opts...) + if err != nil { + return err + } + if expt.SpaceID != request.SpaceID || run.SpaceID != request.SpaceID || expt.LatestRunID != request.RunID { + return errorx.NewByCode(errno.ManualRetryRunConflictCode) + } + if expt.Status != int32(entity.ExptStatus_Processing) || expt.ExptType != int32(entity.ExptType_Offline) || (gptr.Indirect(run.Status) != int64(entity.ExptStatus_Pending) && gptr.Indirect(run.Status) != int64(entity.ExptStatus_Processing)) { + return errorx.NewByCode(errno.ManualRetryRunFinishedCode) + } + chunks, err := decodeManualRetryRunItems(run) + if err != nil { + return err + } + receipt, err := findManualRetryReceipt(chunks, request.RequestID, request.PayloadHash) + if err != nil { + return err + } + if receipt != nil { + return errorx.NewByCode(errno.ManualRetryBusyCode, errorx.WithExtraMsg("manual retry receipt appeared during preflight")) + } + items := append([]*entity.PreparedRetryItem(nil), request.Items...) + for _, item := range items { + if item == nil || item.SourceItem == nil { + return fmt.Errorf("invalid prepared retry item") + } + } + sort.Slice(items, func(i, j int) bool { return items[i].SourceItem.ItemID < items[j].SourceItem.ItemID }) + accepted := make([]*lockedManualRetryItem, 0, len(items)) + for i, item := range items { + if i > 0 && item.SourceItem.ItemID == items[i-1].SourceItem.ItemID { + return fmt.Errorf("duplicate prepared retry item") + } + current, reason, err := d.lockManualRetryItem(ctx, request, item, opts) + if err != nil { + return err + } + if reason == "" { + reason, err = d.lockManualRetryTurns(ctx, request, current, expt, opts) + if err != nil { + return err + } + } + if reason != "" { + plan.RejectedItems = append(plan.RejectedItems, &entity.ManualRetryRejectedItem{ItemID: item.SourceItem.ItemID, Reason: reason}) + continue + } + for _, turn := range current.turns { + raw, err := turn.prepared.EvaluatorResults.Serialize() + if err != nil { + return err + } + if len(raw) > 65535 { + return errorx.NewByCode(errno.ManualRetryCapacityCode, errorx.WithExtraMsg("manual retry evaluator results exceeds blob capacity")) + } + if turn.sourceLog != nil && len(turn.sourceLog.Ext) > 65535 { + return errorx.NewByCode(errno.ManualRetryCapacityCode, errorx.WithExtraMsg("manual retry turn ext exceeds blob capacity")) + } + } + plan.Items = append(plan.Items, item) + accepted = append(accepted, current) + } + if len(accepted) > 0 { + stats, err := NewExptStatsDAO(d.provider).Get(ctx, request.ExptID, request.SpaceID, opts...) + if err != nil { + return err + } + if stats == nil { + return fmt.Errorf("manual retry stats missing") + } + deltas := map[entity.ItemRunState]int{} + for _, item := range accepted { + deltas[entity.ItemRunState(item.item.Status)]-- + } + counts := map[entity.ItemRunState]int32{entity.ItemRunState_Success: stats.SuccessCnt, entity.ItemRunState_Fail: stats.FailCnt, entity.ItemRunState_Terminal: stats.TerminatedCnt} + for status, delta := range deltas { + if int(counts[status])+delta < 0 { + return fmt.Errorf("manual retry stats underflow") + } + } + } + // Reserve both the maximum business append and a rejection for every candidate. + contained := map[int64]bool{} + for _, id := range (&entity.ExptRunLog{ItemIds: chunks}).GetItemIDs() { + contained[id] = true + } + var added []int64 + worstRejected := append([]*entity.ManualRetryRejectedItem(nil), plan.RejectedItems...) + for _, item := range plan.Items { + id := item.SourceItem.ItemID + if item.CurrentItemLog == nil && !contained[id] { + added = append(added, id) + contained[id] = true + } + worstRejected = append(worstRejected, &entity.ManualRetryRejectedItem{ItemID: id, Reason: "reservation_mismatch"}) + } + if len(added) > 0 { + chunks = append(chunks, entity.ExptRunLogItems{ItemIDs: added, CreateAt: gptr.Of(time.Now().Unix())}) + } + chunks = append(chunks, entity.ExptRunLogItems{CreateAt: gptr.Of(time.Now().Unix()), ManualRetryReceipt: &entity.ManualRetryReceipt{RequestID: request.RequestID, PayloadHash: request.PayloadHash, SnapshotAtMS: request.SnapshotAtMS, AcceptedCount: 2147483647, RejectedItems: worstRejected}}) + _, err = entity.EncodeExptRunLogItems(chunks) + if errors.Is(err, entity.ErrExptRunLogItemsCapacity) { + return errorx.WrapByCode(err, errno.ManualRetryCapacityCode, errorx.WithExtraMsg(err.Error())) + } + return err + }, db.WithMaster()) + if err != nil { + return nil, err + } + sort.Slice(plan.RejectedItems, func(i, j int) bool { return plan.RejectedItems[i].ItemID < plan.RejectedItems[j].ItemID }) + return plan, nil +} + +func validateManualRetryTurnExt(raw []byte) error { + if len(raw) == 0 { + return nil + } + var fields map[string]*string + if err := json.Unmarshal(raw, &fields); err != nil { + return err + } + for key, value := range fields { + if value == nil { + return fmt.Errorf("retry turn ext %q must be a string", key) + } + } + if len(raw) > 65535 { + return errorx.NewByCode(errno.ManualRetryCapacityCode, errorx.WithExtraMsg("retry turn ext exceeds blob capacity")) + } + return nil +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_absent_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_absent_test.go new file mode 100644 index 0000000000..d050eefbc2 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_absent_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +func TestManualRetryMySQLAbsentCurrentRun(t *testing.T) { + f := newManualRetryFixture(t) + oldRun := f.run + newRun := f.base + 999 + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Update("latest_run_id", newRun).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("status", 2).Error) + require.NoError(t, f.conn.Model(&model.ExptStats{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Updates(map[string]any{"fail_cnt": 0, "success_cnt": 1}).Error) + require.NoError(t, f.conn.Create(&model.ExptRunLog{ID: newRun, SpaceID: f.space, ExptID: f.expt, ExptRunID: newRun, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeRetryItems)), ItemIds: gptr.Of([]byte(`[{"ItemIDs":[12,13],"CreateAt":1700000000}]`))}).Error) + require.NoError(t, f.conn.Exec("INSERT INTO expt_item_result_run_log (id,space_id,expt_id,expt_run_id,item_id,item_version_id,status) SELECT id+1000,space_id,expt_id,?,item_id,item_version_id,status FROM expt_item_result_run_log WHERE space_id=? AND expt_id=? AND item_id IN (12,13)", newRun, f.space, f.expt).Error) + req := f.request(entity.EvaluationModeRetryItems) + req.RunID = newRun + req.Items[0].CurrentItemLog = nil + for _, turn := range req.Items[0].Turns { + turn.CurrentRunLog = nil + } + before := f.snapshot() + receipt, err := f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + require.Equal(t, int32(1), receipt.AcceptedCount) + var item model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).First(&item).Error) + require.Equal(t, req.Items[0].SourceItem.ID, item.ID) + require.Equal(t, newRun, item.ExptRunID) + var itemLog model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", f.space, f.expt, newRun, 11).First(&itemLog).Error) + require.Equal(t, f.base+111, itemLog.ItemVersionID) + require.Zero(t, itemLog.RetryTimes) + require.Zero(t, *itemLog.ResultState) + var logs []*model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND expt_run_id=?", f.space, f.expt, newRun).Order("turn_id").Find(&logs).Error) + require.Len(t, logs, 2) + for _, log := range logs { + require.Equal(t, f.base+111, log.ItemVersionID) + require.NotEqual(t, req.Items[0].Turns[0].SourceTurn.ID, log.ID) + require.Zero(t, log.Status) + require.Nil(t, log.ErrMsg) + require.JSONEq(t, `{"other":"keep"}`, string(log.Ext), "new run logs retain unrelated frozen source Ext") + } + var run model.ExptRunLog + require.NoError(t, f.conn.Where("id=?", newRun).First(&run).Error) + var chunks []entity.ExptRunLogItems + require.NoError(t, json.Unmarshal(*run.ItemIds, &chunks)) + require.Equal(t, []int64{12, 13, 11}, (&entity.ExptRunLog{ItemIds: chunks}).GetItemIDs()) + require.Equal(t, int32(2), run.PendingCnt) + require.Equal(t, int32(1), run.ProcessingCnt) + var oldItemLog model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", f.space, f.expt, oldRun, 11).First(&oldItemLog).Error) + require.Equal(t, int32(3), oldItemLog.Status) + require.Equal(t, int32(3), oldItemLog.RetryTimes) + require.Equal(t, []byte("old-run-error"), *oldItemLog.ErrMsg) + var oldTurnLogs []*model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND expt_run_id=?", f.space, f.expt, oldRun).Find(&oldTurnLogs).Error) + require.Len(t, oldTurnLogs, 2) + for _, log := range oldTurnLogs { + require.Equal(t, "turn-run-log", log.LogID) + } + require.Equal(t, before["experiment"], f.snapshot()["experiment"]) + first := f.snapshot() + _, err = f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + require.Equal(t, first, f.snapshot()) + f.run = newRun + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("id=?", item.ID).Update("status", 2).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("id=?", itemLog.ID).Updates(map[string]any{"status": 2, "result_state": 1}).Error) + require.NoError(t, f.conn.Model(&model.ExptStats{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Updates(map[string]any{"pending_cnt": 1, "success_cnt": 1}).Error) + again := f.request(entity.EvaluationModeRetryItems) + again.RequestID = "rq2" + again.PayloadHash = "hash2" + _, err = f.port.RequeueTerminalItems(context.Background(), again) + require.NoError(t, err) + require.NoError(t, f.conn.Where("id=?", newRun).First(&run).Error) + require.NoError(t, json.Unmarshal(*run.ItemIds, &chunks)) + require.Equal(t, []int64{12, 13, 11}, (&entity.ExptRunLog{ItemIds: chunks}).GetItemIDs()) + require.Len(t, chunks, 4) + t.Log("REAL_MYSQL_ABSENT_LOGS_VERIFIED all frozen turns/version R1 unchanged R2 total+1 repeated business IDs once") +} + +func TestManualRetryMySQLAbsentInsertRollback(t *testing.T) { + for _, table := range []string{"expt_item_result_run_log", "expt_turn_result_run_log"} { + t.Run(table, func(t *testing.T) { + f := newManualRetryFixture(t) + req := f.request(entity.EvaluationModeRetryItems) + newRun := f.base + 999 + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Update("latest_run_id", newRun).Error) + require.NoError(t, f.conn.Create(&model.ExptRunLog{ID: newRun, SpaceID: f.space, ExptID: f.expt, ExptRunID: newRun, Status: gptr.Of(int64(entity.ExptStatus_Processing)), ItemIds: gptr.Of([]byte(`[]`))}).Error) + req.RunID = newRun + req.Items[0].CurrentItemLog = nil + for _, turn := range req.Items[0].Turns { + turn.CurrentRunLog = nil + } + before := f.snapshot() + hits := 0 + name := "fault_insert_retry" + require.NoError(t, f.conn.Callback().Create().Before("gorm:create").Register(name, func(tx *gorm.DB) { + if tx.Statement.Table == table { + hits++ + _ = tx.AddError(errors.New("injected insert failure")) + } + })) + defer func() { require.NoError(t, f.conn.Callback().Create().Remove(name)) }() + _, err := f.port.RequeueTerminalItems(context.Background(), req) + require.ErrorContains(t, err, "injected insert failure") + require.Positive(t, hits) + require.Equal(t, before, f.snapshot()) + t.Logf("REAL_MYSQL_INSERT_ROLLBACK_VERIFIED table=%s snapshots=11", table) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_boundary_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_boundary_test.go new file mode 100644 index 0000000000..6f64738452 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_boundary_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/cloudwego/kitex/pkg/kerrors" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/plugin/dbresolver" + + dbmock "github.com/coze-dev/coze-loop/backend/infra/db/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + loopjson "github.com/coze-dev/coze-loop/backend/pkg/json" +) + +func TestManualRetryReceiptBizBoundary(t *testing.T) { + receipt := &entity.ManualRetryReceipt{RequestID: "request", PayloadHash: "original"} + chunks := []entity.ExptRunLogItems{{ManualRetryReceipt: receipt}} + got, err := findManualRetryReceipt(chunks, "request", "original") + require.NoError(t, err) + require.Same(t, receipt, got) + _, err = findManualRetryReceipt(chunks, "request", "changed") + biz, ok := kerrors.FromBizStatusError(err) + require.True(t, ok, "request conflict must survive PacketAdapter") + require.Equal(t, int32(601204103), biz.BizStatusCode()) +} + +func TestManualRetryTurnCapacityBizBoundary(t *testing.T) { + err := validateManualRetryTurnExt([]byte(`{"padding":"` + strings.Repeat("x", 65536) + `"}`)) + biz, ok := kerrors.FromBizStatusError(err) + require.True(t, ok, "turn ext capacity must survive PacketAdapter") + require.Equal(t, int32(601204104), biz.BizStatusCode()) +} + +func TestManualRetryReceiptDBErrorRetainsCause(t *testing.T) { + conn, source, _ := newRunLogResolverTestDB(t) + provider := dbmock.NewMockProvider(gomock.NewController(t)) + provider.EXPECT().NewSession(gomock.Any(), gomock.Any()).Return(conn.Clauses(dbresolver.Write)) + cause := errors.New("database unavailable") + source.ExpectQuery("SELECT .* FROM `expt_run_log`").WillReturnError(cause) + _, err := NewExptManualRetryDAO(provider, nil).FindReceipt(context.Background(), 9, 7, 8, "request", "hash") + require.ErrorIs(t, err, cause) + _, ok := kerrors.FromBizStatusError(err) + require.False(t, ok) + require.NoError(t, source.ExpectationsWereMet()) +} + +func TestOrdinaryRunLogConversionCapacity(t *testing.T) { + ids := make([]int64, 4000) + for i := range ids { + ids[i] = 9223372036854700000 + int64(i) + } + _, err := convert.NewExptRunLogConvertor().DO2PO(&entity.ExptRunLog{ItemIds: []entity.ExptRunLogItems{{ItemIDs: ids}}}) + require.Error(t, err) + _, ok := kerrors.FromBizStatusError(err) + require.False(t, ok, "ordinary Create/Save conversion must not produce a manual retry business error") +} + +func TestRunLogCapacityMySQLBoundaries(t *testing.T) { + for _, operation := range []string{"preflight", "requeue", "ordinary_append"} { + for _, kind := range []string{"capacity", "corrupt"} { + t.Run(operation+"/"+kind, func(t *testing.T) { + f := newManualRetryFixture(t) + req := f.request(entity.EvaluationModeRetryItems) + raw := []byte(`[{"ItemIDs":[11],"padding":"` + strings.Repeat("x", 65420) + `","future_id":9223372036854775806}]`) + if kind == "corrupt" { + raw = []byte(`{`) + } + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("id=?", f.run).Update("item_ids", raw).Error) + before := f.snapshot() + var err error + switch operation { + case "preflight": + plan, callErr := f.port.PreflightTerminalItems(context.Background(), req) + err = callErr + require.Nil(t, plan) + case "requeue": + receipt, callErr := f.port.RequeueTerminalItems(context.Background(), req) + err = callErr + require.Nil(t, receipt) + case "ordinary_append": + ids := make([]int64, 20) + for i := range ids { + ids[i] = 9223372036854700000 + int64(i) + } + err = NewExptRunLogDAO(f.provider).AppendRunItemIDs(context.Background(), f.expt, f.run, f.space, ids) + } + require.Error(t, err) + require.Equal(t, before, f.snapshot(), "capacity/corruption must leave all fixture tables unchanged") + biz, ok := kerrors.FromBizStatusError(err) + if kind == "capacity" && operation != "ordinary_append" { + require.True(t, ok) + require.Equal(t, int32(601204104), biz.BizStatusCode()) + } else { + require.False(t, ok) + } + if kind == "capacity" { + require.ErrorIs(t, err, entity.ErrExptRunLogItemsCapacity) + } + if kind == "corrupt" { + require.NotErrorIs(t, err, entity.ErrExptRunLogItemsCapacity) + if operation == "ordinary_append" { + var chunks []entity.ExptRunLogItems + expected := loopjson.Unmarshal(raw, &chunks) + cause := err + for errors.Unwrap(cause) != nil { + cause = errors.Unwrap(cause) + } + require.IsType(t, expected, cause) + require.Equal(t, expected.Error(), cause.Error()) + } else { + var cause *json.SyntaxError + require.ErrorAs(t, err, &cause) + } + } + }) + } + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_dao_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_dao_test.go new file mode 100644 index 0000000000..eb03e61897 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_dao_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "errors" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" + gomysql "gorm.io/driver/mysql" + "gorm.io/gorm" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +func newManualRetrySQLProvider(t *testing.T) (db.Provider, sqlmock.Sqlmock) { + t.Helper() + sqlDB, mock, err := sqlmock.New() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + provider, err := db.NewDB(gomysql.New(gomysql.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), &gorm.Config{SkipDefaultTransaction: true}) + require.NoError(t, err) + return provider, mock +} + +func TestManualRetryStatsAtomicUpdateError(t *testing.T) { + provider, mock := newManualRetrySQLProvider(t) + want := errors.New("stats write failed") + mock.ExpectExec("UPDATE `expt_stats` SET .*`fail_cnt`=fail_cnt \\+ \\?.*`pending_cnt`=pending_cnt \\+ \\?.*WHERE \\(space_id = \\? AND expt_id = \\?\\)"). + WillReturnError(want) + err := NewExptStatsDAO(provider).ArithOperateCount(context.Background(), 7, 9, &entity.StatsCntArithOp{OpStatusCnt: map[entity.ItemRunState]int{entity.ItemRunState_Fail: -1, entity.ItemRunState_Queueing: 1}}) + require.ErrorContains(t, err, "stats write failed") + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestManualRetryDAOTransactionOptions(t *testing.T) { + provider, rootMock := newManualRetrySQLProvider(t) + txProvider, mock := newManualRetrySQLProvider(t) + expt, ok := any(NewExptDAO(provider)).(interface { + GetByID(context.Context, int64, ...db.Option) (*model.Experiment, error) + UpdateFields(context.Context, int64, map[string]any, ...db.Option) error + }) + require.True(t, ok, "experiment DAO must consume transaction options") + stats, ok := any(NewExptStatsDAO(provider)).(interface { + Get(context.Context, int64, int64, ...db.Option) (*model.ExptStats, error) + UpdateByExptID(context.Context, int64, int64, *model.ExptStats, ...db.Option) error + ArithOperateCount(context.Context, int64, int64, *entity.StatsCntArithOp, ...db.Option) error + }) + require.True(t, ok, "stats DAO must consume transaction options") + ctx := context.Background() + mock.ExpectBegin() + mock.ExpectQuery("SELECT .* FROM `experiment` WHERE .*`id` = \\?.*FOR UPDATE").WithArgs(int64(7), 1). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(7)) + mock.ExpectExec("UPDATE `experiment` SET `latest_run_id`=\\? WHERE .*`id` = \\?").WithArgs(int64(8), int64(7)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery("SELECT .* FROM `expt_stats` WHERE .*`expt_id` = \\?.*`space_id` = \\?.*FOR UPDATE").WithArgs(int64(7), int64(9), 1). + WillReturnRows(sqlmock.NewRows([]string{"expt_id", "space_id"}).AddRow(7, 9)) + mock.ExpectExec("UPDATE `expt_stats` SET .*`fail_cnt`=\\?.*`pending_cnt`=\\?.*`processing_cnt`=\\?.*`success_cnt`=\\?.*`terminated_cnt`=\\?.*WHERE .*`expt_id` = \\?.*`space_id` = \\?"). + WithArgs(int32(0), int32(1), int32(0), int32(0), int32(0), sqlmock.AnyArg(), int64(7), int64(9)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery("SELECT .* FROM `expt_run_log` WHERE \\(id = \\? AND expt_id = \\? AND expt_run_id = \\?\\).*FOR UPDATE").WithArgs(int64(8), int64(7), int64(8), 1). + WillReturnRows(sqlmock.NewRows([]string{"id", "expt_id", "expt_run_id", "space_id"}).AddRow(8, 7, 8, 9)) + mock.ExpectExec("UPDATE `expt_run_log` SET `status_message`=\\? WHERE .*`space_id` = \\?.*`expt_id` = \\?.*`expt_run_id` = \\?").WithArgs(nil, int64(9), int64(7), int64(8)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("UPDATE `expt_stats` SET `fail_cnt`=fail_cnt \\+ \\?,`pending_cnt`=pending_cnt \\+ \\?,`updated_at`=\\? WHERE \\(space_id = \\? AND expt_id = \\?\\)"). + WithArgs(-1, 1, sqlmock.AnyArg(), int64(9), int64(7)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + err := txProvider.Transaction(ctx, func(tx *gorm.DB) error { + opts := []db.Option{db.WithTransaction(tx), db.WithMaster(), db.WithSelectForUpdate()} + _, err := expt.GetByID(ctx, 7, opts...) + if err != nil { + return err + } + if err = expt.UpdateFields(ctx, 7, map[string]any{"latest_run_id": int64(8)}, opts...); err != nil { + return err + } + if _, err = stats.Get(ctx, 7, 9, opts...); err != nil { + return err + } + if err = stats.UpdateByExptID(ctx, 7, 9, &model.ExptStats{PendingCnt: 1}, opts...); err != nil { + return err + } + runDAO := NewExptRunLogDAO(provider) + if _, err = runDAO.Get(ctx, 7, 8, opts...); err != nil { + return err + } + if err = runDAO.Update(ctx, 7, 8, 9, map[string]any{"status_message": nil}, opts...); err != nil { + return err + } + return stats.ArithOperateCount(ctx, 7, 9, &entity.StatsCntArithOp{OpStatusCnt: map[entity.ItemRunState]int{entity.ItemRunState_Fail: -1, entity.ItemRunState_Queueing: 1}}, opts...) + }) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + require.NoError(t, rootMock.ExpectationsWereMet()) +} + +func TestManualRetryRefsDeleteScoped(t *testing.T) { + provider, rootMock := newManualRetrySQLProvider(t) + txProvider, mock := newManualRetrySQLProvider(t) + dao, ok := any(NewExptTurnEvaluatorResultRefDAO(provider)).(interface { + DeleteByTurnResultIDs(context.Context, int64, int64, []int64, []int64, ...db.Option) error + }) + require.True(t, ok, "refs deletion must be scoped and preserve selected ref IDs") + ctx := context.Background() + mock.ExpectBegin() + mock.ExpectExec("DELETE FROM `expt_turn_evaluator_result_ref` WHERE \\(space_id = \\? AND expt_id = \\? AND expt_turn_result_id IN \\(\\?,\\?\\)\\) AND id NOT IN \\(\\?\\)"). + WithArgs(int64(9), int64(7), int64(11), int64(12), int64(100)).WillReturnResult(sqlmock.NewResult(0, 2)) + mock.ExpectExec("DELETE FROM `expt_turn_evaluator_result_ref` WHERE space_id = \\? AND expt_id = \\? AND expt_turn_result_id IN \\(\\?\\)"). + WithArgs(int64(9), int64(7), int64(13)).WillReturnError(errors.New("delete failed")) + mock.ExpectRollback() + err := txProvider.Transaction(ctx, func(tx *gorm.DB) error { + opts := []db.Option{db.WithTransaction(tx), db.WithMaster()} + if err := dao.DeleteByTurnResultIDs(ctx, 9, 7, nil, nil, opts...); err != nil { + return err + } + if err := dao.DeleteByTurnResultIDs(ctx, 9, 7, []int64{11, 12}, []int64{100}, opts...); err != nil { + return err + } + return dao.DeleteByTurnResultIDs(ctx, 9, 7, []int64{13}, nil, opts...) + }) + require.ErrorContains(t, err, "delete failed") + require.NoError(t, mock.ExpectationsWereMet()) + require.NoError(t, rootMock.ExpectationsWereMet()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_filter.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_filter.go new file mode 100644 index 0000000000..51bb4ad7f3 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_filter.go @@ -0,0 +1,50 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "fmt" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +var _ repo.IExptManualRetryFilterRepo = (*exptItemResultDAOImpl)(nil) + +func (dao *exptItemResultDAOImpl) ScanManualRetryFilterItems(ctx context.Context, space, expt, run, cursor int64, limit int) ([]int64, int64, error) { + if limit <= 0 { + return nil, 0, fmt.Errorf("invalid manual retry filter page size") + } + var rows []*model.ExptItemResult + err := dao.provider.NewSession(ctx, db.WithMaster()).Select("id,item_id,expt_run_id,ext").Where("space_id=? AND expt_id=? AND expt_run_id=? AND id>? AND ext IS NOT NULL", space, expt, run, cursor).Order("id ASC").Limit(limit).Find(&rows).Error + if err != nil { + return nil, 0, err + } + ids := []int64{} + for _, row := range rows { + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(row.Ext)) + if err != nil { + return nil, 0, err + } + manual, err := entity.ParseManualRetryContext(ext) + if err != nil { + return nil, 0, err + } + if manual == nil { + continue + } + if manual.RunID != row.ExptRunID { + return nil, 0, fmt.Errorf("manual retry filter canonical run mismatch") + } + ids = append(ids, row.ItemID) + } + next := int64(0) + if len(rows) == limit { + next = rows[len(rows)-1].ID + } + return ids, next, nil +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_preflight_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_preflight_test.go new file mode 100644 index 0000000000..0d23ac2b51 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_preflight_test.go @@ -0,0 +1,80 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "strings" + "testing" + + "github.com/cloudwego/kitex/pkg/kerrors" + "github.com/stretchr/testify/require" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +type terminalRetryPreflight interface { + PreflightTerminalItems(context.Context, *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) +} + +func TestManualRetryMySQLPreflight(t *testing.T) { + for _, kind := range []string{"valid_read_only", "source_changed", "capacity", "ext_capacity", "evaluator_capacity", "corrupt_turn_ext"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + req := f.request(entity.EvaluationModeRetryItems) + switch kind { + case "source_changed": + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("id=?", req.Items[0].SourceItem.ID).Update("log_id", "changed").Error) + case "capacity": + raw := []byte(`[{"ItemIDs":[11],"padding":"` + strings.Repeat("x", 65420) + `"}]`) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("id=?", f.run).Update("item_ids", raw).Error) + case "ext_capacity": + raw := []byte(`{"other":"` + strings.Repeat("x", 65450) + `"}`) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("id=?", req.Items[0].SourceItem.ID).Update("ext", raw).Error) + req = f.request(entity.EvaluationModeRetryItems) + case "evaluator_capacity": + req = f.request(entity.EvaluationModeFailRetry) + req.Items[0].Turns[0].EvaluatorResults = &entity.EvaluatorResults{Inline: []*entity.InlineEvalResult{{InlineKey: strings.Repeat("x", 65536), RecordID: 51}}} + case "corrupt_turn_ext": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Update("ext", []byte(`{"bad":null}`)).Error) + } + before := f.snapshot() + port, ok := f.port.(terminalRetryPreflight) + require.True(t, ok) + plan, err := port.PreflightTerminalItems(context.Background(), req) + require.Equal(t, before, f.snapshot(), "preflight must not write a receipt, reset, or allocate log IDs") + if kind == "capacity" || kind == "ext_capacity" || kind == "evaluator_capacity" || kind == "corrupt_turn_ext" { + require.Error(t, err) + require.Nil(t, plan) + biz, ok := kerrors.FromBizStatusError(err) + if kind == "corrupt_turn_ext" { + require.False(t, ok) + } else { + require.True(t, ok) + require.Equal(t, int32(601204104), biz.BizStatusCode()) + } + return + } + require.NoError(t, err) + if kind == "source_changed" { + require.Empty(t, plan.Items) + require.Equal(t, []*entity.ManualRetryRejectedItem{{ItemID: 11, Reason: "source_changed"}}, plan.RejectedItems) + return + } + require.Len(t, plan.Items, 1) + require.Empty(t, plan.RejectedItems) + require.Equal(t, int64(1700000000123), plan.SnapshotAtMS) + receipt, err := f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + require.Equal(t, int32(1), receipt.AcceptedCount) + got, err := f.port.FindReceipt(context.Background(), f.space, f.expt, f.run, req.RequestID, req.PayloadHash) + require.NoError(t, err) + require.Equal(t, receipt.RequestID, got.RequestID) + require.Equal(t, receipt.AcceptedCount, got.AcceptedCount) + require.Equal(t, receipt.SnapshotAtMS, got.SnapshotAtMS) + require.Equal(t, receipt.RejectedItems, got.RejectedItems) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_read_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_read_test.go new file mode 100644 index 0000000000..3aac2c003d --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_read_test.go @@ -0,0 +1,37 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + dbmock "github.com/coze-dev/coze-loop/backend/infra/db/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" +) + +func TestManualRetryPreparationReadsMaster(t *testing.T) { + for _, kind := range []string{"item_run_log", "frozen_ref"} { + t.Run(kind, func(t *testing.T) { + conn, source, _ := newRunLogResolverTestDB(t) + provider := dbmock.NewMockProvider(gomock.NewController(t)) + provider.EXPECT().NewSession(gomock.Any()).Return(conn) + ctx := contexts.WithCtxWriteDB(context.Background()) + if kind == "item_run_log" { + source.ExpectQuery("SELECT .* FROM `expt_item_result_run_log`").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + _, err := NewExptItemResultDAO(provider).MGetItemRunLog(ctx, 7, 8, []int64{11}, 9) + require.NoError(t, err) + } else { + source.ExpectQuery("SELECT .* FROM `expt_item_ref`").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + _, err := NewExptItemRefDAO(provider).GetByExptIDAndItemID(ctx, 9, 7, 11) + require.NoError(t, err) + } + require.NoError(t, source.ExpectationsWereMet()) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_single_set_fixture_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_single_set_fixture_test.go new file mode 100644 index 0000000000..f95cd6a090 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_single_set_fixture_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +// The external manager integration test uses the existing real MySQL fixture. +// Keeping this bridge in test code avoids the production Wire import cycle. +func NewManualRetryFixtureForTest(t *testing.T) (db.Provider, *gorm.DB, repo.IExptManualRetryRepo, int64, int64, int64, int64, func() map[string]string) { + f := newManualRetryFixture(t) + return f.provider, f.conn, f.port, f.space, f.expt, f.run, f.base, f.snapshot +} + +func TestManualRetryLockedSingleSetSource(t *testing.T) { + for _, kind := range []string{"unchanged", "source_type", "set_id", "set_version", "set_space", "target_id", "target_version", "target_space", "target_type", "draft_zero", "draft_set_id", "item_log_version"} { + t.Run(kind, func(t *testing.T) { + expt := &model.Experiment{EvalSetSourceType: 1, EvalSetID: 31, EvalSetVersionID: 32, EvalSetSpaceID: 91, TargetID: 41, TargetVersionID: 42, TargetSpaceID: 92, TargetType: 1} + item := &lockedManualRetryItem{item: &model.ExptItemResult{ItemVersionID: 0}, log: &model.ExptItemResultRunLog{ItemVersionID: 0}, prepared: &entity.PreparedRetryItem{ItemVersionID: 0, SingleSetSource: &entity.PreparedRetrySingleSetSource{SourceType: 1, EvalSetID: 31, EvalSetVersionID: 32, EvalSetSpaceID: 91, TargetID: 41, TargetVersionID: 42, TargetSpaceID: 92, TargetType: 1}}} + switch kind { + case "source_type": + expt.EvalSetSourceType = 2 + case "set_id": + expt.EvalSetID++ + case "set_version": + expt.EvalSetVersionID++ + case "set_space": + expt.EvalSetSpaceID++ + case "target_id": + expt.TargetID++ + case "target_version": + expt.TargetVersionID++ + case "target_space": + expt.TargetSpaceID++ + case "target_type": + expt.TargetType++ + case "draft_zero": + expt.EvalSetVersionID = 0 + item.prepared.SingleSetSource.EvalSetVersionID = 0 + case "draft_set_id": + expt.EvalSetVersionID = 31 + item.prepared.SingleSetSource.EvalSetVersionID = 31 + case "item_log_version": + item.log.ItemVersionID = 1 + } + reason, err := (&ExptManualRetryDAOImpl{}).lockManualRetrySource(context.Background(), &entity.RequeueTerminalItemsRequest{}, item, expt, nil) + require.NoError(t, err) + if kind == "unchanged" { + require.Empty(t, reason) + } else { + require.Equal(t, "source_changed", reason) + } + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_single_set_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_single_set_test.go new file mode 100644 index 0000000000..555762cbde --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_single_set_test.go @@ -0,0 +1,501 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gorm.io/gorm" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/infra/lock" + lockmocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" + componentmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +type manualRetryFixture struct { + provider db.Provider + conn *gorm.DB + port repo.IExptManualRetryRepo + space, expt, run, base int64 + snapshot func() map[string]string +} + +func newManualRetryFixture(t *testing.T) *manualRetryFixture { + provider, conn, port, space, expt, run, base, snapshot := mysql.NewManualRetryFixtureForTest(t) + return &manualRetryFixture{provider, conn, port, space, expt, run, base, snapshot} +} + +// Keep the manager and its SQL preparation/acceptance ports real. Only remote +// configuration, the renewable locks, and the central ledger are test doubles. +type singleSetRetryStorage struct { + f *manualRetryFixture +} + +type singleSetExperimentStorage struct { + repo.IExperimentRepo + *singleSetRetryStorage +} + +type singleSetRunStorage struct { + repo.IExptRunLogRepo + *singleSetRetryStorage +} + +type singleSetItemStorage struct { + repo.IExptItemResultRepo + *singleSetRetryStorage +} + +type singleSetTurnStorage struct { + repo.IExptTurnResultRepo + *singleSetRetryStorage +} + +type singleSetRefStorage struct { + repo.IExptItemRefRepo + *singleSetRetryStorage +} + +type singleSetTargetStorage struct { + service.IEvalTargetService + *singleSetRetryStorage +} + +type singleSetEvaluatorStorage struct { + service.EvaluatorRecordService + *singleSetRetryStorage +} + +func (s *singleSetExperimentStorage) GetByID(ctx context.Context, expt, space int64) (*entity.Experiment, error) { + row, err := mysql.NewExptDAO(s.f.provider).GetByID(ctx, expt) + if err != nil { + return nil, err + } + return convert.NewExptConverter().PO2DO(row, nil) +} + +func (s *singleSetRunStorage) Get(ctx context.Context, expt, run int64) (*entity.ExptRunLog, error) { + row, err := mysql.NewExptRunLogDAO(s.f.provider).Get(ctx, expt, run) + if err != nil { + return nil, err + } + return convert.NewExptRunLogConvertor().PO2DO(row) +} + +func (s *singleSetItemStorage) BatchGet(ctx context.Context, space, expt int64, ids []int64) ([]*entity.ExptItemResult, error) { + rows, err := mysql.NewExptItemResultDAO(s.f.provider).BatchGet(ctx, space, expt, ids) + var result []*entity.ExptItemResult + for _, row := range rows { + item := convert.NewExptItemResultConvertor().PO2DO(row) + item.Ext, err = entity.ParseExptItemResultExt(gptr.Indirect(row.Ext)) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, err +} + +func (s *singleSetItemStorage) ScanItemResults(ctx context.Context, expt, cursor, limit int64, statuses []int32, space int64) ([]*entity.ExptItemResult, int64, error) { + // The fixture has exactly one failed item, as in the captured A/B/C case. + items, err := s.BatchGet(ctx, space, expt, []int64{11}) + return items, 0, err +} + +func (s *singleSetItemStorage) MGetItemRunLog(ctx context.Context, expt, run int64, ids []int64, space int64) ([]*entity.ExptItemResultRunLog, error) { + var result []*entity.ExptItemResultRunLog + for _, id := range ids { + row, err := mysql.NewExptItemResultDAO(s.f.provider).GetItemRunLog(ctx, expt, run, id, space) + if err != nil { + return nil, err + } + result = append(result, convert.NewExptItemResultRunLogConverter().PO2DO(row)) + } + return result, nil +} + +func (s *singleSetRefStorage) GetByExptIDAndItemID(ctx context.Context, space, expt, item int64) (*entity.ExptItemRef, error) { + row, err := mysql.NewExptItemRefDAO(s.f.provider).GetByExptIDAndItemID(ctx, space, expt, item) + return convert.NewExptItemRefConvertor().PO2DO(row), err +} + +func (s *singleSetTurnStorage) GetItemTurnResults(ctx context.Context, expt, item, space int64) ([]*entity.ExptTurnResult, error) { + rows, err := mysql.NewExptTurnResultDAO(s.f.provider).GetItemTurnResults(ctx, expt, item, space) + var result []*entity.ExptTurnResult + for _, row := range rows { + result = append(result, convert.NewExptTurnResultConvertor().PO2DO(row, nil)) + } + return result, err +} + +func (s *singleSetTurnStorage) GetItemTurnRunLogs(ctx context.Context, expt, run, item, space int64) ([]*entity.ExptTurnResultRunLog, error) { + rows, err := mysql.NewExptTurnResultDAO(s.f.provider).GetItemTurnRunLogs(ctx, expt, run, item, space) + var result []*entity.ExptTurnResultRunLog + for _, row := range rows { + log, err := convert.NewExptTurnResultRunLogConvertor().PO2DO(row) + if err != nil { + return nil, err + } + result = append(result, log) + } + return result, err +} + +func (s *singleSetTurnStorage) BatchGetTurnEvaluatorResultRef(ctx context.Context, space int64, ids []int64) ([]*entity.ExptTurnEvaluatorResultRef, error) { + rows, err := mysql.NewExptTurnEvaluatorResultRefDAO(s.f.provider).BatchGet(ctx, space, ids) + var result []*entity.ExptTurnEvaluatorResultRef + for _, row := range rows { + result = append(result, convert.NewExptTurnEvaluatorResultRefConvertor().PO2DO(row)) + } + return result, err +} + +func (s *singleSetTargetStorage) GetRecordByID(ctx context.Context, space, id int64) (*entity.EvalTargetRecord, error) { + var row struct { + ID, SpaceID, TargetVersionID, ExperimentRunID, ItemID, ItemVersionID, TurnID int64 + Status entity.EvalTargetRunStatus + } + err := s.f.conn.WithContext(ctx).Raw("SELECT id, space_id, target_version_id, experiment_run_id, item_id, item_version_id, turn_id, status FROM eval_target_record WHERE space_id=? AND id=?", space, id).Scan(&row).Error + return &entity.EvalTargetRecord{ID: row.ID, SpaceID: row.SpaceID, TargetVersionID: row.TargetVersionID, ExperimentRunID: row.ExperimentRunID, ItemID: row.ItemID, ItemVersionID: row.ItemVersionID, TurnID: row.TurnID, Status: &row.Status}, err +} + +func (s *singleSetEvaluatorStorage) BatchGetEvaluatorRecord(ctx context.Context, ids []int64, include, storage bool) ([]*entity.EvaluatorRecord, error) { + var rows []struct { + ID, SpaceID, ExperimentID, ExperimentRunID, ItemID, ItemVersionID, TurnID, EvaluatorVersionID int64 + Status entity.EvaluatorRunStatus + } + err := s.f.conn.WithContext(ctx).Table("evaluator_record").Select("id, space_id, experiment_id, experiment_run_id, item_id, item_version_id, turn_id, evaluator_version_id, status").Where("space_id=? AND id IN ?", s.f.space, ids).Find(&rows).Error + var records []*entity.EvaluatorRecord + for _, row := range rows { + records = append(records, &entity.EvaluatorRecord{ID: row.ID, SpaceID: row.SpaceID, ExperimentID: row.ExperimentID, ExperimentRunID: row.ExperimentRunID, ItemID: row.ItemID, ItemVersionID: row.ItemVersionID, TurnID: row.TurnID, EvaluatorVersionID: row.EvaluatorVersionID, Status: row.Status}) + } + return records, err +} + +type singleSetRetryPort struct { + repo.IExptManualRetryRepo + beforePreflight func() + beforeCommit func() + prepared *entity.RequeueTerminalItemsRequest +} + +type singleSetRetryWaker struct{} + +func (singleSetRetryWaker) WakeCurrentRun(context.Context, string) error { return nil } + +func (p *singleSetRetryPort) PreflightTerminalItems(ctx context.Context, req *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryPreparation, error) { + p.prepared = req + if p.beforePreflight != nil { + p.beforePreflight() + } + return p.IExptManualRetryRepo.PreflightTerminalItems(ctx, req) +} + +func (p *singleSetRetryPort) RequeueTerminalItems(ctx context.Context, req *entity.RequeueTerminalItemsRequest) (*entity.ManualRetryReceipt, error) { + if p.beforeCommit != nil { + p.beforeCommit() + } + return p.IExptManualRetryRepo.RequeueTerminalItems(ctx, req) +} + +func singleSetRetryManager(t *testing.T, f *manualRetryFixture, port repo.IExptManualRetryRepo, enforce bool) service.IExptManager { + t.Helper() + ctrl := gomock.NewController(t) + config := componentmocks.NewMockIConfiger(ctrl) + config.EXPECT().GetExptExecConf(gomock.Any(), f.space).Return(&entity.ExptExecConf{RunningRetryEnabled: gptr.Of(true)}).AnyTimes() + locker := lockmocks.NewMockILocker(ctrl) + locker.EXPECT().LockWithRenew(gomock.Any(), gomock.Any(), 5*time.Second, 5*time.Minute).DoAndReturn(func(ctx context.Context, key string, ttl, renewal time.Duration) (bool, context.Context, func(), error) { + child, cancel := context.WithCancel(ctx) + return true, child, cancel, nil + }).AnyTimes() + guard := componentmocks.NewMockICentralReservationGuard(ctrl) + waker := singleSetRetryWaker{} + if enforce { + guard.EXPECT().ReleaseTerminalReservation(gomock.Any(), "single-set-test", f.run, int64(11), "").Return(component.TerminalReservationAbsent, nil).MaxTimes(1) + } + s := &singleSetRetryStorage{f: f} + return service.NewExptManager( + nil, &singleSetExperimentStorage{singleSetRetryStorage: s}, &singleSetRunStorage{singleSetRetryStorage: s}, nil, + &singleSetItemStorage{singleSetRetryStorage: s}, &singleSetRefStorage{singleSetRetryStorage: s}, &singleSetTurnStorage{singleSetRetryStorage: s}, config, nil, locker, nil, nil, + nil, nil, nil, nil, nil, nil, &singleSetTargetStorage{singleSetRetryStorage: s}, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, guard, port, &singleSetEvaluatorStorage{singleSetRetryStorage: s}, waker, + func() lock.ILocker { return locker }, + ) +} + +func prepareSingleSetFixture(t *testing.T, f *manualRetryFixture, enforce, zeroVersion bool) { + t.Helper() + dispatch := entity.ExptDispatchModeLegacy + if enforce { + dispatch = entity.ExptDispatchModeEnforce + } + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Updates(map[string]any{ + "eval_set_source_type": 1, "eval_set_id": f.base + 31, "eval_set_version_id": f.base + 32, + "target_id": f.base + 41, "target_version_id": f.base + 42, + "scheduler_mode": dispatch, "scheduler_scope": "single-set-test", + }).Error) + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Delete(&model.ExptItemRef{}).Error) + // Turn zero is a real first turn, not an absence sentinel. + for _, table := range []string{"expt_turn_result", "expt_turn_result_run_log", "eval_target_record", "evaluator_record"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=? AND turn_id=?", f.space, 11, 21).Update("turn_id", 0).Error) + } + if zeroVersion { + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "eval_target_record", "evaluator_record"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, 11).Update("item_version_id", 0).Error) + } + } +} + +func TestManualRetryMySQLSingleSetManagerSameRun(t *testing.T) { + for _, enforce := range []bool{false, true} { + for _, mode := range []entity.ExptRunMode{entity.EvaluationModeFailRetry, entity.EvaluationModeRetryItems} { + for _, zeroVersion := range []bool{false, true} { + name := "legacy" + if enforce { + name = "enforce" + } + t.Run(fmt.Sprintf("%s/mode=%d/zero-version=%t", name, mode, zeroVersion), func(t *testing.T) { + f := newManualRetryFixture(t) + prepareSingleSetFixture(t, f, enforce, zeroVersion) + before := f.snapshot() + var beforeItems []model.ExptItemResultRunLog + var beforeTurns []model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Order("id").Find(&beforeItems).Error) + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Order("id").Find(&beforeTurns).Error) + manager := singleSetRetryManager(t, f, f.port, enforce) + req := &entity.RetryInCurrentRunRequest{SpaceID: f.space, ExptID: f.expt, ExpectedRunID: f.run, RequestID: "9ef59b9a-fbdb-4c9a-a027-8c969493aaf3", Mode: mode} + if mode == entity.EvaluationModeRetryItems { + req.ItemIDs = []int64{13, 11, 12, 11} + } + receipt, err := manager.RetryInCurrentRun(context.Background(), req) + require.NoError(t, err, "single_set current items have frozen bindings without expt_item_ref") + require.Equal(t, int32(1), receipt.AcceptedCount) + if mode == entity.EvaluationModeRetryItems { + require.Equal(t, []*entity.ManualRetryRejectedItem{{ItemID: 12, Reason: "processing"}, {ItemID: 13, Reason: "queueing"}}, receipt.RejectedItems) + } + after := f.snapshot() + for _, table := range []string{"experiment", "expt_item_ref", "eval_target_record", "evaluator_record"} { + require.Equal(t, before[table], after[table], table) + } + var items []model.ExptItemResultRunLog + var turns []model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Order("id").Find(&items).Error) + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Order("id").Find(&turns).Error) + require.Len(t, items, len(beforeItems)) + require.Len(t, turns, len(beforeTurns)) + for i := range items { + require.Equal(t, beforeItems[i].ID, items[i].ID) + require.Equal(t, beforeItems[i].RetryTimes, items[i].RetryTimes) + if items[i].ItemID != 11 { + require.Equal(t, beforeItems[i], items[i]) + } else { + require.Zero(t, items[i].Status) + require.Zero(t, gptr.Indirect(items[i].ResultState)) + } + } + for i := range turns { + require.Equal(t, beforeTurns[i].ID, turns[i].ID) + if mode == entity.EvaluationModeFailRetry && turns[i].TurnID == 22 { + // The original writer rebinds expt_run_id and refreshes its + // timestamp even when the run is unchanged. Payload stays intact. + expected := beforeTurns[i] + expected.UpdatedAt = turns[i].UpdatedAt + require.Equal(t, expected, turns[i], "successful whole turn payload stays intact") + continue + } + var kept entity.EvaluatorResults + require.NoError(t, json.Unmarshal(gptr.Indirect(turns[i].EvaluatorResultIds), &kept)) + if mode == entity.EvaluationModeFailRetry { + require.Equal(t, f.base+521, turns[i].TargetResultID) + require.Equal(t, map[int64]int64{51: f.base + 1061}, kept.EvalVerIDToResID, "only E2 is eligible for a fresh call") + } else { + require.Zero(t, turns[i].TargetResultID) + require.Empty(t, kept.EvalVerIDToResID) + } + } + replay, err := manager.RetryInCurrentRun(context.Background(), req) + require.NoError(t, err) + expectedJSON, err := json.Marshal(receipt) + require.NoError(t, err) + replayedJSON, err := json.Marshal(replay) + require.NoError(t, err) + require.JSONEq(t, string(expectedJSON), string(replayedJSON)) + require.Equal(t, after, f.snapshot(), "receipt duplicate performs no second reset") + t.Log("REAL_MYSQL_SINGLE_SET_MANAGER_VERIFIED sameRun rowIDs reused turn0 refsAbsent target/E1 preserved receipt replay") + }) + } + } + } +} + +func TestManualRetryMySQLSingleSetFrozenSourceDrift(t *testing.T) { + for _, boundary := range []string{"preflight", "commit"} { + for _, column := range []string{"eval_set_id", "eval_set_version_id", "eval_set_space_id", "eval_set_source_type", "target_id", "target_version_id", "target_space_id", "target_type"} { + t.Run(boundary+"/"+column, func(t *testing.T) { + f := newManualRetryFixture(t) + if !f.conn.Migrator().HasColumn(&model.Experiment{}, column) { + t.Skip("local fixture schema lacks " + column + "; source-field comparison is covered by the locked-source unit test") + } + prepareSingleSetFixture(t, f, false, false) + port := &singleSetRetryPort{IExptManualRetryRepo: f.port} + var before map[string]string + drift := func() { + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).UpdateColumn(column, gorm.Expr(column+"+1")).Error) + before = f.snapshot() + } + if boundary == "preflight" { + port.beforePreflight = drift + } else { + port.beforeCommit = drift + } + manager := singleSetRetryManager(t, f, port, false) + receipt, err := manager.RetryInCurrentRun(context.Background(), &entity.RetryInCurrentRunRequest{SpaceID: f.space, ExptID: f.expt, ExpectedRunID: f.run, RequestID: "9ef59b9a-fbdb-4c9a-a027-8c969493aaf3", Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{11}}) + require.NoError(t, err) + require.Zero(t, receipt.AcceptedCount) + require.Equal(t, []*entity.ManualRetryRejectedItem{{ItemID: 11, Reason: "source_changed"}}, receipt.RejectedItems) + after := f.snapshot() + for table, original := range before { + if table != "expt_run_log" { + require.Equal(t, original, after[table], table) + } + } + t.Log("FROZEN_SOURCE_DRIFT_REJECTED boundary=" + boundary + " column=" + column) + }) + } + } +} + +func TestManualRetryMySQLSingleSetPreparationFences(t *testing.T) { + for _, kind := range []string{"multi_set_missing_ref", "draft_zero", "draft_set_id", "missing_set", "item_log_version", "turn_version", "turn_log_version", "not_resulted"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + prepareSingleSetFixture(t, f, false, true) + switch kind { + case "multi_set_missing_ref": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Update("eval_set_source_type", 2).Error) + case "draft_zero": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Update("eval_set_version_id", 0).Error) + case "draft_set_id": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Update("eval_set_version_id", f.base+31).Error) + case "missing_set": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Update("eval_set_id", 0).Error) + case "item_log_version": + require.NoError(t, f.conn.Table("expt_item_result_run_log").Where("space_id=? AND item_id=?", f.space, 11).Update("item_version_id", 1).Error) + case "turn_version": + require.NoError(t, f.conn.Table("expt_turn_result").Where("space_id=? AND item_id=?", f.space, 11).Update("item_version_id", 1).Error) + case "turn_log_version": + require.NoError(t, f.conn.Table("expt_turn_result_run_log").Where("space_id=? AND item_id=?", f.space, 11).Update("item_version_id", 1).Error) + case "not_resulted": + require.NoError(t, f.conn.Table("expt_item_result_run_log").Where("space_id=? AND item_id=?", f.space, 11).Update("result_state", 0).Error) + } + before := f.snapshot() + manager := singleSetRetryManager(t, f, f.port, false) + receipt, err := manager.RetryInCurrentRun(context.Background(), &entity.RetryInCurrentRunRequest{SpaceID: f.space, ExptID: f.expt, ExpectedRunID: f.run, RequestID: "9ef59b9a-fbdb-4c9a-a027-8c969493aaf3", Mode: entity.EvaluationModeRetryItems, ItemIDs: []int64{11}}) + if kind == "not_resulted" { + require.NoError(t, err) + require.Zero(t, receipt.AcceptedCount) + require.Equal(t, "result_not_ready", receipt.RejectedItems[0].Reason) + before["expt_run_log"] = f.snapshot()["expt_run_log"] + } else { + require.Error(t, err) + require.Nil(t, receipt) + if kind == "multi_set_missing_ref" { + require.ErrorContains(t, err, "record not found") + } + } + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestManualRetryMySQLSingleSetConcurrentReplay(t *testing.T) { + f := newManualRetryFixture(t) + prepareSingleSetFixture(t, f, false, true) + port := &singleSetRetryPort{IExptManualRetryRepo: f.port} + manager := singleSetRetryManager(t, f, port, false) + // Capture the real manager-prepared source before acceptance. Concurrent SQL + // acceptors then contend on the actual experiment/run/item locks and receipt. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + port.beforeCommit = cancel + _, err := manager.RetryInCurrentRun(ctx, &entity.RetryInCurrentRunRequest{SpaceID: f.space, ExptID: f.expt, ExpectedRunID: f.run, RequestID: "9ef59b9a-fbdb-4c9a-a027-8c969493aaf3", Mode: entity.EvaluationModeFailRetry}) + require.ErrorIs(t, err, context.Canceled) + require.NotNil(t, port.prepared) + before := f.snapshot() + var wg sync.WaitGroup + var receipts [2]*entity.ManualRetryReceipt + var errs [2]error + start := make(chan struct{}) + for i := range receipts { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + receipts[i], errs[i] = f.port.RequeueTerminalItems(context.Background(), port.prepared) + }(i) + } + close(start) + wg.Wait() + for i := range receipts { + require.NoError(t, errs[i]) + require.Equal(t, int32(1), receipts[i].AcceptedCount) + } + a, err := json.Marshal(receipts[0]) + require.NoError(t, err) + b, err := json.Marshal(receipts[1]) + require.NoError(t, err) + require.JSONEq(t, string(a), string(b)) + after := f.snapshot() + require.Equal(t, before["expt_item_ref"], after["expt_item_ref"]) + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).First(&stats).Error) + require.Equal(t, int32(2), stats.PendingCnt) + require.Zero(t, stats.FailCnt) + t.Log("REAL_MYSQL_SINGLE_SET_CONCURRENT_REPLAY one atomic reset/stat delta/receipt") +} + +func TestManualRetryMySQLSingleSetWriteRollback(t *testing.T) { + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_turn_evaluator_result_ref", "expt_stats", "expt_run_log"} { + t.Run(table, func(t *testing.T) { + f := newManualRetryFixture(t) + prepareSingleSetFixture(t, f, false, false) + before := f.snapshot() + cause := errors.New("single-set atomic write failure") + callback := func(tx *gorm.DB) { + if tx.Statement.Table == table { + _ = tx.AddError(cause) + } + } + name := "single_set_write_failure" + require.NoError(t, f.conn.Callback().Update().After("gorm:update").Register(name, callback)) + require.NoError(t, f.conn.Callback().Delete().After("gorm:delete").Register(name, callback)) + defer func() { + require.NoError(t, f.conn.Callback().Update().Remove(name)) + require.NoError(t, f.conn.Callback().Delete().Remove(name)) + }() + manager := singleSetRetryManager(t, f, f.port, false) + receipt, err := manager.RetryInCurrentRun(context.Background(), &entity.RetryInCurrentRunRequest{SpaceID: f.space, ExptID: f.expt, ExpectedRunID: f.run, RequestID: "9ef59b9a-fbdb-4c9a-a027-8c969493aaf3", Mode: entity.EvaluationModeFailRetry}) + require.ErrorIs(t, err, cause) + require.Nil(t, receipt) + require.Equal(t, before, f.snapshot(), "four tables, refs, stats and receipt roll back together") + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_source.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_source.go new file mode 100644 index 0000000000..2ea48460aa --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_source.go @@ -0,0 +1,51 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "encoding/json" + "reflect" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +// The caller holds the experiment and canonical item row locks in both +// preflight and acceptance. Single-set starts have no stored item refs. +func (d *ExptManualRetryDAOImpl) lockManualRetrySource(ctx context.Context, r *entity.RequeueTerminalItemsRequest, item *lockedManualRetryItem, expt *model.Experiment, opts []db.Option) (string, error) { + if source := item.prepared.SingleSetSource; source != nil { + actual := entity.PreparedRetrySingleSetSource{ + SourceType: entity.ExptEvalSetSourceType(expt.EvalSetSourceType), + EvalSetID: expt.EvalSetID, EvalSetVersionID: expt.EvalSetVersionID, + EvalSetSpaceID: expt.EvalSetSpaceID, TargetID: expt.TargetID, + TargetVersionID: expt.TargetVersionID, TargetSpaceID: expt.TargetSpaceID, + TargetType: entity.EvalTargetType(expt.TargetType), + } + if (actual.SourceType != 0 && actual.SourceType != entity.ExptEvalSetSourceType_SingleSet) || + actual.EvalSetID <= 0 || actual.EvalSetVersionID <= 0 || actual.EvalSetVersionID == actual.EvalSetID || + actual != *source || item.prepared.FrozenItemRef != nil || item.item.ItemVersionID < 0 || + (item.log != nil && item.log.ItemVersionID != item.prepared.ItemVersionID) { + return "source_changed", nil + } + return "", nil + } + // Preserve the original multi-set stored-ref check and its missing-ref error. + frozen, err := NewExptItemRefDAO(d.provider).GetByExptIDAndItemID(ctx, r.SpaceID, r.ExptID, item.item.ItemID, opts...) + if err != nil { + return "", err + } + if frozen.ItemConfig != nil { + var cfg *entity.ExptItemConfig + if err := json.Unmarshal(*frozen.ItemConfig, &cfg); err != nil { + return "", err + } + } + if !reflect.DeepEqual(convert.NewExptItemRefConvertor().PO2DO(frozen), item.prepared.FrozenItemRef) || frozen.ItemVersionID != item.prepared.ItemVersionID { + return "source_changed", nil + } + return "", nil +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_test.go new file mode 100644 index 0000000000..84ed88d87c --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_manual_retry_test.go @@ -0,0 +1,574 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + mysqldriver "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/db" + idgenmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +var retryFixtureSeq atomic.Int64 + +type manualRetryFixture struct { + t *testing.T + provider db.Provider + conn *gorm.DB + port repo.IExptManualRetryRepo + space, expt, run, base int64 +} + +func newManualRetryFixture(t *testing.T) *manualRetryFixture { + t.Helper() + dsn := os.Getenv("EXPT_RETRY_MYSQL_DSN") + if dsn == "" { + t.Skip("EXPT_RETRY_MYSQL_DSN not configured; real MySQL integration not executed") + } + provider, err := db.NewDB(mysqldriver.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + conn := provider.NewSession(context.Background(), db.WithMaster()) + var identity struct { + Version string + Schema string + } + require.NoError(t, conn.Raw("SELECT VERSION() AS version, DATABASE() AS `schema`").Scan(&identity).Error) + require.Equal(t, "expt_running_retry_test", identity.Schema) + t.Logf("REAL_MYSQL_EXECUTED version=%s schema=%s", identity.Version, identity.Schema) + base := int64(860000000000000000) + time.Now().UnixNano()%1000000000000 + retryFixtureSeq.Add(1)*10000 + f := &manualRetryFixture{t: t, provider: provider, conn: conn, space: base + 1, expt: base + 2, run: base + 3, base: base} + t.Cleanup(func() { + for _, table := range []string{"expt_turn_evaluator_result_ref", "expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_item_ref", "expt_run_log", "expt_stats"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id = ? AND expt_id = ?", f.space, f.expt).Error) + } + require.NoError(t, conn.Exec("DELETE FROM experiment WHERE space_id = ? AND id = ?", f.space, f.expt).Error) + for _, table := range []string{"eval_target_record", "evaluator_record"} { + require.NoError(t, conn.Exec("DELETE FROM "+table+" WHERE space_id = ? AND item_id = ?", f.space, 11).Error) + } + sqlDB, err := conn.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + }) + gen := idgenmocks.NewMockIIDGenerator(gomock.NewController(t)) + var inTransaction atomic.Bool + require.NoError(t, conn.Callback().Query().After("gorm:query").Register("track_manual_retry_transaction", func(tx *gorm.DB) { + if _, ok := tx.Statement.ConnPool.(*sql.Tx); ok { + inTransaction.Store(true) + } + })) + gen.EXPECT().GenID(gomock.Any()).DoAndReturn(func(context.Context) (int64, error) { + require.False(t, inTransaction.Load(), "IDGen may use Redis and must run before SQL transaction") + return base + 9000 + retryFixtureSeq.Add(1), nil + }).AnyTimes() + port, ok := any(&ExptManualRetryDAOImpl{provider: provider, idgen: gen}).(repo.IExptManualRetryRepo) + require.True(t, ok, "same-run four-table transaction is missing") + f.port = port + require.NoError(t, conn.Exec("INSERT INTO experiment (id,space_id,latest_run_id,status,expt_type) VALUES (?,?,?,?,?)", f.expt, f.space, f.run, entity.ExptStatus_Processing, entity.ExptType_Offline).Error) + raw := `[{"ItemIDs":[11,12,13],"CreateAt":1700000000,"future":{"id":9223372036854775806}}]` + require.NoError(t, conn.Create(&model.ExptRunLog{ID: f.run, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, Status: gptr.Of(int64(entity.ExptStatus_Processing)), Mode: gptr.Of(int32(entity.EvaluationModeSubmit)), CreatedBy: "original", ItemIds: gptr.Of([]byte(raw)), PendingCnt: 1, ProcessingCnt: 1, FailCnt: 1, CreditCost: 42, TokenCost: gptr.Of(int64(123))}).Error) + require.NoError(t, conn.Create(&model.ExptStats{ID: base + 4, SpaceID: f.space, ExptID: f.expt, PendingCnt: 1, ProcessingCnt: 1, FailCnt: 1, CreditCost: 42, InputTokenCost: gptr.Of(int64(100)), OutputTokenCost: gptr.Of(int64(23))}).Error) + for i, status := range []int32{3, 1, 0} { + item := int64(11 + i) + version := base + 100 + item + require.NoError(t, conn.Create(&model.ExptItemResult{ID: base + 200 + item, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: version, Status: status, ErrMsg: gptr.Of([]byte("old-error")), LogID: "old-item-log", Ext: gptr.Of([]byte(`{"other":"keep","future_id":"9223372036854775806"}`))}).Error) + result := int32(0) + if status == 3 { + result = 1 + } + require.NoError(t, conn.Create(&model.ExptItemResultRunLog{ID: base + 300 + item, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: item, ItemVersionID: version, Status: status, ResultState: gptr.Of(result), RetryTimes: 3, ErrMsg: gptr.Of([]byte("old-run-error")), LogID: "old-item-run"}).Error) + require.NoError(t, conn.Create(&model.ExptItemRef{ID: base + 400 + item, SpaceID: f.space, ExptID: f.expt, ItemID: item, ItemVersionID: version, ItemConfig: gptr.Of([]byte(`{"target_source_space_id":19}`))}).Error) + } + for i, status := range []int32{2, 1} { + turn := int64(21 + i) + target := base + 500 + turn + require.NoError(t, conn.Create(&model.ExptTurnResult{ID: base + 600 + turn, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: 11, ItemVersionID: base + 111, TurnID: turn, Status: status, TargetResultID: target, WeightedScore: gptr.Of(0.0), ErrMsg: gptr.Of([]byte("turn-error")), TraceID: 55, LogID: "turn-log"}).Error) + eval, err := json.Marshal(&entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: base + 1061, 52: base + 1062}}) + require.NoError(t, err) + if status == 1 { + eval, err = json.Marshal(&entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: base + 1063}}) + require.NoError(t, err) + } + require.NoError(t, conn.Create(&model.ExptTurnResultRunLog{ID: base + 700 + turn, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: 11, ItemVersionID: base + 111, TurnID: turn, Status: status, TargetResultID: target, EvaluatorResultIds: &eval, ErrMsg: gptr.Of([]byte("turn-run-error")), TraceID: 55, LogID: "turn-run-log", Ext: []byte(`{"other":"keep"}`)}).Error) + for j, record := range []int64{base + 1061, base + 1062} { + if status == 1 && j == 1 { + continue + } + if status == 1 { + record = base + 1063 + } + require.NoError(t, conn.Create(&model.ExptTurnEvaluatorResultRef{ID: base + 800 + turn*10 + int64(j), SpaceID: f.space, ExptID: f.expt, ExptTurnResultID: base + 600 + turn, EvaluatorVersionID: 51 + int64(j), EvaluatorResultID: record}).Error) + } + } + for _, turn := range []int64{21, 22} { + require.NoError(t, conn.Exec("INSERT INTO eval_target_record (id,space_id,target_id,target_version_id,experiment_run_id,item_id,item_version_id,turn_id,log_id,trace_id,status) VALUES (?,?,?,?,?,?,?,?,?,?,?)", base+500+turn, f.space, base+41, base+42, f.run, 11, base+111, turn, "target-log", "trace", 1).Error) + } + for i, record := range []int64{base + 1061, base + 1062, base + 1063} { + turn := int64(21) + if i == 2 { + turn = 22 + } + status := 1 + if i == 1 { + status = 2 + } + require.NoError(t, conn.Exec("INSERT INTO evaluator_record (id,space_id,evaluator_version_id,experiment_id,experiment_run_id,item_id,item_version_id,turn_id,trace_id,status,score,target_record_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", record, f.space, 51+int64(i%2), f.expt, f.run, 11, base+111, turn, "evaluator-trace", status, nil, 0).Error) + } + return f +} + +func (f *manualRetryFixture) snapshot() map[string]string { + f.t.Helper() + result := make(map[string]string) + for _, table := range []string{"experiment", "expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_turn_evaluator_result_ref", "expt_stats", "expt_run_log", "expt_item_ref"} { + idField := "expt_id" + if table == "experiment" { + idField = "id" + } + rows, err := f.conn.Raw("SELECT * FROM "+table+" WHERE space_id = ? AND "+idField+" = ? ORDER BY id", f.space, f.expt).Rows() + require.NoError(f.t, err) + columns, err := rows.Columns() + require.NoError(f.t, err) + var values [][]string + for rows.Next() { + cells := make([]any, len(columns)) + ptr := make([]any, len(cells)) + for i := range cells { + ptr[i] = &cells[i] + } + require.NoError(f.t, rows.Scan(ptr...)) + converted := make([]string, len(cells)) + for i, cell := range cells { + if b, ok := cell.([]byte); ok { + converted[i] = string(b) + } else { + b, err := json.Marshal(cell) + require.NoError(f.t, err) + converted[i] = string(b) + } + } + values = append(values, converted) + } + require.NoError(f.t, rows.Err()) + require.NoError(f.t, rows.Close()) + encoded, err := json.Marshal(values) + require.NoError(f.t, err) + result[table] = string(encoded) + } + for _, table := range []string{"eval_target_record", "evaluator_record"} { + var values []map[string]any + require.NoError(f.t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, 11).Order("id").Find(&values).Error) + encoded, err := json.Marshal(values) + require.NoError(f.t, err) + result[table] = string(encoded) + } + return result +} + +func (f *manualRetryFixture) request(mode entity.ExptRunMode) *entity.RequeueTerminalItemsRequest { + f.t.Helper() + ctx := context.Background() + itemRows, err := NewExptItemResultDAO(f.provider).BatchGet(ctx, f.space, f.expt, []int64{11}) + require.NoError(f.t, err) + source := convert.NewExptItemResultConvertor().PO2DO(itemRows[0]) + source.Ext, err = entity.ParseExptItemResultExt(gptr.Indirect(itemRows[0].Ext)) + require.NoError(f.t, err) + itemLog, err := NewExptItemResultDAO(f.provider).GetItemRunLog(ctx, f.expt, f.run, 11, f.space) + require.NoError(f.t, err) + ref, err := NewExptItemRefDAO(f.provider).GetByExptIDAndItemID(ctx, f.space, f.expt, 11) + require.NoError(f.t, err) + prepared := &entity.PreparedRetryItem{PreviousRunID: source.ExptRunID, SourceItem: source, ItemVersionID: source.ItemVersionID, CurrentItemLog: convert.NewExptItemResultRunLogConverter().PO2DO(itemLog), FrozenItemRef: convert.NewExptItemRefConvertor().PO2DO(ref)} + previous, err := entity.ParseManualRetryContext(source.Ext) + require.NoError(f.t, err) + if previous != nil { + prepared.PreviousRequestID = previous.RequestID + } + turns, err := NewExptTurnResultDAO(f.provider).GetItemTurnResults(ctx, f.expt, 11, f.space) + require.NoError(f.t, err) + logs, err := NewExptTurnResultDAO(f.provider).GetItemTurnRunLogs(ctx, f.expt, f.run, 11, f.space) + require.NoError(f.t, err) + for _, turn := range turns { + plan := &entity.PreparedRetryTurn{SourceTurn: convert.NewExptTurnResultConvertor().PO2DO(turn, nil)} + for _, log := range logs { + if log.TurnID == turn.TurnID { + plan.CurrentRunLog, err = convert.NewExptTurnResultRunLogConvertor().PO2DO(log) + require.NoError(f.t, err) + } + } + refs, err := NewExptTurnEvaluatorResultRefDAO(f.provider).BatchGet(ctx, f.space, []int64{turn.ID}) + require.NoError(f.t, err) + for _, ref := range refs { + plan.SourceRefs = append(plan.SourceRefs, convert.NewExptTurnEvaluatorResultRefConvertor().PO2DO(ref)) + } + if mode == entity.EvaluationModeFailRetry { + plan.TargetResultID = turn.TargetResultID + plan.PreserveSuccess = turn.Status == 1 + if plan.PreserveSuccess { + plan.EvaluatorResults = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: f.base + 1063}} + } else { + plan.EvaluatorResults = &entity.EvaluatorResults{EvalVerIDToResID: map[int64]int64{51: f.base + 1061}} + } + for _, ref := range plan.SourceRefs { + if ref.EvaluatorResultID == f.base+1061 || plan.PreserveSuccess { + plan.KeepRefIDs = append(plan.KeepRefIDs, ref.ID) + } + } + } + prepared.Turns = append(prepared.Turns, plan) + } + return &entity.RequeueTerminalItemsRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, RequestID: "rq1", PayloadHash: "hash1", Operator: "operator", Mode: mode, SnapshotAtMS: 1700000000123, Items: []*entity.PreparedRetryItem{prepared}} +} + +func TestManualRetryMySQLReceiptUnknownRun(t *testing.T) { + f := newManualRetryFixture(t) + before := f.snapshot() + ctx := context.Background() + run, err := NewExptRunLogDAO(f.provider).Get(ctx, f.expt, f.run+1, db.WithMaster()) + require.Nil(t, run) + require.ErrorIs(t, err, gorm.ErrRecordNotFound) + require.NotEqual(t, gorm.ErrRecordNotFound, err, "run DAO wraps the missing-row error") + t.Log("WRAPPED_NOT_FOUND_VERIFIED errors.Is=true direct_equality=false") + receipt, err := f.port.FindReceipt(ctx, f.space, f.expt, f.run+1, "unknown-run", "hash") + require.Equal(t, before, f.snapshot(), "receipt lookup must not write any fixture data") + require.NoError(t, err, "an unknown run has no receipt; the manager must validate the expected run") + require.Nil(t, receipt) +} + +func TestManualRetryMySQLReceiptQueryErrorRetainsCause(t *testing.T) { + f := newManualRetryFixture(t) + before := f.snapshot() + // Matching text is not the GORM sentinel and must remain a database error. + cause := errors.New("record not found") + name := "manual_retry_receipt_query_error" + require.NoError(t, f.conn.Callback().Query().After("gorm:query").Register(name, func(tx *gorm.DB) { + if tx.Statement.Table == "expt_run_log" { + require.NoError(t, tx.Error) + require.Equal(t, int64(1), tx.RowsAffected, "inject only after the real run query succeeds") + _ = tx.AddError(cause) + } + })) + defer func() { require.NoError(t, f.conn.Callback().Query().Remove(name)) }() + receipt, err := f.port.FindReceipt(context.Background(), f.space, f.expt, f.run, "query-error", "hash") + require.Nil(t, receipt) + require.ErrorIs(t, err, cause) + require.NotErrorIs(t, err, gorm.ErrRecordNotFound) + require.Equal(t, before, f.snapshot()) +} + +func TestManualRetryMySQLSameRun(t *testing.T) { + f := newManualRetryFixture(t) + before := f.snapshot() + req := f.request(entity.EvaluationModeFailRetry) + receipt, err := f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + require.Equal(t, int32(1), receipt.AcceptedCount) + after := f.snapshot() + require.NotEqual(t, before["expt_item_result"], after["expt_item_result"]) + require.Equal(t, before["experiment"], after["experiment"]) + require.Equal(t, before["eval_target_record"], after["eval_target_record"]) + require.Equal(t, before["evaluator_record"], after["evaluator_record"]) + var item model.ExptItemResult + require.NoError(t, f.conn.Where("space_id = ? AND expt_id = ? AND item_id = ?", f.space, f.expt, 11).First(&item).Error) + require.Zero(t, item.Status) + require.Nil(t, item.ErrMsg) + require.Empty(t, item.LogID) + require.Equal(t, f.base+111, item.ItemVersionID) + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + require.NoError(t, err) + require.Equal(t, "keep", ext["other"]) + require.Equal(t, "9223372036854775806", ext["future_id"]) + retry, err := entity.ParseManualRetryContext(ext) + require.NoError(t, err) + require.Equal(t, entity.EvaluationModeFailRetry, retry.Mode) + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id = ? AND expt_id = ? AND item_id = ?", f.space, f.expt, 11).First(&log).Error) + require.Zero(t, log.Status) + require.Zero(t, *log.ResultState) + require.Zero(t, log.QuotaReservationState) + require.Equal(t, int32(3), log.RetryTimes) + require.Nil(t, log.ErrMsg) + require.Empty(t, log.LogID) + var turns []*model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id = ? AND expt_id = ?", f.space, f.expt).Order("turn_id").Find(&turns).Error) + require.Nil(t, turns[0].WeightedScore) + require.Nil(t, turns[0].ErrMsg) + require.Zero(t, turns[0].TraceID) + require.Empty(t, turns[0].LogID) + require.Equal(t, f.base+521, turns[0].TargetResultID) + require.Equal(t, 0.0, *turns[1].WeightedScore) + require.Equal(t, int32(1), turns[1].Status) + require.Equal(t, "turn-log", turns[1].LogID) + var turnLogs []*model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id = ? AND expt_id = ?", f.space, f.expt).Order("turn_id").Find(&turnLogs).Error) + require.Zero(t, turnLogs[0].Status) + require.Nil(t, turnLogs[0].ErrMsg) + require.Empty(t, turnLogs[0].LogID) + require.Zero(t, turnLogs[0].TraceID) + var kept entity.EvaluatorResults + require.NoError(t, json.Unmarshal(*turnLogs[0].EvaluatorResultIds, &kept)) + require.Equal(t, map[int64]int64{51: f.base + 1061}, kept.EvalVerIDToResID) + require.JSONEq(t, `{"other":"keep"}`, string(turnLogs[0].Ext)) + require.Equal(t, "turn-run-log", turnLogs[1].LogID) + require.Equal(t, int64(55), turnLogs[1].TraceID) + var refs []*model.ExptTurnEvaluatorResultRef + require.NoError(t, f.conn.Where("space_id = ? AND expt_id = ?", f.space, f.expt).Order("id").Find(&refs).Error) + require.Len(t, refs, 2) + require.Equal(t, f.base+1061, refs[0].EvaluatorResultID) + require.Equal(t, f.base+1063, refs[1].EvaluatorResultID) + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id = ? AND expt_id = ?", f.space, f.expt).First(&stats).Error) + require.Equal(t, int32(2), stats.PendingCnt) + require.Equal(t, int32(1), stats.ProcessingCnt) + require.Zero(t, stats.FailCnt) + require.Equal(t, float64(42), stats.CreditCost) + require.Equal(t, int64(100), *stats.InputTokenCost) + var run model.ExptRunLog + require.NoError(t, f.conn.Where("id = ?", f.run).First(&run).Error) + require.Equal(t, int32(2), run.PendingCnt) + require.Equal(t, int32(1), run.ProcessingCnt) + require.Zero(t, run.FailCnt) + require.Equal(t, "original", run.CreatedBy) + require.Equal(t, int32(1), *run.Mode) + var chunks []map[string]json.RawMessage + require.NoError(t, json.Unmarshal(*run.ItemIds, &chunks)) + require.JSONEq(t, `{"id":9223372036854775806}`, string(chunks[0]["future"])) + for _, id := range []int64{12, 13} { + var got model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, id).First(&got).Error) + require.Equal(t, "old-item-log", got.LogID) + require.Equal(t, []byte("old-error"), *got.ErrMsg) + } + replay, err := f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + expectedReceipt, err := json.Marshal(receipt) + require.NoError(t, err) + replayReceipt, err := json.Marshal(replay) + require.NoError(t, err) + require.JSONEq(t, string(expectedReceipt), string(replayReceipt)) + require.Equal(t, after, f.snapshot()) + req.PayloadHash = "different" + _, err = f.port.RequeueTerminalItems(context.Background(), req) + require.ErrorContains(t, err, "request_conflict") + require.Equal(t, after, f.snapshot()) +} + +func TestManualRetryMySQLWriteRollback(t *testing.T) { + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_turn_evaluator_result_ref", "expt_stats", "expt_run_log"} { + t.Run(table, func(t *testing.T) { + f := newManualRetryFixture(t) + req := f.request(entity.EvaluationModeFailRetry) + before := f.snapshot() + hits := 0 + callback := func(tx *gorm.DB) { + if tx.Statement.Table == table { + _, sameTx := tx.Statement.ConnPool.(*sql.Tx) + require.True(t, sameTx, "write must use the Provider transaction connection") + hits++ + _ = tx.AddError(errors.New("injected write failure " + table)) + } + } + name := "manual_retry_fault_" + table + if table == "expt_turn_evaluator_result_ref" { + require.NoError(t, f.conn.Callback().Delete().Before("gorm:delete").Register(name, callback)) + defer func() { require.NoError(t, f.conn.Callback().Delete().Remove(name)) }() + } else { + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register(name, callback)) + defer func() { require.NoError(t, f.conn.Callback().Update().Remove(name)) }() + } + _, err := f.port.RequeueTerminalItems(context.Background(), req) + require.ErrorContains(t, err, "injected write failure") + require.Positive(t, hits) + require.True(t, reflect.DeepEqual(before, f.snapshot()), "all four tables + refs + stats + receipt must roll back") + t.Logf("REAL_MYSQL_ROLLBACK_VERIFIED table=%s snapshots=11", table) + }) + } +} + +func TestManualRetryMySQLBoundaries(t *testing.T) { + for _, kind := range []string{"logged", "source_changed", "capacity", "manual_whole_row", "zero_accepted", "finished_replay", "replaced_replay"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + mode := entity.EvaluationModeFailRetry + if kind == "manual_whole_row" { + mode = entity.EvaluationModeRetryItems + } + req := f.request(mode) + switch kind { + case "logged": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("result_state", 2).Error) + case "source_changed": + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=? AND expt_id=? AND turn_id=?", f.space, f.expt, 21).Update("target_result_id", 991).Error) + case "capacity": + raw := []byte(`[{"ItemIDs":[11,12,13],"future":"` + strings.Repeat("x", 65450) + `"}]`) + require.Less(t, len(raw), 65535) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("id=?", f.run).Update("item_ids", raw).Error) + case "zero_accepted": + req.Items = nil + req.RejectedItems = []*entity.ManualRetryRejectedItem{{ItemID: 12, Reason: "processing"}, {ItemID: 13, Reason: "queueing"}} + } + before := f.snapshot() + receipt, err := f.port.RequeueTerminalItems(context.Background(), req) + if kind == "capacity" { + require.ErrorContains(t, err, "capacity") + require.Equal(t, before, f.snapshot()) + return + } + require.NoError(t, err) + switch kind { + case "logged", "source_changed", "zero_accepted": + require.Zero(t, receipt.AcceptedCount) + if kind != "zero_accepted" { + want := "source_changed" + if kind == "logged" { + want = "result_not_ready" + } + require.Equal(t, want, receipt.RejectedItems[0].Reason) + } + after := f.snapshot() + for table, snapshot := range before { + if table != "expt_run_log" { + require.Equal(t, snapshot, after[table], table) + } + } + _, err = f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + require.Equal(t, after, f.snapshot()) + case "manual_whole_row": + var turns []*model.ExptTurnResult + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Find(&turns).Error) + for _, turn := range turns { + require.Zero(t, turn.Status) + require.Zero(t, turn.TargetResultID) + require.Nil(t, turn.WeightedScore) + } + var logs []*model.ExptTurnResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Find(&logs).Error) + for _, log := range logs { + require.Zero(t, log.TargetResultID) + require.JSONEq(t, `null`, string(*log.EvaluatorResultIds)) + require.JSONEq(t, `{"other":"keep"}`, string(log.Ext)) + } + var refs int64 + require.NoError(t, f.conn.Model(&model.ExptTurnEvaluatorResultRef{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Count(&refs).Error) + require.Zero(t, refs) + case "finished_replay", "replaced_replay": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Update("status", int32(entity.ExptStatus_Success)).Error) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("id=?", f.run).Update("status", int64(entity.ExptStatus_Success)).Error) + if kind == "replaced_replay" { + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("id=?", f.expt).Updates(map[string]any{"latest_run_id": f.run + 1, "status": int32(entity.ExptStatus_Processing)}).Error) + } + after := f.snapshot() + expectedReceipt, err := json.Marshal(receipt) + require.NoError(t, err) + replay, err := f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + replayedReceipt, err := json.Marshal(replay) + require.NoError(t, err) + require.JSONEq(t, string(expectedReceipt), string(replayedReceipt)) + require.Equal(t, after, f.snapshot()) + found, err := f.port.FindReceipt(context.Background(), f.space, f.expt, f.run, req.RequestID, req.PayloadHash) + require.NoError(t, err) + foundReceipt, err := json.Marshal(found) + require.NoError(t, err) + require.JSONEq(t, string(expectedReceipt), string(foundReceipt)) + require.Equal(t, after, f.snapshot()) + } + }) + } +} + +func TestManualRetryMySQLConcurrentReplay(t *testing.T) { + f := newManualRetryFixture(t) + req := f.request(entity.EvaluationModeFailRetry) + results := make(chan *entity.ManualRetryReceipt, 2) + failures := make(chan error, 2) + var group sync.WaitGroup + for i := 0; i < 2; i++ { + group.Add(1) + go func() { + defer group.Done() + r, err := f.port.RequeueTerminalItems(context.Background(), req) + results <- r + failures <- err + }() + } + group.Wait() + close(results) + close(failures) + for err := range failures { + require.NoError(t, err) + } + for r := range results { + require.Equal(t, int32(1), r.AcceptedCount) + } + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).First(&stats).Error) + require.Equal(t, int32(2), stats.PendingCnt) + require.Zero(t, stats.FailCnt) + var run model.ExptRunLog + require.NoError(t, f.conn.Where("id=?", f.run).First(&run).Error) + var chunks []entity.ExptRunLogItems + require.NoError(t, json.Unmarshal(*run.ItemIds, &chunks)) + require.Len(t, chunks, 2) + t.Log("REAL_MYSQL_CONCURRENT_REPLAY_VERIFIED same request one write set") +} + +func TestManualRetryMySQLCorruptionNoReceipt(t *testing.T) { + for _, kind := range []string{"item_ext", "run_metadata", "turn_ext", "turn_results", "frozen_config", "source_trace"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + req := f.request(entity.EvaluationModeFailRetry) + switch kind { + case "item_ext": + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("ext", []byte(`{"manual_retry":"broken"}`)).Error) + case "run_metadata": + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("id=?", f.run).Update("item_ids", []byte(`{`)).Error) + case "turn_ext": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND expt_id=? AND turn_id=?", f.space, f.expt, 21).Update("ext", []byte(`{"other":42}`)).Error) + case "turn_results": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND expt_id=? AND turn_id=?", f.space, f.expt, 21).Update("evaluator_result_ids", []byte(`{`)).Error) + case "source_trace": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND expt_id=? AND turn_id=?", f.space, f.expt, 21).Update("trace_id", 99).Error) + before := f.snapshot() + receipt, err := f.port.RequeueTerminalItems(context.Background(), req) + require.NoError(t, err) + require.Zero(t, receipt.AcceptedCount) + require.Equal(t, "source_changed", receipt.RejectedItems[0].Reason) + after := f.snapshot() + for table, raw := range before { + if table != "expt_run_log" { + require.Equal(t, raw, after[table], table) + } + } + return + case "frozen_config": + require.NoError(t, f.conn.Model(&model.ExptItemRef{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, 11).Update("item_config", []byte(`{`)).Error) + } + before := f.snapshot() + _, err := f.port.RequeueTerminalItems(context.Background(), req) + require.Error(t, err) + require.Equal(t, before, f.snapshot()) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_ndb_observation_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_ndb_observation_test.go new file mode 100644 index 0000000000..52dd749d83 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_ndb_observation_test.go @@ -0,0 +1,258 @@ +// Copyright (c) 2026 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// Only Begin options are simulated; all data queries use the actual MySQL driver. +// This cannot certify production NDB acceptance. +func TestNDBObservationMySQLDefaultBegin(t *testing.T) { + t.Run("scan", func(t *testing.T) { + f := newCurrentResultFixture(t, "") + provider := newObservationProvider(t, "READ-COMMITTED") + provider.rejectExplicit = true + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=?", f.space).Update("status", int32(entity.ItemRunState_Queueing)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=?", f.space).Update("status", int32(entity.ItemRunState_Queueing)).Error) + before := f.snapshot() + rows, err := NewExptItemResultDAO(provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + require.NoError(t, err, "default-off legacy queue must not fail at NDB Begin") + require.Len(t, rows, 3) + for _, row := range rows { + require.Equal(t, entity.ItemRunState_Queueing, row.Status) + require.Empty(t, row.RequestID) + } + require.Equal(t, before, f.snapshot(), "observers must leave all eleven tables unchanged") + }) +} + +type readonlyObservationProbe struct { + writes, locks, recordLocks int + queries []string +} + +func observeReadonlySQL(t *testing.T, conn *gorm.DB) *readonlyObservationProbe { + t.Helper() + p := &readonlyObservationProbe{} + write := func(*gorm.DB) { p.writes++ } + require.NoError(t, conn.Callback().Create().Before("gorm:create").Register("observation_create", write)) + require.NoError(t, conn.Callback().Update().Before("gorm:update").Register("observation_update", write)) + require.NoError(t, conn.Callback().Delete().Before("gorm:delete").Register("observation_delete", write)) + require.NoError(t, conn.Callback().Raw().Before("gorm:raw").Register("observation_raw", func(tx *gorm.DB) { + if !strings.HasPrefix(strings.ToUpper(strings.TrimSpace(tx.Statement.SQL.String())), "SELECT") { + p.writes++ + } + })) + require.NoError(t, conn.Callback().Query().After("gorm:query").Register("observation_query", func(tx *gorm.DB) { + query := strings.ToUpper(tx.Statement.SQL.String()) + p.queries = append(p.queries, query) + if strings.Contains(query, "FOR UPDATE") || strings.Contains(query, "FOR SHARE") || strings.Contains(query, "LOCK IN SHARE") { + p.locks++ + } + var count int + require.NoError(t, tx.Session(&gorm.Session{NewDB: true}).Raw("SELECT COUNT(*) FROM performance_schema.data_locks WHERE LOCK_TYPE='RECORD' AND THREAD_ID=(SELECT THREAD_ID FROM performance_schema.threads WHERE PROCESSLIST_ID=CONNECTION_ID())").Scan(&count).Error) + if count > p.recordLocks { + p.recordLocks = count + } + })) + t.Cleanup(func() { + require.NoError(t, conn.Callback().Create().Remove("observation_create")) + require.NoError(t, conn.Callback().Update().Remove("observation_update")) + require.NoError(t, conn.Callback().Delete().Remove("observation_delete")) + require.NoError(t, conn.Callback().Raw().Remove("observation_raw")) + require.NoError(t, conn.Callback().Query().Remove("observation_query")) + }) + return p +} + +func (p *readonlyObservationProbe) assert(t *testing.T, f *currentResultFixture, expected map[string]string) { + t.Helper() + require.Zero(t, p.writes, "observer emitted DML/session statements") + require.Zero(t, p.locks, "observer used locking SELECT") + require.Zero(t, p.recordLocks, "actual observer transaction acquired record locks") + require.NotEmpty(t, p.queries) + for _, query := range p.queries { + require.True(t, strings.HasPrefix(query, "SELECT"), query) + require.NotContains(t, query, "EVAL_TARGET_RECORD") + require.NotContains(t, query, "EVALUATOR_RECORD") + } + require.Equal(t, expected, f.snapshot(), "observer changed nine source/accounting tables or two record tables") + t.Logf("REAL_MYSQL_READONLY queries=%d writes=%d locking_selects=%d record_locks=%d snapshots=11 records_unchanged=true", len(p.queries), p.writes, p.locks, p.recordLocks) +} + +func TestNDBObservationMySQLFullSourceChanges(t *testing.T) { + changes := []string{"latest", "new_run", "raw_manual", "same_request_new_record", "registered", "inline", "physical_ref", "canonical_terminal", "runlog_processing", "run_terminal", "turn_status", "item_config", "raw_ext", "retry_cycle"} + for _, isolation := range []string{"READ-COMMITTED", "REPEATABLE-READ"} { + for _, change := range changes { + t.Run(isolation+"/scan/"+change, func(t *testing.T) { + f := newScanObservationFixture(t) + if change == "raw_manual" { + f.token("") + } + if change == "retry_cycle" { + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Processing)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Processing), "result_state": int32(entity.ExptItemResultStateDefault)}).Error) + } + expectedRows, err := NewExptItemResultDAO(f.provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + require.NoError(t, err) + provider := newObservationProvider(t, isolation) + conn := provider.NewSession(context.Background()) + probe := observeReadonlySQL(t, conn) + expectedSnapshot := f.snapshot() + var once sync.Once + // End of the complete first collection: all old source values + // already exist, before the inner observer queries start. + require.NoError(t, conn.Callback().Query().After("observation_query").Register("observation_change", func(tx *gorm.DB) { + if tx.Statement.Table != "expt_turn_evaluator_result_ref" { + return + } + once.Do(func() { + switch change { + case "latest": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("latest_run_id", f.run+1).Error) + case "new_run": + require.NoError(t, f.provider.Transaction(context.Background(), func(writer *gorm.DB) error { + if err := writer.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("latest_run_id", f.run+1).Error; err != nil { + return err + } + return writer.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("expt_run_id", f.run+1).Error + })) + case "raw_manual": + f.token("rq2") + case "same_request_new_record": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=? AND turn_id=0", f.space, f.item).Update("target_result_id", f.base+99).Error) + case "registered", "inline": + refs := &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: f.base + 61}}, Inline: []*entity.InlineEvalResult{{InlineKey: "inline", RecordID: f.base + 62}}} + if change == "registered" { + refs.Registered[0].RecordID++ + } else { + refs.Inline[0].RecordID++ + } + raw, err := json.Marshal(refs) + require.NoError(t, err) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=? AND turn_id=0", f.space, f.item).Update("evaluator_result_ids", raw).Error) + case "physical_ref": + require.NoError(t, f.conn.Model(&model.ExptTurnEvaluatorResultRef{}).Where("space_id=?", f.space).Update("evaluator_result_id", f.base+99).Error) + case "canonical_terminal": + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Terminal)).Error) + case "runlog_processing": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Processing)).Error) + case "run_terminal": + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + case "turn_status": + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=? AND turn_id=0", f.space, f.item).Update("status", int32(entity.TurnRunState_Terminal)).Error) + case "item_config": + require.NoError(t, f.conn.Model(&model.ExptItemRef{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_config", []byte(`{"target_source_space_id":99}`)).Error) + case "raw_ext": + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("ext", []byte(`{"other":"changed"}`)).Error) + case "retry_cycle": + // Retry state returns to Processing, but monotonic retry + // count remains evidence of the new observation generation. + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("retry_times", 2).Error) + } + expectedSnapshot = f.snapshot() + }) + })) + defer func() { require.NoError(t, conn.Callback().Query().Remove("observation_change")) }() + rows, err := NewExptItemResultDAO(provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + if isolation == "READ-COMMITTED" { + require.Error(t, err) + require.Nil(t, rows) + } else { + require.NoError(t, err) + require.Equal(t, expectedRows, rows) + } + probe.assert(t, f, expectedSnapshot) + }) + } + } +} + +// After its first value was observed, a config mutation must invalidate the +// complete RC observation rather than return any partially collected source. +func TestNDBObservationMySQLChangedSourceDiscardsAll(t *testing.T) { + t.Run("scan", func(t *testing.T) { + f := newScanObservationFixture(t) + provider := newObservationProvider(t, "READ-COMMITTED") + conn := provider.NewSession(context.Background()) + var once sync.Once + require.NoError(t, conn.Callback().Query().After("gorm:query").Register("change_after_first_config", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_ref" { + once.Do(func() { + require.NoError(t, f.conn.Model(&model.ExptItemRef{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_config", []byte(`{"target_source_space_id":99}`)).Error) + }) + } + })) + defer func() { require.NoError(t, conn.Callback().Query().Remove("change_after_first_config")) }() + rows, err := NewExptItemResultDAO(provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + require.Error(t, err, "RC change must fail closed without partial scan rows") + require.Nil(t, rows) + }) +} + +func TestNDBObservationMySQLStableFullSource(t *testing.T) { + for _, isolation := range []string{"READ-COMMITTED", "REPEATABLE-READ"} { + for _, kind := range []string{"logged", "resulted_finished", "raw_ref0_version0"} { + t.Run(isolation+"/"+kind, func(t *testing.T) { + f := newScanObservationFixture(t) + if kind == "resulted_finished" { + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("result_state", int32(entity.ExptItemResultStateResulted)).Error) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + for _, table := range []string{"expt_turn_result", "expt_turn_result_run_log"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.TurnRunState_Success)).Error) + } + } + if kind == "raw_ref0_version0" { + f.token("") + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_item_ref", "eval_target_record", "evaluator_record"} { + require.NoError(t, f.conn.Table(table).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_version_id", 0).Error) + } + require.NoError(t, f.conn.Model(&model.ExptTurnEvaluatorResultRef{}).Where("space_id=?", f.space).Update("source_type", 0).Error) + } + expectedRows, err := NewExptItemResultDAO(f.provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, true) + require.NoError(t, err) + before := f.snapshot() + provider := newObservationProvider(t, isolation) + provider.rejectExplicit = true + probe := observeReadonlySQL(t, provider.NewSession(context.Background())) + rows, err := NewExptItemResultDAO(provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, true) + require.NoError(t, err) + require.Equal(t, expectedRows, rows) + if kind == "raw_ref0_version0" { + require.Len(t, rows, 3) + require.NotNil(t, rows[0].Publication) + require.Empty(t, rows[0].Publication.Source.Current.RequestID) + require.Zero(t, rows[0].Publication.Source.Current.ItemVersionID) + require.Zero(t, rows[0].Publication.Source.CurrentRefs[0].SourceType) + } + probe.assert(t, f, before) + }) + } + } +} + +func TestNDBObservationMySQLOldLatestRunRejected(t *testing.T) { + for _, isolation := range []string{"READ-COMMITTED", "REPEATABLE-READ"} { + t.Run(isolation, func(t *testing.T) { + f := newCurrentResultFixture(t, "") + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("latest_run_id", f.run+1).Error) + provider := newObservationProvider(t, isolation) + before := f.snapshot() + probe := observeReadonlySQL(t, provider.NewSession(context.Background())) + rows, err := NewExptItemResultDAO(provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + require.NoError(t, err) + require.Empty(t, rows, "old queue/run logs must not dispatch after the authoritative latest run moved") + probe.assert(t, f, before) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_read_observation.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_read_observation.go new file mode 100644 index 0000000000..cc8ee3f1cc --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_read_observation.go @@ -0,0 +1,86 @@ +// Copyright (c) 2026 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "fmt" + "reflect" + + "gorm.io/gorm" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +// Keep raw persisted rows: converted sources intentionally omit fields and +// normalize JSON, so their execution/CAS equality is too weak for RC reads. +// This observes only SELECTs, never session settings or locking clauses. +type itemReadObservation struct { + Items []model.ExptItemResult + ItemLogs []model.ExptItemResultRunLog + Turns []model.ExptTurnResult + TurnLogs []model.ExptTurnResultRunLog + Refs []model.ExptTurnEvaluatorResultRef + ItemRefs []model.ExptItemRef + Runs []model.ExptRunLog + Experiment []model.Experiment +} + +// The scheduler observes the complete current run scan. +// Collection order is identical on both passes. The complete item/turn/ref +// membership is reread so a new row cannot silently enter the frozen source. +func collectItemReadObservation(tx *gorm.DB, space, expt, run int64, frozen *itemReadObservation) (*itemReadObservation, error) { + o := &itemReadObservation{} + items := tx.Where("space_id=? AND expt_id=? AND expt_run_id=?", space, expt, run) + if err := items.Order("id ASC").Find(&o.Items).Error; err != nil { + return nil, err + } + if err := tx.Where("space_id=? AND id=?", space, expt).Find(&o.Experiment).Error; err != nil { + return nil, err + } + if run <= 0 { + return o, nil + } + if err := tx.Where("space_id=? AND expt_id=? AND expt_run_id=?", space, expt, run).Order("id ASC").Find(&o.Runs).Error; err != nil { + return nil, err + } + logs := tx.Where("space_id=? AND expt_id=? AND expt_run_id=?", space, expt, run) + if err := logs.Order("id ASC").Find(&o.ItemLogs).Error; err != nil { + return nil, err + } + itemLogs := o.ItemLogs + if frozen != nil { + itemLogs = frozen.ItemLogs + } + frozenItems := make([]int64, 0, len(itemLogs)) + for _, log := range itemLogs { + frozenItems = append(frozenItems, log.ItemID) + } + if err := tx.Where("space_id=? AND expt_id=? AND item_id IN ?", space, expt, frozenItems).Order("id ASC").Find(&o.ItemRefs).Error; err != nil { + return nil, err + } + if err := tx.Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id IN ?", space, expt, run, frozenItems).Order("id ASC").Find(&o.Turns).Error; err != nil { + return nil, err + } + if err := tx.Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id IN ?", space, expt, run, frozenItems).Order("id ASC").Find(&o.TurnLogs).Error; err != nil { + return nil, err + } + turns := o.Turns + if frozen != nil { + turns = frozen.Turns + } + turnIDs := make([]int64, 0, len(turns)) + for _, turn := range turns { + turnIDs = append(turnIDs, turn.ID) + } + if err := tx.Where("space_id=? AND expt_id=? AND expt_turn_result_id IN ?", space, expt, turnIDs).Order("id ASC").Find(&o.Refs).Error; err != nil { + return nil, err + } + return o, nil +} + +func verifyItemReadObservation(before, after *itemReadObservation) error { + if !reflect.DeepEqual(before, after) { + return fmt.Errorf("current item observation changed; retry primary read") + } + return nil +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_result_publish.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_result_publish.go new file mode 100644 index 0000000000..b1b3749ea5 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_result_publish.go @@ -0,0 +1,259 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type ExptResultPublishDAOImpl struct{ provider db.Provider } + +func NewExptResultPublishDAO(provider db.Provider) repo.IExptResultPublishRepo { + return &ExptResultPublishDAOImpl{provider: provider} +} + +func readItemResultPublishSource(tx *gorm.DB, space, expt, run, item int64) (*entity.ExptItemResultPublishSource, error) { + return readItemResultPublishSourceWithLocks(tx, space, expt, run, item, true) +} + +func readItemResultPublishSourceWithLocks(tx *gorm.DB, space, expt, run, item int64, lockRows bool) (*entity.ExptItemResultPublishSource, error) { + var locks []clause.Expression + if lockRows { + locks = append(locks, clause.Locking{Strength: "UPDATE"}) + } + source, err := readItemRunSource(tx, space, expt, run, item, true, lockRows) + if err != nil || source == nil { + return nil, err + } + var experiment model.Experiment + if err := tx.Where("space_id=? AND id=?", space, expt).Take(&experiment).Error; err != nil { + return nil, err + } + if experiment.LatestRunID != run { + return nil, nil + } + var canonical model.ExptItemResult + if err := tx.Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", space, expt, run, item).Take(&canonical).Error; err != nil { + return nil, err + } + var log model.ExptItemResultRunLog + if err := tx.Where("space_id=? AND expt_id=? AND expt_run_id=? AND item_id=?", space, expt, run, item).Take(&log).Error; err != nil { + return nil, err + } + itemDO := convert.NewExptItemResultConvertor().PO2DO(&canonical) + itemDO.Ext, err = entity.ParseExptItemResultExt(gptr.Indirect(canonical.Ext)) + if err != nil { + return nil, err + } + result := &entity.ExptItemResultPublishSource{Source: source, Item: itemDO, ItemLog: convert.NewExptItemResultRunLogConverter().PO2DO(&log)} + var ref model.ExptItemRef + err = tx.Clauses(locks...).Where("space_id=? AND expt_id=? AND item_id=?", space, expt, item).Take(&ref).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, err + } + if err == nil { + if ref.ItemVersionID != canonical.ItemVersionID { + return nil, fmt.Errorf("publication frozen item version mismatch") + } + result.ItemRef = convert.NewExptItemRefConvertor().PO2DO(&ref) + result.ItemConfigJSON = append([]byte(nil), gptr.Indirect(ref.ItemConfig)...) + if len(result.ItemConfigJSON) > 0 { + var config entity.ExptItemConfig + if err := json.Unmarshal(result.ItemConfigJSON, &config); err != nil { + return nil, err + } + if string(result.ItemConfigJSON) == "null" { + return nil, fmt.Errorf("publication frozen item configuration null") + } + result.ItemRef.ItemConfig = &config + } + } + return result, nil +} + +func validPublicationSource(source *entity.ExptItemResultPublishSource) bool { + return source != nil && source.Source != nil && source.Item != nil && source.ItemLog != nil && source.ItemLog.ResultState == int32(entity.ExptItemResultStateLogged) && entity.IsItemRunFinished(entity.ItemRunState(source.ItemLog.Status)) && (!entity.IsItemRunFinished(source.Item.Status) || source.Item.Status == entity.ItemRunState(source.ItemLog.Status)) +} + +func equalPublicationSource(a, b *entity.ExptItemResultPublishSource) bool { + if !validPublicationSource(a) || !validPublicationSource(b) || !currentSourceMatches(a.Source, b.Source) { + return false + } + if !reflect.DeepEqual(a.Item, b.Item) || !reflect.DeepEqual(a.ItemLog, b.ItemLog) || !reflect.DeepEqual(a.Source.CanonicalTurns, b.Source.CanonicalTurns) || !reflect.DeepEqual(a.ItemRef, b.ItemRef) || !reflect.DeepEqual(a.ItemConfigJSON, b.ItemConfigJSON) { + return false + } + return reflect.DeepEqual(a.Source.Turns, b.Source.Turns) +} + +func (r *ExptResultPublishDAOImpl) GetItemResultPublishSource(ctx context.Context, space, expt, run, item int64) (*entity.ExptItemResultPublishSource, error) { + var result *entity.ExptItemResultPublishSource + err := r.provider.Transaction(ctx, func(tx *gorm.DB) error { + var err error + result, err = readItemResultPublishSource(tx, space, expt, run, item) + return err + }, db.WithMaster()) + return result, err +} + +func (r *ExptResultPublishDAOImpl) IsItemResultPublishSourceCurrent(ctx context.Context, source *entity.ExptItemResultPublishSource) (bool, error) { + if source == nil || source.Source == nil { + return false, fmt.Errorf("publication source missing") + } + c := source.Source.Current + valid := false + err := r.provider.Transaction(ctx, func(tx *gorm.DB) error { + actual, err := readItemResultPublishSource(tx, c.SpaceID, c.ExptID, c.RunID, c.ItemID) + if err != nil { + return err + } + valid = equalPublicationSource(source, actual) + return ctx.Err() + }, db.WithMaster()) + return valid && err == nil, err +} + +func (r *ExptResultPublishDAOImpl) CommitItemResult(ctx context.Context, publication *entity.ExptItemResultPublication) (bool, error) { + if publication == nil || publication.Source == nil || publication.Source.Source == nil { + return false, fmt.Errorf("publication source missing") + } + source := publication.Source + c := source.Source.Current + applied := false + err := r.provider.Transaction(ctx, func(tx *gorm.DB) error { + actual, err := readItemResultPublishSource(tx, c.SpaceID, c.ExptID, c.RunID, c.ItemID) + if err != nil { + return err + } + if !equalPublicationSource(source, actual) { + return nil + } + if len(actual.Source.CanonicalTurns) == 0 || len(actual.Source.CanonicalTurns) != len(actual.Source.Turns) { + return fmt.Errorf("publication frozen turn source incomplete") + } + if len(publication.Turns) != len(actual.Source.CanonicalTurns) { + return fmt.Errorf("publication projected turns incomplete") + } + turns := map[int64]*entity.ExptTurnResult{} + turnIDs := make([]int64, 0, len(publication.Turns)) + for _, turn := range publication.Turns { + if turn == nil || turn.SpaceID != c.SpaceID || turn.ExptID != c.ExptID || turn.ExptRunID != c.RunID || turn.ItemID != c.ItemID || turn.ItemVersionID != c.ItemVersionID { + return fmt.Errorf("foreign publication turn") + } + log := actual.Source.Turns[turn.TurnID] + if log == nil || turn.Status != int32(log.Status) || turn.TargetResultID != log.TargetResultID || turn.ErrMsg != log.ErrMsg || turn.LogID != log.LogID || turn.TraceID != log.TraceID { + return fmt.Errorf("publication turn/source mismatch") + } + if _, exists := turns[turn.ID]; exists { + return fmt.Errorf("duplicate publication turn") + } + turns[turn.ID] = turn + turnIDs = append(turnIDs, turn.ID) + } + expectedRefs := map[string]int64{} + for _, turn := range actual.Source.CanonicalTurns { + if turns[turn.ID] == nil || turns[turn.ID].TurnID != turn.TurnID { + return fmt.Errorf("publication canonical turn mismatch") + } + log := actual.Source.Turns[turn.TurnID] + refs := log.EvaluatorResultIds + if refs == nil { + continue + } + for _, ref := range refs.Registered { + expectedRefs[publicationRefKey(turn.ID, ref.VersionID, ref.Alias, "")] = ref.RecordID + } + for _, ref := range refs.Inline { + expectedRefs[publicationRefKey(turn.ID, 0, "", ref.InlineKey)] = ref.RecordID + } + if !refs.IsNewFormat() { + for ver, id := range refs.EvalVerIDToResID { + expectedRefs[publicationRefKey(turn.ID, ver, "", "")] = id + } + } + } + if len(expectedRefs) != len(publication.Refs) { + return fmt.Errorf("publication refs incomplete") + } + priorRefs := map[string]*entity.ExptTurnEvaluatorResultRef{} + for _, ref := range actual.Source.CurrentRefs { + priorRefs[publicationRefKey(ref.ExptTurnResultID, ref.EvaluatorVersionID, ref.Alias, ref.InlineKey)] = ref + } + keepIDs := []int64{} + inserts := []*model.ExptTurnEvaluatorResultRef{} + used := map[string]bool{} + for _, ref := range publication.Refs { + if ref == nil || ref.ID <= 0 || ref.SpaceID != c.SpaceID || ref.ExptID != c.ExptID || turns[ref.ExptTurnResultID] == nil { + return fmt.Errorf("foreign publication ref") + } + key := publicationRefKey(ref.ExptTurnResultID, ref.EvaluatorVersionID, ref.Alias, ref.InlineKey) + if used[key] || expectedRefs[key] != ref.EvaluatorResultID { + return fmt.Errorf("publication ref source mismatch") + } + used[key] = true + if old := priorRefs[key]; old != nil && old.ID == ref.ID && old.EvaluatorResultID == ref.EvaluatorResultID && old.SourceType == ref.SourceType { + keepIDs = append(keepIDs, old.ID) + } else { + inserts = append(inserts, convert.NewExptTurnEvaluatorResultRefConvertor().DO2PO(ref)) + } + } + deletion := tx.Where("space_id=? AND expt_id=? AND expt_turn_result_id IN ?", c.SpaceID, c.ExptID, turnIDs) + if len(keepIDs) > 0 { + deletion = deletion.Where("id NOT IN ?", keepIDs) + } + if err := deletion.Unscoped().Delete(&model.ExptTurnEvaluatorResultRef{}).Error; err != nil { + return err + } + for _, ref := range inserts { + if err := tx.Create(ref).Error; err != nil { + return err + } + } + for _, turn := range publication.Turns { + fields := map[string]any{"status": turn.Status, "target_result_id": turn.TargetResultID, "trace_id": turn.TraceID, "log_id": turn.LogID, "err_msg": []byte(turn.ErrMsg), "weighted_score": turn.WeightedScore, "expt_run_id": c.RunID} + if err := tx.Model(&model.ExptTurnResult{}).Where("id=? AND space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND item_version_id=?", turn.ID, c.SpaceID, c.ExptID, c.RunID, c.ItemID, c.ItemVersionID).Updates(fields).Error; err != nil { + return err + } + } + if err := tx.Model(&model.ExptItemResult{}).Where("id=? AND space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND item_version_id=?", actual.Item.ID, c.SpaceID, c.ExptID, c.RunID, c.ItemID, c.ItemVersionID).Updates(map[string]any{"status": actual.ItemLog.Status, "log_id": actual.ItemLog.LogID, "err_msg": actual.ItemLog.ErrMsg}).Error; err != nil { + return err + } + if err := tx.Model(&model.ExptItemResultRunLog{}).Where("id=? AND space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND result_state=?", actual.ItemLog.ID, c.SpaceID, c.ExptID, c.RunID, c.ItemID, int32(entity.ExptItemResultStateLogged)).Update("result_state", int32(entity.ExptItemResultStateResulted)).Error; err != nil { + return err + } + before, after := actual.Item.Status, entity.ItemRunState(actual.ItemLog.Status) + if before != after { + var stats model.ExptStats + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=?", c.SpaceID, c.ExptID).Take(&stats).Error; err != nil { + return err + } + columns := map[entity.ItemRunState]string{entity.ItemRunState_Queueing: "pending_cnt", entity.ItemRunState_Processing: "processing_cnt", entity.ItemRunState_Success: "success_cnt", entity.ItemRunState_Fail: "fail_cnt", entity.ItemRunState_Terminal: "terminated_cnt"} + from, to := columns[before], columns[after] + if from == "" || to == "" { + return fmt.Errorf("publication stats source invalid") + } + if err := tx.Model(&model.ExptStats{}).Where("id=? AND space_id=? AND expt_id=?", stats.ID, c.SpaceID, c.ExptID).Updates(map[string]any{from: gorm.Expr(from + "-1"), to: gorm.Expr(to + "+1")}).Error; err != nil { + return err + } + } + applied = true + return ctx.Err() + }, db.WithMaster()) + return applied && err == nil, err +} + +func publicationRefKey(turn, version int64, alias, key string) string { + return fmt.Sprintf("%d/%d/%s/%s", turn, version, alias, key) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_result_publish_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_result_publish_test.go new file mode 100644 index 0000000000..078033cc35 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_result_publish_test.go @@ -0,0 +1,108 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestResultPublishMySQLReuseRefsAndReceipt(t *testing.T) { + f := newCurrentResultFixture(t, "rq1") + ctx := context.Background() + raw := []byte(`[{"ItemIDs":[9223372036854775806],"CreateAt":1700000000,"manual_retry_receipt":{"request_id":"rq1","payload_hash":"original","accepted_count":1,"rejected_items":null,"snapshot_at_ms":1}}]`) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("item_ids", raw).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Fail), "result_state": int32(entity.ExptItemResultStateLogged)}).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.TurnRunState_Fail), "evaluator_result_ids": []byte(`null`)}).Error) + refs, _ := json.Marshal(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: f.base + 61}}}) + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND turn_id=0", f.space).Update("evaluator_result_ids", refs).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnEvaluatorResultRef{}).Where("space_id=?", f.space).Update("source_type", int32(entity.EvaluatorRecordSourceTypeBuiltin)).Error) + port := NewExptResultPublishDAO(f.provider) + source, err := port.GetItemResultPublishSource(ctx, f.space, f.expt, f.run, f.item) + require.NoError(t, err) + publication := &entity.ExptItemResultPublication{Source: source, Refs: source.Source.CurrentRefs} + for _, old := range source.Source.CanonicalTurns { + turn := *old + log := source.Source.Turns[old.TurnID] + turn.Status = int32(log.Status) + turn.TargetResultID = log.TargetResultID + turn.TraceID = log.TraceID + turn.LogID = log.LogID + turn.ErrMsg = log.ErrMsg + turn.WeightedScore = nil + publication.Turns = append(publication.Turns, &turn) + } + before := f.snapshot() + ok, err := port.CommitItemResult(ctx, publication) + require.NoError(t, err) + require.True(t, ok) + after := f.snapshot() + require.Equal(t, before["expt_turn_evaluator_result_ref"], after["expt_turn_evaluator_result_ref"]) + require.Equal(t, before["expt_run_log"], after["expt_run_log"]) + ok, err = port.CommitItemResult(ctx, publication) + require.NoError(t, err) + require.False(t, ok) + require.Equal(t, after, f.snapshot()) +} + +func TestResultPublishMySQLCanonicalLockAgainstAdmission(t *testing.T) { + f := newCurrentResultFixture(t, "rq1") + ctx := context.Background() + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Fail), "result_state": int32(entity.ExptItemResultStateLogged)}).Error) + port := NewExptResultPublishDAO(f.provider) + source, err := port.GetItemResultPublishSource(ctx, f.space, f.expt, f.run, f.item) + require.NoError(t, err) + publication := &entity.ExptItemResultPublication{Source: source} + blocked := make(chan struct{}) + release := make(chan struct{}) + admissionDone := make(chan error, 1) + go func() { + admissionDone <- f.provider.Transaction(ctx, func(tx *gorm.DB) error { + var row model.ExptItemResult + if err := tx.Raw("SELECT * FROM expt_item_result WHERE space_id=? AND expt_id=? AND item_id=? FOR UPDATE", f.space, f.expt, f.item).Scan(&row).Error; err != nil { + return err + } + require.IsType(t, &sql.Tx{}, tx.Statement.ConnPool) + ext := map[string]string{} + inner, _ := json.Marshal(&entity.ManualRetryContext{RunID: f.run, RequestID: "rq2", Mode: entity.EvaluationModeFailRetry, Operator: "test", AcceptedAtMS: 1}) + ext[entity.ManualRetryExtKey] = string(inner) + raw, _ := json.Marshal(ext) + if err := tx.Model(&row).Update("ext", raw).Error; err != nil { + return err + } + close(blocked) + <-release + return nil + }) + }() + <-blocked + result := make(chan error, 1) + go func() { + ok, err := port.CommitItemResult(ctx, publication) + if ok { + err = errors.New("old publication wrote after admission") + } + result <- err + }() + select { + case err := <-result: + t.Fatalf("publication ignored canonical row lock: %v", err) + case <-time.After(30 * time.Millisecond): + } + close(release) + require.NoError(t, <-admissionDone) + require.NoError(t, <-result) + var log model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&log).Error) + require.Equal(t, int32(entity.ExptItemResultStateLogged), gptr.Indirect(log.ResultState)) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish.go new file mode 100644 index 0000000000..250f161beb --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish.go @@ -0,0 +1,184 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "fmt" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "gorm.io/gorm" +) + +type ExptRunFinishDAOImpl struct{ provider db.Provider } + +func NewExptRunFinishDAO(provider db.Provider) repo.IExptRunFinishRepo { + return &ExptRunFinishDAOImpl{provider: provider} +} + +func (d *ExptRunFinishDAOImpl) FinishCurrentRun(ctx context.Context, r *entity.ExptRunFinishRequest) (*entity.ExptRunFinishResult, error) { + if r == nil || r.SpaceID <= 0 || r.ExptID <= 0 || r.RunID <= 0 { + return nil, fmt.Errorf("invalid finish identities") + } + result := &entity.ExptRunFinishResult{} + err := d.provider.Transaction(ctx, func(tx *gorm.DB) error { + opts := []db.Option{db.WithTransaction(tx), db.WithMaster(), db.WithSelectForUpdate()} + expt, err := NewExptDAO(d.provider).GetByID(ctx, r.ExptID, opts...) + if err != nil { + return err + } + run, err := NewExptRunLogDAO(d.provider).Get(ctx, r.ExptID, r.RunID, opts...) + if err != nil { + return err + } + if expt.SpaceID != r.SpaceID || run.SpaceID != r.SpaceID || run.ExptID != r.ExptID || run.ExptRunID != r.RunID { + return fmt.Errorf("finish run binding corrupt") + } + if expt.LatestRunID != r.RunID { + result.Disposition = entity.ExptRunFinishNonCurrent + return nil + } + if expt.ExptType != int32(entity.ExptType_Offline) { + return fmt.Errorf("natural current-run finish requires offline experiment") + } + runFinished := entity.IsExptFinished(entity.ExptStatus(gptr.Indirect(run.Status))) + exptFinished := entity.IsExptFinished(entity.ExptStatus(expt.Status)) + if runFinished || exptFinished { + if !runFinished || !exptFinished || expt.EndAt == nil { + return fmt.Errorf("inconsistent partially finished current run") + } + result.Disposition = entity.ExptRunFinishAlreadyFinished + result.Experiment, err = convert.NewExptConverter().PO2DO(expt, nil) + return err + } + if expt.Status != int32(entity.ExptStatus_Processing) || (gptr.Indirect(run.Status) != int64(entity.ExptStatus_Pending) && gptr.Indirect(run.Status) != int64(entity.ExptStatus_Processing)) { + return fmt.Errorf("finish requires active experiment and run") + } + session := d.provider.NewSession(ctx, opts...) + var items []model.ExptItemResult + if err := session.Select("id,item_id,expt_run_id,item_version_id,status").Where("space_id=? AND expt_id=?", r.SpaceID, r.ExptID).Order("item_id").Find(&items).Error; err != nil { + return err + } + var itemLogs []model.ExptItemResultRunLog + if err := session.Select("id,item_id,item_version_id,status,result_state").Where("space_id=? AND expt_id=? AND expt_run_id=?", r.SpaceID, r.ExptID, r.RunID).Order("item_id").Find(&itemLogs).Error; err != nil { + return err + } + for _, log := range itemLogs { + if log.Status == int32(entity.ItemRunState_Queueing) || log.Status == int32(entity.ItemRunState_Processing) || gptr.Indirect(log.ResultState) == int32(entity.ExptItemResultStateLogged) { + result.Disposition = entity.ExptRunFinishPending + return nil + } + } + canonical := make(map[int64]model.ExptItemResult, len(items)) + var itemCounts finishCounts + for _, item := range items { + canonical[item.ItemID] = item + itemCounts.add(item.Status) + } + for _, log := range itemLogs { + item, ok := canonical[log.ItemID] + if !ok || item.ExptRunID != r.RunID || item.Status != log.Status || item.ItemVersionID != log.ItemVersionID || gptr.Indirect(log.ResultState) != int32(entity.ExptItemResultStateResulted) || !entity.IsItemRunFinished(entity.ItemRunState(log.Status)) { + return fmt.Errorf("finish canonical item/run log binding corrupt") + } + } + loggedItems := make(map[int64]bool, len(itemLogs)) + for _, log := range itemLogs { + loggedItems[log.ItemID] = true + } + for _, item := range items { + if item.ExptRunID == r.RunID && !loggedItems[item.ItemID] { + return fmt.Errorf("finish current item run log missing") + } + } + var turns []model.ExptTurnResult + if err := session.Select("id,item_id,turn_id,status").Where("space_id=? AND expt_id=?", r.SpaceID, r.ExptID).Order("item_id,turn_id").Find(&turns).Error; err != nil { + return err + } + var turnCounts finishCounts + for _, turn := range turns { + turnCounts.addTurn(entity.TurnRunState(turn.Status)) + } + var stats model.ExptStats + if err := session.Where("space_id=? AND expt_id=?", r.SpaceID, r.ExptID).Take(&stats).Error; err != nil { + return err + } + runStatus := entity.ExptStatus_Success + // Preserve calculateRunLogStats: canonical turns across all runs, including + // unexecuted turns after an item failure. Those counts do not block finish. + if turnCounts.pending > 0 || turnCounts.fail > 0 { + runStatus = entity.ExptStatus_Failed + } + exptStatus := entity.ExptStatus_Success + // User-terminated items are not evaluation failures (CalculateStats contract). + if itemCounts.fail > 0 || itemCounts.pending > 0 || itemCounts.processing > 0 { + exptStatus = entity.ExptStatus_Failed + } + if err := session.Model(&model.ExptRunLog{}).Where("space_id=? AND expt_id=? AND expt_run_id=?", r.SpaceID, r.ExptID, r.RunID).UpdateColumns(turnCounts.fields("status", int64(runStatus))).Error; err != nil { + return err + } + now := time.Now() + if err := session.Model(&model.Experiment{}).Where("space_id=? AND id=? AND latest_run_id=?", r.SpaceID, r.ExptID, r.RunID).UpdateColumns(map[string]any{"status": int32(exptStatus), "end_at": now}).Error; err != nil { + return err + } + if err := session.Model(&model.ExptStats{}).Where("space_id=? AND expt_id=?", r.SpaceID, r.ExptID).UpdateColumns(itemCounts.fields("", nil)).Error; err != nil { + return err + } + result.Disposition = entity.ExptRunFinishFinished + expt.Status = int32(exptStatus) + expt.EndAt = &now + result.Experiment, err = convert.NewExptConverter().PO2DO(expt, nil) + return err + }, db.WithMaster()) + if err != nil { + return nil, err + } + return result, nil +} + +// Counts follow the existing canonical projection readers. Item and turn +// status enums differ; keep their mappings separate. +type finishCounts struct{ pending, processing, success, fail, terminal int32 } + +func (c *finishCounts) add(status int32) { + switch status { + case int32(entity.ItemRunState_Queueing): + c.pending++ + case int32(entity.ItemRunState_Processing): + c.processing++ + case int32(entity.ItemRunState_Success): + c.success++ + case int32(entity.ItemRunState_Fail): + c.fail++ + case int32(entity.ItemRunState_Terminal): + c.terminal++ + } +} + +func (c finishCounts) fields(statusKey string, status any) map[string]any { + result := map[string]any{"pending_cnt": c.pending, "processing_cnt": c.processing, "success_cnt": c.success, "fail_cnt": c.fail, "terminated_cnt": c.terminal} + if statusKey != "" { + result[statusKey] = status + } + return result +} + +func (c *finishCounts) addTurn(status entity.TurnRunState) { + switch status { + case entity.TurnRunState_Queueing: + c.pending++ + case entity.TurnRunState_Processing: + c.processing++ + case entity.TurnRunState_Success: + c.success++ + case entity.TurnRunState_Fail: + c.fail++ + case entity.TurnRunState_Terminal: + c.terminal++ + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish_pending_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish_pending_test.go new file mode 100644 index 0000000000..7254773607 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish_pending_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "fmt" + "testing" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" +) + +// Port boundary coverage complements the service test that calls real LogRun. +func TestRunFinishPendingAdmissionSQLPorts(t *testing.T) { + for _, stage := range []string{"preflight", "requeue"} { + t.Run(stage, func(t *testing.T) { + f := newManualRetryFixture(t) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=? AND expt_run_id=?", f.space, f.run).Update("status", int64(entity.ExptStatus_Pending)).Error) + request := f.request(entity.EvaluationModeRetryItems) + if stage == "preflight" { + plan, err := f.port.PreflightTerminalItems(context.Background(), request) + require.NoError(t, err) + require.Len(t, plan.Items, 1) + } else { + receipt, err := f.port.RequeueTerminalItems(context.Background(), request) + require.NoError(t, err) + require.Equal(t, int32(1), receipt.AcceptedCount) + } + var run model.ExptRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_run_id=?", f.space, f.run).Take(&run).Error) + require.Equal(t, int64(entity.ExptStatus_Pending), gptr.Indirect(run.Status)) + }) + } +} + +func TestRunFinishPendingDoesNotAdmitInactiveExperimentOrRun(t *testing.T) { + for _, state := range []entity.ExptStatus{entity.ExptStatus_Pending, entity.ExptStatus_Terminating, entity.ExptStatus_Draining, entity.ExptStatus_Success, entity.ExptStatus_Failed} { + t.Run(fmt.Sprint(state), func(t *testing.T) { + f := newManualRetryFixture(t) + request := f.request(entity.EvaluationModeRetryItems) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("status", int32(state)).Error) + before := f.snapshot() + _, err := f.port.PreflightTerminalItems(context.Background(), request) + require.Error(t, err) + _, err = f.port.RequeueTerminalItems(context.Background(), request) + require.Error(t, err) + require.Equal(t, before, f.snapshot()) + }) + } + for _, state := range []entity.ExptStatus{entity.ExptStatus_Terminating, entity.ExptStatus_Draining, entity.ExptStatus_Success, entity.ExptStatus_Failed} { + t.Run("run_"+fmt.Sprint(state), func(t *testing.T) { + f := newManualRetryFixture(t) + request := f.request(entity.EvaluationModeRetryItems) + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(state)).Error) + before := f.snapshot() + _, err := f.port.PreflightTerminalItems(context.Background(), request) + require.Error(t, err) + _, err = f.port.RequeueTerminalItems(context.Background(), request) + require.Error(t, err) + require.Equal(t, before, f.snapshot()) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish_test.go new file mode 100644 index 0000000000..e622decfdf --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_finish_test.go @@ -0,0 +1,339 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// Removing the primary pending recheck would close a run with newly queued work. +func TestRunFinishMySQLPendingDoesNotWrite(t *testing.T) { + for _, pending := range []struct { + name string + status entity.ItemRunState + state int32 + reserved int32 + }{ + {"queueing", entity.ItemRunState_Queueing, 0, 0}, + {"reserved", entity.ItemRunState_Queueing, 0, 1}, + {"processing", entity.ItemRunState_Processing, 0, 0}, + {"logged", entity.ItemRunState_Fail, 2, 0}, + } { + t.Run(pending.name, func(t *testing.T) { + f := newManualRetryFixture(t) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": 1}).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, 11).Updates(map[string]any{"status": int32(pending.status), "result_state": pending.state, "quota_reservation_state": pending.reserved}).Error) + before := f.snapshot() + result, err := NewExptRunFinishDAO(f.provider).FinishCurrentRun(context.Background(), &entity.ExptRunFinishRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run}) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishPending, result.Disposition) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func terminalFinishFixture(t *testing.T) *manualRetryFixture { + f := newManualRetryFixture(t) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Updates(map[string]any{"status": int32(entity.ItemRunState_Success)}).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND expt_id=?", f.space, f.expt).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": 1}).Error) + return f +} + +func finishFixture(f *manualRetryFixture) (*entity.ExptRunFinishResult, error) { + return NewExptRunFinishDAO(f.provider).FinishCurrentRun(context.Background(), &entity.ExptRunFinishRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run}) +} + +// Canonical turn counts span historical success, while item counts retain the +// experiment projection; credit/token and opaque receipt bytes are untouched. +func TestRunFinishMySQLFinalStatsAndIdempotency(t *testing.T) { + f := terminalFinishFixture(t) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, 11).Update("status", int32(entity.ItemRunState_Fail)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, 11).Update("status", int32(entity.ItemRunState_Fail)).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=? AND turn_id=?", f.space, 22).Update("status", int32(entity.TurnRunState_Queueing)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, 12).Updates(map[string]any{"expt_run_id": f.run - 1}).Error) + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, 12).Delete(&model.ExptItemResultRunLog{}).Error) + require.NoError(t, f.conn.Create(&model.ExptTurnResult{ID: f.base + 9501, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run - 1, ItemID: 12, ItemVersionID: f.base + 112, TurnID: 0, Status: int32(entity.TurnRunState_Success), WeightedScore: nil}).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, 13).Update("status", int32(entity.ItemRunState_Terminal)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, 13).Update("status", int32(entity.ItemRunState_Terminal)).Error) + require.NoError(t, f.conn.Create(&model.ExptTurnResult{ID: f.base + 9502, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: 13, ItemVersionID: f.base + 113, TurnID: 0, Status: int32(entity.TurnRunState_Terminal)}).Error) + before := f.snapshot() + var originalRun model.ExptRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_run_id=?", f.space, f.run).Take(&originalRun).Error) + result, err := finishFixture(f) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishFinished, result.Disposition) + var run model.ExptRunLog + require.NoError(t, f.conn.Where("space_id=? AND expt_run_id=?", f.space, f.run).Take(&run).Error) + require.Equal(t, int64(entity.ExptStatus_Failed), gptr.Indirect(run.Status)) + require.Equal(t, int32(1), run.PendingCnt) + require.Equal(t, int32(1), run.SuccessCnt) + require.Equal(t, int32(1), run.FailCnt) + require.Equal(t, int32(1), run.TerminatedCnt) + require.Equal(t, float64(42), run.CreditCost) + require.Equal(t, int64(123), gptr.Indirect(run.TokenCost)) + require.Equal(t, gptr.Indirect(originalRun.ItemIds), gptr.Indirect(run.ItemIds)) + require.Contains(t, string(gptr.Indirect(run.ItemIds)), `"future":{"id":9223372036854775806}`) + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Take(&stats).Error) + require.Equal(t, int32(1), stats.SuccessCnt) + require.Equal(t, int32(1), stats.FailCnt) + require.Equal(t, int32(1), stats.TerminatedCnt) + require.Zero(t, stats.PendingCnt) + require.Equal(t, float64(42), stats.CreditCost) + require.Equal(t, int64(100), gptr.Indirect(stats.InputTokenCost)) + after := f.snapshot() + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_turn_evaluator_result_ref"} { + require.Equal(t, before[table], after[table], table) + } + var writes atomic.Int32 + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("finish_second_write", func(*gorm.DB) { writes.Add(1) })) + result, err = finishFixture(f) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishAlreadyFinished, result.Disposition) + require.Zero(t, writes.Load()) + require.Equal(t, after, f.snapshot()) +} + +func TestRunFinishMySQLTerminalItemsAreNotExperimentFailures(t *testing.T) { + f := terminalFinishFixture(t) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=?", f.space).Update("status", int32(entity.ItemRunState_Terminal)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=?", f.space).Update("status", int32(entity.ItemRunState_Terminal)).Error) + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=?", f.space).Update("status", int32(entity.TurnRunState_Terminal)).Error) + result, err := finishFixture(f) + require.NoError(t, err) + require.Equal(t, entity.ExptStatus_Success, result.Experiment.Status) +} + +type finishFaultProvider struct{ db.Provider } + +func (p finishFaultProvider) Transaction(ctx context.Context, fn func(*gorm.DB) error, opts ...db.Option) error { + return p.Provider.Transaction(ctx, func(tx *gorm.DB) error { + committer := tx.Statement.ConnPool.(gorm.TxCommitter) + tx.Statement.ConnPool = &finishCommitFaultConnection{ConnPool: tx.Statement.ConnPool, TxCommitter: committer} + return fn(tx) + }, opts...) +} + +func TestRunFinishMySQLRollbackEveryWriteAndCommit(t *testing.T) { + for _, table := range []string{"expt_run_log", "experiment", "expt_stats", "commit"} { + t.Run(table, func(t *testing.T) { + f := terminalFinishFixture(t) + before := f.snapshot() + provider := f.provider + if table == "commit" { + provider = finishFaultProvider{provider} + } else { + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("finish_fault", func(tx *gorm.DB) { + if tx.Statement.Table == table { + _ = tx.AddError(errors.New("finish write fault")) + } + })) + } + result, err := NewExptRunFinishDAO(provider).FinishCurrentRun(context.Background(), &entity.ExptRunFinishRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run}) + require.Error(t, err) + require.Nil(t, result) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestRunFinishMySQLMissingBindingAndHalfTerminalFailClosed(t *testing.T) { + for _, which := range []string{"missing_run", "missing_experiment", "wrong_space", "noncurrent", "half_run", "half_expt", "missing_stats"} { + t.Run(which, func(t *testing.T) { + f := terminalFinishFixture(t) + switch which { + case "missing_run": + require.NoError(t, f.conn.Where("space_id=?", f.space).Delete(&model.ExptRunLog{}).Error) + case "missing_experiment": + require.NoError(t, f.conn.Where("space_id=?", f.space).Delete(&model.Experiment{}).Error) + case "wrong_space": + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("space_id", f.space+100).Error) + t.Cleanup(func() { f.conn.Exec("DELETE FROM expt_run_log WHERE space_id=? AND expt_id=?", f.space+100, f.expt) }) + case "noncurrent": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("latest_run_id", f.run+100).Error) + case "half_run": + require.NoError(t, f.conn.Model(&model.ExptRunLog{}).Where("space_id=?", f.space).Update("status", int64(entity.ExptStatus_Success)).Error) + case "half_expt": + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Updates(map[string]any{"status": int32(entity.ExptStatus_Success), "end_at": time.Now()}).Error) + case "missing_stats": + require.NoError(t, f.conn.Where("space_id=?", f.space).Delete(&model.ExptStats{}).Error) + } + before := f.snapshot() + r, err := finishFixture(f) + if which == "noncurrent" { + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishNonCurrent, r.Disposition) + } else { + require.Error(t, err) + require.Nil(t, r) + } + require.Equal(t, before, f.snapshot()) + }) + } +} + +// Both participants execute real provider transactions. The first holds its +// experiment/run locks until the second has attempted admission or finish. +type finishBarrierProvider struct { + db.Provider + written chan struct{} + release chan struct{} +} + +func (p finishBarrierProvider) Transaction(ctx context.Context, fn func(*gorm.DB) error, opts ...db.Option) error { + return p.Provider.Transaction(ctx, func(tx *gorm.DB) error { + if err := fn(tx); err != nil { + return err + } + close(p.written) + <-p.release + return nil + }, opts...) +} + +func TestRunFinishMySQLConcurrentAdmissionAndReceiptReplay(t *testing.T) { + for _, first := range []string{"admission", "finish"} { + t.Run(first, func(t *testing.T) { + f := terminalFinishFixture(t) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, 11).Update("status", int32(entity.ItemRunState_Fail)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, 11).Update("status", int32(entity.ItemRunState_Fail)).Error) + request := f.request(entity.EvaluationModeRetryItems) + written, release := make(chan struct{}), make(chan struct{}) + barrier := finishBarrierProvider{Provider: f.provider, written: written, release: release} + done1, done2 := make(chan error, 1), make(chan error, 1) + var finished *entity.ExptRunFinishResult + var receipt *entity.ManualRetryReceipt + if first == "admission" { + port := f.port.(*ExptManualRetryDAOImpl) + copyPort := *port + copyPort.provider = barrier + go func() { + var err error + receipt, err = copyPort.RequeueTerminalItems(context.Background(), request) + done1 <- err + }() + <-written + go func() { var err error; finished, err = finishFixture(f); done2 <- err }() + } else { + go func() { + var err error + finished, err = NewExptRunFinishDAO(barrier).FinishCurrentRun(context.Background(), &entity.ExptRunFinishRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run}) + done1 <- err + }() + <-written + go func() { + var err error + receipt, err = f.port.RequeueTerminalItems(context.Background(), request) + done2 <- err + }() + } + select { + case err := <-done2: + close(release) + t.Fatalf("second transaction ignored first experiment/run locks: %v", err) + case <-time.After(40 * time.Millisecond): + } + close(release) + require.NoError(t, <-done1) + err2 := <-done2 + if first == "admission" { + require.NoError(t, err2) + require.Equal(t, int32(1), receipt.AcceptedCount) + require.Equal(t, entity.ExptRunFinishPending, finished.Disposition) + after := f.snapshot() + again, err := f.port.RequeueTerminalItems(context.Background(), request) + require.NoError(t, err) + originalRaw, _ := json.Marshal(receipt) + replayRaw, _ := json.Marshal(again) + require.Equal(t, string(originalRaw), string(replayRaw)) + require.Equal(t, after, f.snapshot()) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, 11).Update("status", int32(entity.ItemRunState_Success)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, 11).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": 1}).Error) + finished, err = finishFixture(f) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishFinished, finished.Disposition) + after = f.snapshot() + again, err = f.port.RequeueTerminalItems(context.Background(), request) + require.NoError(t, err) + originalRaw, _ = json.Marshal(receipt) + replayRaw, _ = json.Marshal(again) + require.Equal(t, string(originalRaw), string(replayRaw)) + require.Equal(t, after, f.snapshot()) + } else { + require.Error(t, err2) + require.Nil(t, receipt) + require.Equal(t, entity.ExptRunFinishFinished, finished.Disposition) + } + }) + } +} + +// Real SQL runs on the real transaction; the final Commit boundary fails after +// rolling it back, just as a server that rejects COMMIT before durable commit. +type finishCommitFaultConnection struct { + gorm.ConnPool + gorm.TxCommitter +} + +func (c *finishCommitFaultConnection) Commit() error { + _ = c.Rollback() + return errors.New("commit fault at transaction commit boundary") +} + +func TestRunFinishMySQLCanonicalBindingCorruptDoesNotFinish(t *testing.T) { + for _, kind := range []string{"missing_item", "wrong_run", "wrong_version", "unpublished_terminal"} { + t.Run(kind, func(t *testing.T) { + f := terminalFinishFixture(t) + switch kind { + case "missing_item": + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, 11).Delete(&model.ExptItemResult{}).Error) + case "wrong_run": + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, 11).Update("expt_run_id", f.run-1).Error) + case "wrong_version": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, 11).Update("item_version_id", f.base+999).Error) + case "unpublished_terminal": + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, 11).Update("result_state", 0).Error) + } + before := f.snapshot() + result, err := finishFixture(f) + require.Error(t, err) + require.Nil(t, result) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestRunFinishMySQLLocksExperimentThenRunAndStats(t *testing.T) { + f := terminalFinishFixture(t) + var queries []string + require.NoError(t, f.conn.Callback().Query().After("gorm:query").Register("finish_lock_sequence", func(tx *gorm.DB) { + sql := tx.Statement.SQL.String() + if strings.Contains(sql, "FOR UPDATE") { + queries = append(queries, sql) + } + })) + result, err := finishFixture(f) + require.NoError(t, err) + require.Equal(t, entity.ExptRunFinishFinished, result.Disposition) + require.GreaterOrEqual(t, len(queries), 5) + require.Contains(t, queries[0], "`experiment`") + require.Contains(t, queries[1], "`expt_run_log`") + require.Contains(t, queries[len(queries)-1], "`expt_stats`") + for _, sql := range queries { + require.NotContains(t, sql, "SLEEP") + require.Contains(t, sql, "FOR UPDATE") + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log.go index aae541a020..208e763947 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log.go @@ -5,8 +5,13 @@ package mysql import ( "context" + "time" + + "github.com/bytedance/gg/gptr" + "gorm.io/gorm" "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/query" "github.com/coze-dev/coze-loop/backend/pkg/errorx" @@ -20,6 +25,33 @@ type IExptRunLogDAO interface { Save(ctx context.Context, exptRunLog *model.ExptRunLog, opts ...db.Option) error Update(ctx context.Context, exptID, exptRunID, spaceID int64, ufields map[string]any, opts ...db.Option) error Get(ctx context.Context, exptID, exptRunID int64, opts ...db.Option) (*model.ExptRunLog, error) + AppendRunItemIDs(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64) error + UpdateRunStats(ctx context.Context, exptID, exptRunID, spaceID int64, fields map[string]any, stats *model.ExptStats) error +} + +func (dao *ExptRunLogDAOImpl) UpdateRunStats(ctx context.Context, exptID, exptRunID, spaceID int64, fields map[string]any, stats *model.ExptStats) error { + return dao.provider.Transaction(ctx, func(tx *gorm.DB) error { + opts := []db.Option{db.WithTransaction(tx), db.WithMaster(), db.WithSelectForUpdate()} + expt, err := NewExptDAO(dao.provider).GetByID(ctx, exptID, opts...) + if err != nil { + return err + } + if expt.SpaceID != spaceID || expt.LatestRunID != exptRunID || + (expt.Status != int32(entity.ExptStatus_Pending) && expt.Status != int32(entity.ExptStatus_Processing)) { + return nil + } + run, err := dao.Get(ctx, exptID, exptRunID, opts...) + if err != nil { + return err + } + if run.SpaceID != spaceID || (gptr.Indirect(run.Status) != int64(entity.ExptStatus_Pending) && gptr.Indirect(run.Status) != int64(entity.ExptStatus_Processing)) { + return nil + } + if err := NewExptStatsDAO(dao.provider).UpdateByExptID(ctx, exptID, spaceID, stats, opts...); err != nil { + return err + } + return dao.Update(ctx, exptID, exptRunID, spaceID, fields, opts...) + }, db.WithMaster()) } type ExptRunLogDAOImpl struct { @@ -35,7 +67,13 @@ func NewExptRunLogDAO(db db.Provider) IExptRunLogDAO { func (dao *ExptRunLogDAOImpl) Get(ctx context.Context, exptID, exptRunID int64, opts ...db.Option) (*model.ExptRunLog, error) { var exptRunLog model.ExptRunLog db := dao.provider.NewSession(ctx, opts...) - if err := db.WithContext(ctx).Where("id = ?", exptRunID).First(&exptRunLog).Error; err != nil { + cond := db.WithContext(ctx).Where("id = ?", exptRunID) + // exptID 未知时只按 run id 查：destroy 链路要按 run id 反查 expt（resolveSandboxTaskIDByRunID 传 0）， + // 这里再要求 expt_id 相等会让查询恒空、taskID 变空串，双沙箱的 -orch 销毁请求被丢弃。 + if exptID > 0 { + cond = db.WithContext(ctx).Where("id = ? AND expt_id = ? AND expt_run_id = ?", exptRunID, exptID, exptRunID) + } + if err := cond.First(&exptRunLog).Error; err != nil { return nil, errorx.Wrapf(err, "mget expt fail, expt_id: %v, expt_run_id: %v", exptID, exptRunID) } return &exptRunLog, nil @@ -50,12 +88,40 @@ func (dao *ExptRunLogDAOImpl) Create(ctx context.Context, exptRunLog *model.Expt } func (dao *ExptRunLogDAOImpl) Save(ctx context.Context, exptRunLog *model.ExptRunLog, opts ...db.Option) error { - db := dao.provider.NewSession(ctx, opts...) - if err := db.WithContext(ctx).Save(exptRunLog).Error; err != nil { - return errorx.Wrapf(err, "save expt_run_log fail, model: %v", json.Jsonify(exptRunLog)) - } - logs.CtxInfo(ctx, "save expt_run_log success, model: %v", json.Jsonify(exptRunLog)) - return nil + // Existing callers may hold a stale snapshot; immutable metadata is never saved. + return dao.Update(ctx, exptRunLog.ExptID, exptRunLog.ExptRunID, exptRunLog.SpaceID, map[string]any{ + "status": exptRunLog.Status, "status_message": exptRunLog.StatusMessage, + "pending_cnt": exptRunLog.PendingCnt, "processing_cnt": exptRunLog.ProcessingCnt, + "success_cnt": exptRunLog.SuccessCnt, "fail_cnt": exptRunLog.FailCnt, + "terminated_cnt": exptRunLog.TerminatedCnt, "credit_cost": exptRunLog.CreditCost, + "token_cost": exptRunLog.TokenCost, "updated_at": time.Now(), + }, opts...) +} + +func (dao *ExptRunLogDAOImpl) AppendRunItemIDs(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64) error { + return dao.provider.Transaction(ctx, func(tx *gorm.DB) error { + opts := []db.Option{db.WithTransaction(tx), db.WithMaster(), db.WithSelectForUpdate()} + var row model.ExptRunLog + if err := dao.provider.NewSession(ctx, opts...). + Where("space_id = ? AND expt_id = ? AND expt_run_id = ?", spaceID, exptID, exptRunID).First(&row).Error; err != nil { + return err + } + var items []entity.ExptRunLogItems + if raw := gptr.Indirect(row.ItemIds); len(raw) > 0 { + if err := json.Unmarshal(raw, &items); err != nil { + return errorx.Wrapf(err, "parse current run items") + } + } + run := &entity.ExptRunLog{ItemIds: items} + if err := run.AppendItemIDs(itemIDs); err != nil { + return err + } + raw, err := entity.EncodeExptRunLogItems(run.ItemIds) + if err != nil { + return err + } + return dao.Update(ctx, exptID, exptRunID, spaceID, map[string]any{"item_ids": raw, "updated_at": time.Now()}, opts...) + }, db.WithMaster()) } func (dao *ExptRunLogDAOImpl) Update(ctx context.Context, exptID, exptRunID, spaceID int64, ufields map[string]any, opts ...db.Option) error { diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log_test.go index 3056cc2d43..5b9e45c6e0 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_log_test.go @@ -66,3 +66,24 @@ func TestExptRunLogDAO_Update_OnlyMatchingLiveRun(t *testing.T) { require.Equal(t, want, *got[i].Status, "run log id=%d", got[i].ID) } } + +// Get 在 exptID 未知（<=0）时必须只按 run id 查：destroy 链路用 Get(0, runID) 反查 expt 作为 sandbox taskID， +// 若这里要求 expt_id 相等，查询恒空会让 taskID 变空串、双沙箱的 -orch 销毁请求被丢弃（沙箱泄漏）。 +func TestExptRunLogDAO_Get_ByRunIDOnlyWhenExptUnknown(t *testing.T) { + sqlDB, mock, err := sqlmock.New() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + + provider, err := db.NewDB(mysql.New(mysql.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), + &gorm.Config{SkipDefaultTransaction: true}) + require.NoError(t, err) + + mock.ExpectQuery("SELECT \\* FROM `expt_run_log` WHERE id = \\? AND `expt_run_log`.`deleted_at` IS NULL"). + WithArgs(int64(456), 1). + WillReturnRows(sqlmock.NewRows([]string{"id", "expt_id", "expt_run_id"}).AddRow(456, 123, 456)) + + got, err := NewExptRunLogDAO(provider).Get(context.Background(), 0, 456) + require.NoError(t, err) + require.Equal(t, int64(123), got.ExptID) + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_termination.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_termination.go new file mode 100644 index 0000000000..218f1655bd --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_termination.go @@ -0,0 +1,296 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "fmt" + "time" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/convert" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func (d *ExptRunFinishDAOImpl) GetTerminationItemSource(ctx context.Context, space, expt, run, item int64) (*entity.ExptCurrentItemRunSource, error) { + var source *entity.ExptCurrentItemRunSource + err := d.provider.Transaction(ctx, func(tx *gorm.DB) error { + var err error + source, err = readItemRunSource(tx, space, expt, run, item, true, false) + if err != nil { + return err + } + if source == nil { + return fmt.Errorf("termination item source missing") + } + return ctx.Err() + }, db.WithMaster()) + return source, err +} + +func (d *ExptRunFinishDAOImpl) TerminateCurrentRun(ctx context.Context, r *entity.ExptRunTerminationRequest) (*entity.ExptRunTerminationResult, error) { + if r == nil || r.SpaceID <= 0 || r.ExptID <= 0 || r.RunID <= 0 { + return nil, fmt.Errorf("invalid termination identities") + } + result := &entity.ExptRunTerminationResult{} + err := d.provider.Transaction(ctx, func(tx *gorm.DB) error { + if err := ctx.Err(); err != nil { + return err + } + if r.Begin { + // Drain transactions that already passed the active-run fence before + // atomically publishing Terminating to subsequent execution writers. + var items []model.ExptItemResult + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=? AND expt_run_id=?", r.SpaceID, r.ExptID, r.RunID).Order("item_id").Find(&items).Error; err != nil { + return err + } + } + // Lock the canonical sources before their logs, as execution/publication do. + actual := make([]*entity.ExptCurrentItemRunSource, 0, len(r.Items)) + prior := map[int64]entity.ItemRunState{} + changed := map[int64]bool{} + var lastItem int64 + for _, source := range r.Items { + if source == nil || source.Current.ItemID <= lastItem { + return fmt.Errorf("termination sources must be unique and ascending") + } + lastItem = source.Current.ItemID + var canonical model.ExptItemResult + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=? AND item_id=?", r.SpaceID, r.ExptID, source.Current.ItemID).Take(&canonical).Error; err != nil { + return err + } + prior[source.Current.ItemID] = entity.ItemRunState(canonical.Status) + if source.Current.SpaceID != r.SpaceID || source.Current.ExptID != r.ExptID || source.Current.RunID != r.RunID { + return fmt.Errorf("foreign termination source") + } + s, err := readItemRunSource(tx, r.SpaceID, r.ExptID, r.RunID, source.Current.ItemID, true, true) + if err != nil { + return err + } + if s == nil { + return fmt.Errorf("termination source disappeared") + } + if !currentSourceMatches(source, s) { + if source.Current.RequestID == s.Current.RequestID { + return fmt.Errorf("termination source references advanced") + } + result.NonCurrent = true + return nil + } + actual = append(actual, s) + } + if r.ExpectedSource != nil { + c := r.ExpectedSource.Current + s, err := readItemRunSource(tx, r.SpaceID, r.ExptID, r.RunID, c.ItemID, true, true) + if err != nil { + return err + } + if s == nil { + return fmt.Errorf("termination execution source disappeared") + } + if !currentSourceMatches(r.ExpectedSource, s) { + result.NonCurrent = true + return nil + } + } + var expt model.Experiment + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND id=?", r.SpaceID, r.ExptID).Take(&expt).Error; err != nil { + return err + } + var run model.ExptRunLog + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=? AND expt_run_id=?", r.SpaceID, r.ExptID, r.RunID).Take(&run).Error; err != nil { + return err + } + if expt.LatestRunID != r.RunID { + result.NonCurrent = true + return nil + } + if expt.ExptType != int32(entity.ExptType_Offline) || run.Status == nil { + return fmt.Errorf("invalid offline termination binding") + } + if r.SetupAbort { + // This admission cleanup precedes ExptStart. It never terminates the + // experiment or permits a user cancellation of a Pending experiment. + if r.Begin || r.Finalize || len(r.Items) != 0 || r.ExpectedSource != nil || + expt.Status != int32(entity.ExptStatus_Pending) || *run.Status != int64(entity.ExptStatus_Pending) { + return fmt.Errorf("setup abort requires pending experiment and pending current run") + } + fields := map[string]any{"status": int64(entity.ExptStatus_Failed), "status_message": []byte(r.StatusMessage)} + if err := tx.Model(&model.ExptRunLog{}).Where("id=?", run.ID).UpdateColumns(fields).Error; err != nil { + return err + } + result.Applied = true + return ctx.Err() + } + if entity.IsExptFinished(entity.ExptStatus(expt.Status)) || entity.IsExptFinished(entity.ExptStatus(*run.Status)) { + if !r.Finalize || !entity.IsExptFinished(entity.ExptStatus(expt.Status)) || !entity.IsExptFinished(entity.ExptStatus(*run.Status)) || expt.EndAt == nil { + return fmt.Errorf("termination requires active current run") + } + var err error + result.Experiment, err = convert.NewExptConverter().PO2DO(&expt, nil) + return err + } + active := expt.Status == int32(entity.ExptStatus_Processing) && (*run.Status == int64(entity.ExptStatus_Pending) || *run.Status == int64(entity.ExptStatus_Processing)) + terminating := expt.Status == int32(entity.ExptStatus_Terminating) && *run.Status == int64(entity.ExptStatus_Terminating) + allowed := active || terminating && (r.Begin || r.Finalize) + if !allowed { + return fmt.Errorf("termination requires processing experiment and active run") + } + if r.Begin { + if active { + if err := tx.Model(&model.ExptRunLog{}).Where("id=?", run.ID).UpdateColumns(map[string]any{"status": int64(entity.ExptStatus_Terminating)}).Error; err != nil { + return err + } + if err := tx.Model(&model.Experiment{}).Where("id=? AND latest_run_id=?", expt.ID, r.RunID).UpdateColumns(map[string]any{"status": int32(entity.ExptStatus_Terminating)}).Error; err != nil { + return err + } + } + result.Applied = true + return ctx.Err() + } + for _, source := range actual { + c := source.Current + if c.Status == entity.ItemRunState_Success || c.Status == entity.ItemRunState_Fail { + continue + } + if c.Status == entity.ItemRunState_Terminal && len(source.Turns) == len(source.CanonicalTurns) && c.ResultState != entity.ExptItemResultStateDefault { + continue + } + for _, turn := range source.CanonicalTurns { + if source.Turns[turn.TurnID] != nil { + continue + } + var prepared *entity.ExptTurnResultRunLog + for _, log := range r.MissingTurnLogs { + if log != nil && log.ItemID == c.ItemID && log.TurnID == turn.TurnID { + prepared = log + break + } + } + if c.RequestID != "" || prepared == nil || prepared.ID <= 0 || prepared.SpaceID != c.SpaceID || prepared.ExptID != c.ExptID || prepared.ExptRunID != c.RunID || prepared.ItemVersionID != c.ItemVersionID { + return fmt.Errorf("raw termination turn preparation missing") + } + po, err := convert.NewExptTurnResultRunLogConvertor().DO2PO(prepared) + if err != nil { + return err + } + if err := tx.Create(po).Error; err != nil { + return err + } + } + errBytes := []byte(errno.SerializeErr(errno.NewItemManuallyTerminatedErr())) + scope := "space_id=? AND expt_id=? AND expt_run_id=? AND item_id=? AND item_version_id=?" + args := []any{c.SpaceID, c.ExptID, c.RunID, c.ItemID, c.ItemVersionID} + if !r.NoCompleteItemTurn { + if err := tx.Model(&model.ExptTurnResultRunLog{}).Where(scope, args...).Updates(map[string]any{"status": int32(entity.TurnRunState_Terminal), "err_msg": errBytes}).Error; err != nil { + return err + } + if err := tx.Model(&model.ExptTurnResult{}).Where(scope, args...).Updates(map[string]any{"status": int32(entity.TurnRunState_Terminal)}).Error; err != nil { + return err + } + } + if err := tx.Model(&model.ExptItemResultRunLog{}).Where(scope, args...).Updates(map[string]any{"status": int32(entity.ItemRunState_Terminal), "result_state": int32(entity.ExptItemResultStateLogged), "err_msg": errBytes}).Error; err != nil { + return err + } + if !r.NoCompleteItemTurn { + if err := tx.Model(&model.ExptItemResult{}).Where(scope, args...).Updates(map[string]any{"status": int32(entity.ItemRunState_Terminal), "err_msg": errBytes}).Error; err != nil { + return err + } + } + result.Applied = true + changed[c.ItemID] = true + } + // Row cancellation changes only its own buckets; publication sees the + // same canonical status and contributes no second delta. + + if result.Applied || r.Finalize { + var stats model.ExptStats + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=?", r.SpaceID, r.ExptID).Take(&stats).Error; err != nil { + return err + } + var fields map[string]any + if r.Finalize { + var items []model.ExptItemResult + if err := tx.Where("space_id=? AND expt_id=?", r.SpaceID, r.ExptID).Find(&items).Error; err != nil { + return err + } + var counts finishCounts + for _, item := range items { + counts.add(item.Status) + } + fields = counts.fields("", nil) + } else { + columns := map[entity.ItemRunState]string{entity.ItemRunState_Queueing: "pending_cnt", entity.ItemRunState_Processing: "processing_cnt", entity.ItemRunState_Success: "success_cnt", entity.ItemRunState_Fail: "fail_cnt", entity.ItemRunState_Terminal: "terminated_cnt"} + deltas := map[string]int{} + for _, source := range actual { + if !changed[source.Current.ItemID] || prior[source.Current.ItemID] == entity.ItemRunState_Terminal { + continue + } + from := columns[prior[source.Current.ItemID]] + if from == "" { + return fmt.Errorf("termination stats source invalid") + } + deltas[from]-- + deltas["terminated_cnt"]++ + } + fields = map[string]any{} + for column, delta := range deltas { + if delta != 0 { + fields[column] = gorm.Expr(column+" + ?", delta) + } + } + } + if len(fields) > 0 { + if err := tx.Model(&stats).UpdateColumns(fields).Error; err != nil { + return err + } + } + } + + if r.Finalize { + if r.Status != entity.ExptStatus_Terminated && r.Status != entity.ExptStatus_Failed { + return fmt.Errorf("invalid explicit termination status") + } + var turns []model.ExptTurnResult + if err := tx.Where("space_id=? AND expt_id=?", r.SpaceID, r.ExptID).Find(&turns).Error; err != nil { + return err + } + var turnCounts finishCounts + for _, turn := range turns { + turnCounts.addTurn(entity.TurnRunState(turn.Status)) + } + fields := turnCounts.fields("status", int64(r.Status)) + if r.StatusMessage != "" { + fields["status_message"] = []byte(r.StatusMessage) + } + if err := tx.Model(&model.ExptRunLog{}).Where("id=?", run.ID).UpdateColumns(fields).Error; err != nil { + return err + } + now := time.Now() + fields = map[string]any{"status": int32(r.Status), "end_at": now} + if r.StatusMessage != "" { + fields["status_message"] = []byte(r.StatusMessage) + } + if err := tx.Model(&model.Experiment{}).Where("id=? AND latest_run_id=?", expt.ID, r.RunID).UpdateColumns(fields).Error; err != nil { + return err + } + expt.Status = int32(r.Status) + expt.EndAt = &now + result.Applied = true + var err error + result.Experiment, err = convert.NewExptConverter().PO2DO(&expt, nil) + if err != nil { + return err + } + } + return ctx.Err() + }, db.WithMaster()) + if err != nil { + return nil, err + } + return result, nil +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_termination_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_termination_test.go new file mode 100644 index 0000000000..81166435cf --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_run_termination_test.go @@ -0,0 +1,185 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "errors" + "testing" + "time" + + "gorm.io/gorm/clause" + + "github.com/bytedance/gg/gptr" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestRunCancelMySQLRollbackEveryWrite(t *testing.T) { + for _, table := range []string{"expt_turn_result_run_log", "expt_turn_result", "expt_item_result_run_log", "expt_item_result", "expt_stats", "expt_run_log", "experiment", "commit"} { + t.Run(table, func(t *testing.T) { + f := newCurrentResultFixture(t, "9f6fe7ea-7db6-4d16-8f5b-2d1344556677") + source := f.source() + before := f.snapshot() + provider := f.provider + if table == "commit" { + provider = finishFaultProvider{provider} + } else { + require.NoError(t, f.conn.Callback().Update().Before("gorm:update").Register("termination_fault", func(tx *gorm.DB) { + if tx.Statement.Table == table { + _ = tx.AddError(errors.New("termination write fault")) + } + })) + } + port := NewExptRunFinishDAO(provider).(repo.IExptRunTerminationRepo) + result, err := port.TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Items: []*entity.ExptCurrentItemRunSource{source}, Finalize: true, Status: entity.ExptStatus_Terminated}) + require.Error(t, err) + require.Nil(t, result) + require.Equal(t, before, f.snapshot()) + }) + } +} + +func TestTerminateSourceOldRequestCannotEndSameRun(t *testing.T) { + f := newCurrentResultFixture(t, "9f6fe7ea-7db6-4d16-8f5b-2d1344556677") + source := f.source() + f.token("aa6fe7ea-7db6-4d16-8f5b-2d1344556677") + before := f.snapshot() + port := NewExptRunFinishDAO(f.provider).(repo.IExptRunTerminationRepo) + for _, whole := range []bool{false, true} { + result, err := port.TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Items: []*entity.ExptCurrentItemRunSource{source}, ExpectedSource: source, Finalize: whole, Status: entity.ExptStatus_Terminated}) + require.NoError(t, err) + require.True(t, result.NonCurrent) + require.Equal(t, before, f.snapshot()) + } +} + +func TestRunCancelExecutionWriterRemainsFenced(t *testing.T) { + f := newCurrentResultFixture(t, "") + source := f.source() + port := NewExptRunFinishDAO(f.provider).(repo.IExptRunTerminationRepo) + _, err := port.TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Begin: true}) + require.NoError(t, err) + before := f.snapshot() + applied, err := f.port.CompleteCurrentItemRun(context.Background(), source, &entity.ExptCurrentItemRunWrite{Status: entity.ItemRunState_Success}) + require.NoError(t, err) + require.False(t, applied) + require.Equal(t, before, f.snapshot()) + result, err := port.TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Items: []*entity.ExptCurrentItemRunSource{source}, Finalize: true, Status: entity.ExptStatus_Terminated}) + require.NoError(t, err) + require.True(t, result.Applied) +} + +func TestTerminateRawZeroVersionAndMissingLogs(t *testing.T) { + f := newCurrentResultFixture(t, "") + for _, table := range []string{"expt_item_result", "expt_item_result_run_log", "expt_turn_result", "expt_turn_result_run_log", "expt_item_ref"} { + require.NoError(t, f.conn.Table(table).Where("space_id=?", f.space).Update("item_version_id", 0).Error) + } + require.NoError(t, f.conn.Model(&model.ExptTurnResult{}).Where("space_id=?", f.space).Update("target_result_id", 0).Error) + require.NoError(t, f.conn.Exec("DELETE FROM expt_turn_result_run_log WHERE space_id=? AND expt_id=?", f.space, f.expt).Error) + source := f.source() + require.Zero(t, source.Current.ItemVersionID) + missing := []*entity.ExptTurnResultRunLog{} + for i, turn := range source.CanonicalTurns { + missing = append(missing, &entity.ExptTurnResultRunLog{ID: f.base + 9000 + int64(i), SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item, ItemVersionID: 0, TurnID: turn.TurnID, Status: entity.TurnRunState_Terminal}) + } + port := NewExptRunFinishDAO(f.provider).(repo.IExptRunTerminationRepo) + result, err := port.TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Items: []*entity.ExptCurrentItemRunSource{source}, MissingTurnLogs: missing}) + require.NoError(t, err) + require.True(t, result.Applied) + var item model.ExptItemResultRunLog + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&item).Error) + require.Equal(t, int32(entity.ItemRunState_Terminal), item.Status) + require.Equal(t, int32(entity.ExptItemResultStateLogged), gptr.Indirect(item.ResultState)) +} + +func TestNewRunResetRemovesOnlyManualContextAtMigration(t *testing.T) { + f := newCurrentResultFixture(t, "9f6fe7ea-7db6-4d16-8f5b-2d1344556677") + replacement := f.run + 100 + require.NoError(t, f.conn.Create(&model.ExptRunLog{ID: replacement, SpaceID: f.space, ExptID: f.expt, ExptRunID: replacement, Status: gptr.Of(int64(entity.ExptStatus_Pending)), Mode: gptr.Of(int32(entity.EvaluationModeRetryItems))}).Error) + require.NoError(t, f.conn.Model(&model.Experiment{}).Where("space_id=?", f.space).Update("latest_run_id", replacement).Error) + before := f.snapshot() + require.NoError(t, NewExptItemResultDAO(f.provider).UpdateItemsResult(context.Background(), f.space, f.expt, []int64{f.item}, map[string]any{"expt_run_id": replacement, "status": int32(entity.ItemRunState_Queueing)})) + var item model.ExptItemResult + require.NoError(t, f.conn.Where("space_id=? AND item_id=?", f.space, f.item).Take(&item).Error) + ext, err := entity.ParseExptItemResultExt(gptr.Indirect(item.Ext)) + require.NoError(t, err) + require.Equal(t, map[string]string{"other": "keep"}, ext) + after := f.snapshot() + require.Equal(t, before["expt_run_log"], after["expt_run_log"]) + require.Equal(t, before["expt_item_result_run_log"], after["expt_item_result_run_log"]) +} + +func TestTerminateRowPreservesOtherItemStatsBuckets(t *testing.T) { + f := newCurrentResultFixture(t, "") + source := f.source() + require.NoError(t, f.conn.Model(&model.ExptStats{}).Where("space_id=?", f.space).Updates(map[string]any{"pending_cnt": 7, "processing_cnt": 9, "success_cnt": 11, "fail_cnt": 13, "terminated_cnt": 17}).Error) + _, err := NewExptRunFinishDAO(f.provider).(repo.IExptRunTerminationRepo).TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Items: []*entity.ExptCurrentItemRunSource{source}}) + require.NoError(t, err) + var stats model.ExptStats + require.NoError(t, f.conn.Where("space_id=?", f.space).Take(&stats).Error) + require.Equal(t, int32(7), stats.PendingCnt) + require.Equal(t, int32(8), stats.ProcessingCnt) + require.Equal(t, int32(11), stats.SuccessCnt) + require.Equal(t, int32(13), stats.FailCnt) + require.Equal(t, int32(18), stats.TerminatedCnt) +} + +func TestRunCancelNoCompleteTurnsKeepsCanonicalProjection(t *testing.T) { + f := newCurrentResultFixture(t, "") + source := f.source() + before := f.snapshot() + _, err := NewExptRunFinishDAO(f.provider).(repo.IExptRunTerminationRepo).TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Items: []*entity.ExptCurrentItemRunSource{source}, Finalize: true, Status: entity.ExptStatus_Failed, NoCompleteItemTurn: true}) + require.NoError(t, err) + after := f.snapshot() + require.Equal(t, before["expt_turn_result"], after["expt_turn_result"]) + require.Equal(t, before["expt_item_result"], after["expt_item_result"]) +} + +func TestTerminateSameRequestRefAdvanceMustRetry(t *testing.T) { + f := newCurrentResultFixture(t, "aa6fe7ea-7db6-4d16-8f5b-2d1344556677") + source := f.source() + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=?", f.space).Update("target_result_id", f.base+777).Error) + before := f.snapshot() + _, err := NewExptRunFinishDAO(f.provider).(repo.IExptRunTerminationRepo).TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Items: []*entity.ExptCurrentItemRunSource{source}, Finalize: true, Status: entity.ExptStatus_Terminated}) + require.Error(t, err, "same execution progressed while cancellation waited; retain retry") + require.Equal(t, before, f.snapshot()) +} + +func TestRunCancelBeginSerializesInflightCanonicalWriter(t *testing.T) { + f := newCurrentResultFixture(t, "") + held := make(chan struct{}) + release := make(chan struct{}) + writer := make(chan error, 1) + go func() { + writer <- f.provider.Transaction(context.Background(), func(tx *gorm.DB) error { + var item model.ExptItemResult + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND item_id=?", f.space, f.item).Take(&item).Error; err != nil { + return err + } + close(held) + <-release + return nil + }) + }() + <-held + done := make(chan error, 1) + go func() { + _, err := NewExptRunFinishDAO(f.provider).(repo.IExptRunTerminationRepo).TerminateCurrentRun(context.Background(), &entity.ExptRunTerminationRequest{SpaceID: f.space, ExptID: f.expt, RunID: f.run, Begin: true}) + done <- err + }() + select { + case err := <-done: + close(release) + require.NoError(t, <-writer) + require.Fail(t, "cancel begin committed before inflight canonical writer completed", err) + return + case <-time.After(150 * time.Millisecond): + } + close(release) + require.NoError(t, <-writer) + require.NoError(t, <-done) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_runlog_receipt_writer_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_runlog_receipt_writer_test.go new file mode 100644 index 0000000000..501e01841c --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_runlog_receipt_writer_test.go @@ -0,0 +1,135 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "database/sql/driver" + "encoding/json" + "errors" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/bytedance/gg/gptr" + "github.com/stretchr/testify/require" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +type appendedRunItems struct{} + +func (appendedRunItems) Match(value driver.Value) bool { + bytes, ok := value.([]byte) + if !ok { + return false + } + var items []entity.ExptRunLogItems + if json.Unmarshal(bytes, &items) != nil || len(items) != 3 { + return false + } + return len(items[0].ItemIDs) == 1 && items[0].ItemIDs[0] == 11 && items[1].ManualRetryReceipt != nil && + items[1].ManualRetryReceipt.RequestID == "rq1" && len(items[2].ItemIDs) == 1 && items[2].ItemIDs[0] == 12 && + json.Valid(bytes) && items[1].ManualRetryReceipt.PayloadHash == "hash" +} + +func TestAppendRunItemIDsLocksCurrentJSON(t *testing.T) { + provider, mock := newManualRetrySQLProvider(t) + dao, ok := any(NewExptRunLogDAO(provider)).(interface { + AppendRunItemIDs(context.Context, int64, int64, int64, []int64) error + }) + require.True(t, ok, "existing run append must lock and read current JSON") + raw := `[{"ItemIDs":[11],"CreateAt":1700000000},{"ItemIDs":null,"CreateAt":1700000001,"manual_retry_receipt":{"request_id":"rq1","payload_hash":"hash","accepted_count":1,"rejected_items":[],"snapshot_at_ms":1700000000123},"future":{"id":9223372036854775806}}]` + mock.ExpectBegin() + mock.ExpectQuery("SELECT .* FROM `expt_run_log` WHERE \\(space_id = \\? AND expt_id = \\? AND expt_run_id = \\?\\).*FOR UPDATE"). + WithArgs(int64(9), int64(7), int64(8), 1).WillReturnRows(sqlmock.NewRows([]string{"id", "space_id", "expt_id", "expt_run_id", "item_ids"}).AddRow(8, 9, 7, 8, []byte(raw))) + mock.ExpectExec("UPDATE `expt_run_log` SET `item_ids`=\\?,`updated_at`=\\? WHERE .*`space_id` = \\?.*`expt_id` = \\?.*`expt_run_id` = \\?"). + WithArgs(appendedRunItems{}, sqlmock.AnyArg(), int64(9), int64(7), int64(8)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + require.NoError(t, dao.AppendRunItemIDs(context.Background(), 7, 8, 9, []int64{12})) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestAppendRunItemIDsRejectsCorruptOrDuplicate(t *testing.T) { + for _, tc := range []struct { + name, raw string + ids []int64 + }{ + {"corrupt", `{`, []int64{12}}, {"duplicate", `[{"ItemIDs":[11]}]`, []int64{11}}, + } { + t.Run(tc.name, func(t *testing.T) { + provider, mock := newManualRetrySQLProvider(t) + dao, ok := any(NewExptRunLogDAO(provider)).(interface { + AppendRunItemIDs(context.Context, int64, int64, int64, []int64) error + }) + require.True(t, ok) + mock.ExpectBegin() + mock.ExpectQuery("SELECT .*FOR UPDATE").WillReturnRows(sqlmock.NewRows([]string{"id", "space_id", "expt_id", "expt_run_id", "item_ids"}).AddRow(8, 9, 7, 8, []byte(tc.raw))) + mock.ExpectRollback() + require.Error(t, dao.AppendRunItemIDs(context.Background(), 7, 8, 9, tc.ids)) + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} + +func TestRunLogSavePreservesCurrentReceipt(t *testing.T) { + provider, mock := newManualRetrySQLProvider(t) + mock.ExpectExec("UPDATE `expt_run_log` SET `credit_cost`=\\?,`fail_cnt`=\\?,`pending_cnt`=\\?,`processing_cnt`=\\?,`status`=\\?,`status_message`=\\?,`success_cnt`=\\?,`terminated_cnt`=\\?,`token_cost`=\\?,`updated_at`=\\? WHERE .*`space_id` = \\?.*`expt_id` = \\?.*`expt_run_id` = \\?"). + WithArgs(float64(1), int32(0), int32(0), int32(0), int64(13), nil, int32(1), int32(0), nil, sqlmock.AnyArg(), int64(9), int64(7), int64(8)).WillReturnError(errors.New("write failed")) + err := NewExptRunLogDAO(provider).Save(context.Background(), &model.ExptRunLog{ID: 8, SpaceID: 9, ExptID: 7, ExptRunID: 8, CreditCost: 1, SuccessCnt: 1, Status: gptr.Of(int64(13)), ItemIds: gptr.Of([]byte(`[]`))}) + require.ErrorContains(t, err, "write failed") + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRunLogSaveAndAppendInterleaveKeepsMetadata(t *testing.T) { + ctx := context.Background() + provider := db.NewTestDB(t, &model.ExptRunLog{}) + require.NotNil(t, provider) + dao := NewExptRunLogDAO(provider) + row := &model.ExptRunLog{ID: 8, SpaceID: 9, ExptID: 7, ExptRunID: 8, Status: gptr.Of(int64(3)), ItemIds: gptr.Of([]byte(`[{"ItemIDs":[11]}]`))} + require.NoError(t, dao.Create(ctx, row)) + stale, err := dao.Get(ctx, 7, 8) + require.NoError(t, err) + raw := `[{"ItemIDs":[11],"CreateAt":null},{"ItemIDs":null,"CreateAt":1700000001,"manual_retry_receipt":{"request_id":"rq1","payload_hash":"hash","accepted_count":1,"rejected_items":[],"snapshot_at_ms":1700000000123},"future":{"id":9223372036854775806}}]` + require.NoError(t, dao.Update(ctx, 7, 8, 9, map[string]any{"item_ids": []byte(raw)})) + stale.Status = gptr.Of(int64(13)) + require.NoError(t, dao.Save(ctx, stale)) + got, err := dao.Get(ctx, 7, 8) + require.NoError(t, err) + require.JSONEq(t, raw, string(*got.ItemIds)) + appendDAO, ok := any(dao).(interface { + AppendRunItemIDs(context.Context, int64, int64, int64, []int64) error + }) + require.True(t, ok) + require.NoError(t, appendDAO.AppendRunItemIDs(ctx, 7, 8, 9, []int64{12})) + require.NoError(t, dao.Save(ctx, stale)) + got, err = dao.Get(ctx, 7, 8) + require.NoError(t, err) + var items []entity.ExptRunLogItems + require.NoError(t, json.Unmarshal(*got.ItemIds, &items)) + require.Equal(t, []int64{11, 12}, (&entity.ExptRunLog{ItemIds: items}).GetItemIDs()) + require.Equal(t, "rq1", items[1].ManualRetryReceipt.RequestID) + var fields []map[string]json.RawMessage + require.NoError(t, json.Unmarshal(*got.ItemIds, &fields)) + require.JSONEq(t, `{"id":9223372036854775806}`, string(fields[1]["future"])) +} + +func TestUpdateRunStatsLocksControlRowsAndRollsBackOnError(t *testing.T) { + provider, mock := newManualRetrySQLProvider(t) + mock.ExpectBegin() + mock.ExpectQuery("SELECT .* FROM `experiment` WHERE .*`id` = \\?.*FOR UPDATE").WithArgs(int64(7), 1). + WillReturnRows(sqlmock.NewRows([]string{"id", "space_id", "latest_run_id", "status"}).AddRow(7, 9, 8, int32(entity.ExptStatus_Processing))) + mock.ExpectQuery("SELECT .* FROM `expt_run_log` WHERE \\(id = \\? AND expt_id = \\? AND expt_run_id = \\?\\).*FOR UPDATE").WithArgs(int64(8), int64(7), int64(8), 1). + WillReturnRows(sqlmock.NewRows([]string{"id", "space_id", "expt_id", "expt_run_id", "status"}).AddRow(8, 9, 7, 8, int64(entity.ExptStatus_Processing))) + mock.ExpectExec("UPDATE `expt_stats` SET .*`fail_cnt`=\\?.*`pending_cnt`=\\?.*`processing_cnt`=\\?.*`success_cnt`=\\?.*`terminated_cnt`=\\?.*WHERE .*`expt_id` = \\?.*`space_id` = \\?"). + WithArgs(int32(0), int32(0), int32(0), int32(1), int32(0), sqlmock.AnyArg(), int64(7), int64(9)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("UPDATE `expt_run_log` SET `status`=\\?,`success_cnt`=\\? WHERE .*`space_id` = \\?.*`expt_id` = \\?.*`expt_run_id` = \\?"). + WithArgs(int64(entity.ExptStatus_Processing), int32(1), int64(9), int64(7), int64(8)).WillReturnError(errors.New("run update failed")) + mock.ExpectRollback() + err := NewExptRunLogDAO(provider).UpdateRunStats(context.Background(), 7, 8, 9, + map[string]any{"status": int64(entity.ExptStatus_Processing), "success_cnt": int32(1)}, &model.ExptStats{SuccessCnt: 1}) + require.ErrorContains(t, err, "run update failed") + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_scan_observation_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_scan_observation_test.go new file mode 100644 index 0000000000..5de4cbbab8 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_scan_observation_test.go @@ -0,0 +1,444 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package mysql + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + drivermysql "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + gormmysql "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" + + "github.com/coze-dev/coze-loop/backend/infra/db" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +func newScanObservationFixture(t *testing.T) *currentResultFixture { + t.Helper() + f := newCurrentResultFixture(t, "rq1") + for _, item := range []int64{f.item, f.item + 1} { + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, item).Update("status", int32(entity.ItemRunState_Success)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, item).Updates(map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": int32(entity.ExptItemResultStateLogged)}).Error) + } + require.NoError(t, f.conn.Create(&model.ExptTurnResult{ID: f.base + 450, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, TurnID: 0, Status: int32(entity.TurnRunState_Success)}).Error) + require.NoError(t, f.conn.Create(&model.ExptTurnResultRunLog{ID: f.base + 550, SpaceID: f.space, ExptID: f.expt, ExptRunID: f.run, ItemID: f.item + 1, ItemVersionID: f.version, TurnID: 0, Status: int32(entity.TurnRunState_Success)}).Error) + return f +} + +func observationQueryHasItem(tx *gorm.DB, itemID int64) bool { + where, ok := tx.Statement.Clauses["WHERE"].Expression.(clause.Where) + if !ok { + return false + } + for _, condition := range where.Exprs { + if expr, ok := condition.(clause.Expr); ok { + for _, value := range expr.Vars { + if value == itemID { + return true + } + } + } + } + return false +} + +// A scanner blocked on B must not retain A's canonical/source locks. +// This uses a real separate SQL transaction holding B, and a real UPDATE to A. +func TestScanObservationMySQLDoesNotAccumulateLocks(t *testing.T) { + f := newScanObservationFixture(t) + ctx := context.Background() + blocked := make(chan struct{}) + release := make(chan struct{}) + blockerDone := make(chan error, 1) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + defer unblock() + go func() { + blockerDone <- f.provider.Transaction(ctx, func(tx *gorm.DB) error { + var b model.ExptItemResult + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item+1).Take(&b).Error; err != nil { + return err + } + close(blocked) + <-release + return nil + }, db.WithMaster()) + }() + select { + case <-blocked: + case err := <-blockerDone: + t.Fatalf("SQL blocker failed: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("SQL blocker not reached") + } + secondItem := make(chan struct{}, 1) + locks := 0 + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register("scan_probe_second_item", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_result" && observationQueryHasItem(tx, f.item+1) { + select { + case secondItem <- struct{}{}: + default: + } + } + })) + require.NoError(t, f.conn.Callback().Query().After("gorm:query").Register("scan_count_source_locks", func(tx *gorm.DB) { + if strings.Contains(tx.Statement.SQL.String(), "FOR UPDATE") { + locks++ + } + })) + defer func() { + require.NoError(t, f.conn.Callback().Query().Remove("scan_probe_second_item")) + require.NoError(t, f.conn.Callback().Query().Remove("scan_count_source_locks")) + }() + type scanResult struct { + rows []*entity.ExptItemCurrentRequest + err error + } + scanned := make(chan scanResult, 1) + go func() { + rows, err := NewExptItemResultDAO(f.provider).ScanCurrentRequests(ctx, f.space, f.expt, f.run, false) + scanned <- scanResult{rows, err} + }() + select { + case <-secondItem: + case <-time.After(5 * time.Second): + unblock() + t.Fatal("scanner did not reach second Logged item") + } + updateCtx, cancel := context.WithTimeout(ctx, 400*time.Millisecond) + updateErr := f.provider.NewSession(updateCtx, db.WithMaster()).Model(&model.ExptItemResult{}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Update("log_id", "independent-current-write").Error + cancel() + unblock() + require.NoError(t, <-blockerDone) + result := <-scanned + require.NoError(t, result.err) + require.NoError(t, updateErr, "a blocked scan must not hold previously observed Logged item rows until the entire scan completes") + require.Len(t, result.rows, 3, "all Logged and Queueing items must remain enumerated") + require.Zero(t, locks, "source observation must not emit locking SELECTs") + for _, row := range result.rows { + if row.ResultState == entity.ExptItemResultStateLogged { + require.True(t, row.PublicationObserved) + require.NotNil(t, row.Publication) + } + } + t.Logf("REAL_MYSQL_SCAN_OBSERVATION locks=%d logged_items=2 blocked_second_item_did_not_block_first_update", locks) +} + +// The test wrapper records the real BeginTx options while forwarding to MySQL. +// The concurrency assertions prove either a stable RR snapshot or RC rejection. +type observationProvider struct { + base db.Provider + mu sync.Mutex + options []sql.TxOptions + rejectExplicit bool +} +type observationConnPool struct { + *sql.DB + owner *observationProvider +} + +func (p *observationConnPool) BeginTx(ctx context.Context, options *sql.TxOptions) (*sql.Tx, error) { + option := sql.TxOptions{} + if options != nil { + option = *options + } + p.owner.mu.Lock() + p.owner.options = append(p.owner.options, option) + p.owner.mu.Unlock() + if p.owner.rejectExplicit && (option.Isolation != sql.LevelDefault || option.ReadOnly) { + return nil, fmt.Errorf("SimulatedNDBBegin: Error 1105 near LEVEL REPEATABLE READ (explicit isolation/readOnly rejected)") + } + return p.DB.BeginTx(ctx, options) +} + +func (p *observationProvider) NewSession(ctx context.Context, options ...db.Option) *gorm.DB { + session := p.base.NewSession(ctx, options...) + if pool, ok := session.Statement.ConnPool.(*sql.DB); ok { + session.Statement.ConnPool = &observationConnPool{DB: pool, owner: p} + } + return session +} + +func (p *observationProvider) Transaction(ctx context.Context, callback func(*gorm.DB) error, options ...db.Option) error { + return p.NewSession(ctx, options...).Transaction(callback) +} + +func newObservationProvider(t *testing.T, isolation string) *observationProvider { + t.Helper() + cfg, err := drivermysql.ParseDSN(os.Getenv("EXPT_RETRY_MYSQL_DSN")) + require.NoError(t, err) + if cfg.Params == nil { + cfg.Params = map[string]string{} + } + // The override is connection-local on this test-owned pool, never global. + cfg.Params["transaction_isolation"] = "'" + isolation + "'" + base, err := db.NewDB(gormmysql.Open(cfg.FormatDSN()), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + conn := base.NewSession(context.Background(), db.WithMaster()) + t.Cleanup(func() { pool, err := conn.DB(); require.NoError(t, err); require.NoError(t, pool.Close()) }) + var observed string + require.NoError(t, conn.Raw("SELECT @@session.transaction_isolation").Scan(&observed).Error) + require.Equal(t, isolation, observed) + var engine string + require.NoError(t, conn.Raw("SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='expt_item_result'").Scan(&engine).Error) + require.Equal(t, "InnoDB", engine) + t.Logf("REAL_MYSQL_ISOLATION session_default=%s engine=%s", observed, engine) + return &observationProvider{base: base} +} + +// A concurrent writer commits a new invocation after the initial run/item scan. +// RR preserves the old snapshot; RC must discard all rows when the source changes. +func TestScanObservationMySQLConsistentAcrossSessionIsolation(t *testing.T) { + for _, isolation := range []string{"REPEATABLE-READ", "READ-COMMITTED"} { + t.Run(isolation, func(t *testing.T) { + f := newScanObservationFixture(t) + port := NewExptResultPublishDAO(f.provider) + expected, err := port.GetItemResultPublishSource(context.Background(), f.space, f.expt, f.run, f.item) + require.NoError(t, err) + provider := newObservationProvider(t, isolation) + conn := provider.NewSession(context.Background(), db.WithMaster()) + joined := make(chan struct{}) + resume := make(chan struct{}) + var once sync.Once + defer func() { once.Do(func() { close(resume) }) }() + locks := 0 + var txMetadata struct { + IsolationLevel string + AccessMode string + } + var metadataErr error + require.NoError(t, conn.Callback().Query().After("gorm:query").Register("scan_pause_after_initial_join", func(tx *gorm.DB) { + query := tx.Statement.SQL.String() + if strings.Contains(query, "FOR UPDATE") { + locks++ + } + if strings.Contains(query, "AS l") && strings.Contains(query, "LEFT JOIN") { + metadataErr = tx.Session(&gorm.Session{NewDB: true}).Raw("SELECT et.ISOLATION_LEVEL AS isolation_level,et.ACCESS_MODE AS access_mode FROM performance_schema.events_transactions_current AS et JOIN performance_schema.threads AS th ON th.THREAD_ID=et.THREAD_ID WHERE th.PROCESSLIST_ID=CONNECTION_ID() LIMIT 1").Scan(&txMetadata).Error + close(joined) + <-resume + } + })) + defer func() { require.NoError(t, conn.Callback().Query().Remove("scan_pause_after_initial_join")) }() + type scanResult struct { + rows []*entity.ExptItemCurrentRequest + err error + } + done := make(chan scanResult, 1) + go func() { + rows, err := NewExptItemResultDAO(provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + done <- scanResult{rows, err} + }() + select { + case <-joined: + case <-time.After(5 * time.Second): + t.Fatal("initial consistent scan not reached") + } + writeErr := f.provider.Transaction(context.Background(), func(tx *gorm.DB) error { + if err := tx.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("log_id", "new-invocation-item").Error; err != nil { + return err + } + if err := tx.Model(&model.ExptTurnResult{}).Where("space_id=? AND item_id=? AND turn_id=0", f.space, f.item).Updates(map[string]any{"target_result_id": f.base + 71, "log_id": "new-invocation-turn"}).Error; err != nil { + return err + } + refs, _ := json.Marshal(&entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: f.base + 51, Alias: "alias", RecordID: f.base + 81}}}) + if err := tx.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=? AND turn_id=0", f.space, f.item).Updates(map[string]any{"target_result_id": f.base + 71, "evaluator_result_ids": refs, "log_id": "new-invocation-log"}).Error; err != nil { + return err + } + if err := tx.Model(&model.ExptTurnEvaluatorResultRef{}).Where("space_id=? AND expt_turn_result_id=?", f.space, f.base+400).Update("evaluator_result_id", f.base+81).Error; err != nil { + return err + } + return tx.Model(&model.ExptItemRef{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("item_config", []byte(`{"target_source_space_id":99}`)).Error + }, db.WithMaster()) + once.Do(func() { close(resume) }) + result := <-done + require.NoError(t, writeErr) + if isolation == "READ-COMMITTED" { + require.Error(t, result.err, "changed RC observation must be retried") + require.Nil(t, result.rows, "no mixed or partial source can escape") + } else { + require.NoError(t, result.err) + require.Len(t, result.rows, 3) + } + var observed *entity.ExptItemResultPublishSource + for _, row := range result.rows { + if row.ItemID == f.item { + observed = row.Publication + } + } + if isolation == "REPEATABLE-READ" { + require.Equal(t, expected, observed, "initial request/turn/ref/config observation must not be replaced by a new same-request invocation") + } else { + require.Nil(t, observed) + } + provider.mu.Lock() + options := append([]sql.TxOptions(nil), provider.options...) + provider.mu.Unlock() + require.Len(t, options, 1) + require.Equal(t, sql.LevelDefault, options[0].Isolation) + require.False(t, options[0].ReadOnly, "readonly semantics must not require NDB transaction option SQL") + require.NoError(t, metadataErr) + require.Equal(t, strings.ReplaceAll(isolation, "-", " "), txMetadata.IsolationLevel) + require.Equal(t, "READ WRITE", txMetadata.AccessMode) + require.Zero(t, locks) + var unchangedDefault string + require.NoError(t, provider.NewSession(context.Background(), db.WithMaster()).Raw("SELECT @@session.transaction_isolation").Scan(&unchangedDefault).Error) + require.Equal(t, isolation, unchangedDefault, "one-transaction isolation must not change connection defaults") + t.Logf("REAL_MYSQL_SCAN_SNAPSHOT session_default=%s BeginTx=%+v performance_schema=%+v locking_queries=%d", isolation, options[0], txMetadata, locks) + current, err := port.IsItemResultPublishSourceCurrent(context.Background(), expected) + require.NoError(t, err) + require.False(t, current, "notification validation rejects the old frozen source after the writer committed") + before := f.snapshot() + applied, err := port.CommitItemResult(context.Background(), &entity.ExptItemResultPublication{Source: expected}) + require.NoError(t, err) + require.False(t, applied) + require.Equal(t, before, f.snapshot(), "old scan cannot publish against new references") + }) + } +} + +// Removing the scan locks must not remove the final validation/commit locks. +// Each operation waits on an actual canonical FOR UPDATE held by a writer, +// and makes no writes once the writer commits a different original source. +func TestScanObservationMySQLPublicationStillLocks(t *testing.T) { + for _, operation := range []string{"validate", "commit"} { + t.Run(operation, func(t *testing.T) { + f := newScanObservationFixture(t) + ctx := context.Background() + port := NewExptResultPublishDAO(f.provider) + source, err := port.GetItemResultPublishSource(ctx, f.space, f.expt, f.run, f.item) + require.NoError(t, err) + ready := make(chan struct{}) + release := make(chan struct{}) + writerDone := make(chan error, 1) + var once sync.Once + defer func() { once.Do(func() { close(release) }) }() + go func() { + writerDone <- f.provider.Transaction(ctx, func(tx *gorm.DB) error { + var row model.ExptItemResult + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, f.item).Take(&row).Error; err != nil { + return err + } + if err := tx.Model(&row).Update("log_id", "changed-under-canonical-lock").Error; err != nil { + return err + } + close(ready) + <-release + return nil + }, db.WithMaster()) + }() + select { + case <-ready: + case err := <-writerDone: + t.Fatalf("writer failed: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("writer did not acquire canonical row") + } + started := make(chan struct{}, 1) + require.NoError(t, f.conn.Callback().Query().Before("gorm:query").Register("publication_probe_lock", func(tx *gorm.DB) { + if tx.Statement.Table == "expt_item_result" && observationQueryHasItem(tx, f.item) { + locking, ok := tx.Statement.Clauses["FOR"].Expression.(clause.Locking) + if ok && locking.Strength == "UPDATE" { + select { + case started <- struct{}{}: + default: + } + } + } + })) + defer func() { require.NoError(t, f.conn.Callback().Query().Remove("publication_probe_lock")) }() + type result struct { + applied bool + err error + } + done := make(chan result, 1) + go func() { + var ok bool + var err error + if operation == "validate" { + ok, err = port.IsItemResultPublishSourceCurrent(ctx, source) + } else { + ok, err = port.CommitItemResult(ctx, &entity.ExptItemResultPublication{Source: source}) + } + done <- result{ok, err} + }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("publication did not attempt canonical FOR UPDATE") + } + select { + case result := <-done: + t.Fatalf("%s bypassed writer row lock: %+v", operation, result) + case <-time.After(40 * time.Millisecond): + } + once.Do(func() { close(release) }) + require.NoError(t, <-writerDone) + resultValue := <-done + require.NoError(t, resultValue.err) + require.False(t, resultValue.applied) + before := f.snapshot() + applied, err := port.CommitItemResult(ctx, &entity.ExptItemResultPublication{Source: source}) + require.NoError(t, err) + require.False(t, applied) + require.Equal(t, before, f.snapshot()) + t.Logf("REAL_MYSQL_PUBLICATION_LOCK operation=%s waited_on_canonical_FOR_UPDATE=true stale_writes=0", operation) + }) + } +} + +func TestScanObservationMySQLProcessingSharesSnapshot(t *testing.T) { + f := newScanObservationFixture(t) + require.NoError(t, f.conn.Model(&model.ExptItemResult{}).Where("space_id=? AND item_id=?", f.space, f.item).Update("status", int32(entity.ItemRunState_Processing)).Error) + require.NoError(t, f.conn.Model(&model.ExptItemResultRunLog{}).Where("space_id=? AND item_id=?", f.space, f.item).Updates(map[string]any{"status": int32(entity.ItemRunState_Processing), "result_state": int32(entity.ExptItemResultStateDefault)}).Error) + provider := newObservationProvider(t, "READ-COMMITTED") + conn := provider.NewSession(context.Background(), db.WithMaster()) + scannedInitial := make(chan struct{}) + resume := make(chan struct{}) + var once sync.Once + defer func() { once.Do(func() { close(resume) }) }() + locks := 0 + require.NoError(t, conn.Callback().Query().After("gorm:query").Register("processing_snapshot_pause", func(tx *gorm.DB) { + text := tx.Statement.SQL.String() + if strings.Contains(text, "FOR UPDATE") { + locks++ + } + if strings.Contains(text, "AS l") && strings.Contains(text, "LEFT JOIN") { + close(scannedInitial) + <-resume + } + })) + defer func() { require.NoError(t, conn.Callback().Query().Remove("processing_snapshot_pause")) }() + type result struct { + rows []*entity.ExptItemCurrentRequest + err error + } + done := make(chan result, 1) + go func() { + rows, err := NewExptItemResultDAO(provider).ScanCurrentRequests(context.Background(), f.space, f.expt, f.run, false) + done <- result{rows, err} + }() + select { + case <-scannedInitial: + case <-time.After(5 * time.Second): + t.Fatal("initial Processing scan not reached") + } + require.NoError(t, f.conn.Model(&model.ExptTurnResultRunLog{}).Where("space_id=? AND item_id=? AND turn_id=0", f.space, f.item).Update("target_result_id", f.base+71).Error) + once.Do(func() { close(resume) }) + observed := <-done + require.Error(t, observed.err, "Processing source changed during RC scan") + require.Nil(t, observed.rows, "Logged and Queueing rows cannot escape the failed observation") + require.Zero(t, locks) + t.Log("REAL_MYSQL_PROCESSING_SNAPSHOT changed_source_discarded=true mixed_logged_processing_queueing_discarded=3 locking_queries=0") +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_scan_transition_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_scan_transition_test.go new file mode 100644 index 0000000000..860869a8c7 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_scan_transition_test.go @@ -0,0 +1,130 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package mysql + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" +) + +func TestScanTransitionMySQLSingleWriterWindows(t *testing.T) { + for _, kind := range []string{"yield", "central_start", "terminate_log_first", "terminate_canonical_first", "canonical_terminal_late_success"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + ctx := context.Background() + dao := NewExptItemResultDAO(f.provider) + item := int64(12) + status := entity.ItemRunState_Queueing + switch kind { + case "yield": + require.NoError(t, dao.UpdateItemRunLog(ctx, f.expt, f.run, []int64{item}, map[string]any{"status": int32(entity.ItemRunState_Queueing)}, f.space)) + case "central_start": + item = 13 + status = entity.ItemRunState_Processing + require.NoError(t, dao.UpdateItemRunLog(ctx, f.expt, f.run, []int64{item}, map[string]any{"quota_reservation_state": int32(entity.QuotaReservationStateReserved)}, f.space)) + changed, err := NewExptItemDispatchDAO(f.provider).StartReservedItem(ctx, f.space, f.expt, f.run, item, "") + require.NoError(t, err) + require.True(t, changed.Started) + case "terminate_log_first": + status = entity.ItemRunState_Terminal + require.NoError(t, dao.UpdateItemRunLog(ctx, f.expt, f.run, []int64{item}, map[string]any{"status": int32(entity.ItemRunState_Terminal), "result_state": int32(entity.ExptItemResultStateLogged)}, f.space)) + case "terminate_canonical_first": + status = entity.ItemRunState_Terminal + require.NoError(t, dao.UpdateItemsResult(ctx, f.space, f.expt, []int64{item}, map[string]any{"status": int32(entity.ItemRunState_Terminal)})) + case "canonical_terminal_late_success": + status = entity.ItemRunState_Terminal + require.NoError(t, dao.UpdateItemsResult(ctx, f.space, f.expt, []int64{item}, map[string]any{"status": int32(entity.ItemRunState_Terminal)})) + require.NoError(t, dao.UpdateItemRunLog(ctx, f.expt, f.run, []int64{item}, map[string]any{"status": int32(entity.ItemRunState_Success), "result_state": int32(entity.ExptItemResultStateLogged)}, f.space)) + } + before := f.snapshot() + rows, err := dao.ScanCurrentRequests(ctx, f.space, f.expt, f.run, false) + require.NoError(t, err) + var got *entity.ExptItemCurrentRequest + for _, row := range rows { + if row.ItemID == item { + got = row + } + } + require.NotNil(t, got) + require.Equal(t, status, got.Status) + require.Equal(t, int32(3), got.RetryTimes) + source, err := dao.GetCurrentRequest(ctx, f.space, f.expt, f.run, item) + require.NoError(t, err) + require.Equal(t, status, source.Status) + if status == entity.ItemRunState_Terminal { + require.False(t, source.Matches(f.run, "")) + } + require.Equal(t, before, f.snapshot(), "scan must never heal or overwrite projections") + if kind == "canonical_terminal_late_success" { + require.NotEqual(t, entity.ExptItemResultStateLogged, got.ResultState, "conflicting late terminal result must not be archived over canonical Terminal") + } + t.Logf("REAL_MYSQL_SCAN_TRANSITION kind=%s status=%d canonical/log divergence read-only", kind, status) + }) + } +} + +func TestScanTransitionMySQLIdentityAndCorruption(t *testing.T) { + for _, kind := range []string{"soft_canonical", "soft_log", "old_run", "old_token", "version_mismatch", "bad_json", "unknown_canonical", "unknown_log", "null_context", "missing_run"} { + t.Run(kind, func(t *testing.T) { + f := newManualRetryFixture(t) + ctx := context.Background() + dao := NewExptItemResultDAO(f.provider) + item := int64(13) + bad := false + switch kind { + case "soft_canonical": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, item).Delete(&model.ExptItemResult{}).Error) + case "soft_log": + require.NoError(t, f.conn.Where("space_id=? AND expt_id=? AND item_id=?", f.space, f.expt, item).Delete(&model.ExptItemResultRunLog{}).Error) + case "old_run": + require.NoError(t, dao.UpdateItemsResult(ctx, f.space, f.expt, []int64{item}, map[string]any{"expt_run_id": f.run + 1})) + case "old_token": + _, err := f.port.RequeueTerminalItems(ctx, f.request(entity.EvaluationModeFailRetry)) + require.NoError(t, err) + before := f.snapshot() + changed, err := dao.MarkProcessingIfCurrent(ctx, f.space, f.expt, f.run, 11, "rq0") + require.NoError(t, err) + require.False(t, changed) + require.Equal(t, before, f.snapshot()) + case "version_mismatch": + bad = true + require.NoError(t, dao.UpdateItemsResult(ctx, f.space, f.expt, []int64{item}, map[string]any{"item_version_id": f.base + 99})) + case "bad_json": + bad = true + require.NoError(t, dao.UpdateItemsResult(ctx, f.space, f.expt, []int64{item}, map[string]any{"ext": []byte(`{`)})) + case "unknown_canonical": + bad = true + require.NoError(t, dao.UpdateItemsResult(ctx, f.space, f.expt, []int64{item}, map[string]any{"status": 9})) + case "unknown_log": + bad = true + require.NoError(t, dao.UpdateItemRunLog(ctx, f.expt, f.run, []int64{item}, map[string]any{"status": 9}, f.space)) + case "null_context": + bad = true + require.NoError(t, dao.UpdateItemsResult(ctx, f.space, f.expt, []int64{item}, map[string]any{"ext": []byte(`{"manual_retry":"null"}`)})) + case "missing_run": + bad = true + require.NoError(t, f.conn.Where("space_id=? AND expt_id=?", f.space, f.expt).Delete(&model.ExptRunLog{}).Error) + } + before := f.snapshot() + rows, err := dao.ScanCurrentRequests(ctx, f.space, f.expt, f.run, false) + if bad { + require.Error(t, err) + require.Nil(t, rows) + } else { + require.NoError(t, err) + if kind != "old_token" { + for _, row := range rows { + require.NotEqual(t, item, row.ItemID) + } + } + } + require.Equal(t, before, f.snapshot()) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_stats.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_stats.go index fa2a8d2760..bc11bb5345 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_stats.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_stats.go @@ -7,7 +7,6 @@ import ( "context" "gorm.io/gorm" - "gorm.io/gorm/clause" "github.com/coze-dev/coze-loop/backend/infra/db" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" @@ -21,10 +20,10 @@ import ( //go:generate mockgen -destination=mocks/expt_stats.go -package mocks . IExptStatsDAO type IExptStatsDAO interface { Create(ctx context.Context, stats *model.ExptStats) error - Get(ctx context.Context, exptID, spaceID int64) (*model.ExptStats, error) + Get(ctx context.Context, exptID, spaceID int64, opts ...db.Option) (*model.ExptStats, error) MGet(ctx context.Context, exptIDs []int64, spaceID int64) ([]*model.ExptStats, error) - UpdateByExptID(ctx context.Context, exptID, spaceID int64, stats *model.ExptStats) error - ArithOperateCount(ctx context.Context, exptID, spaceID int64, cntArithOp *entity.StatsCntArithOp) error + UpdateByExptID(ctx context.Context, exptID, spaceID int64, stats *model.ExptStats, opts ...db.Option) error + ArithOperateCount(ctx context.Context, exptID, spaceID int64, cntArithOp *entity.StatsCntArithOp, opts ...db.Option) error Save(ctx context.Context, stats *model.ExptStats) error } @@ -47,9 +46,10 @@ func (e *exptStatsDAOImpl) Create(ctx context.Context, stats *model.ExptStats) e return nil } -func (e *exptStatsDAOImpl) Get(ctx context.Context, exptID, spaceID int64) (*model.ExptStats, error) { - exptStats, err := e.query.ExptStats.WithContext(ctx). - Where(e.query.ExptStats.ExptID.Eq(exptID), e.query.ExptStats.SpaceID.Eq(spaceID)). +func (e *exptStatsDAOImpl) Get(ctx context.Context, exptID, spaceID int64, opts ...db.Option) (*model.ExptStats, error) { + q := query.Use(e.db.NewSession(ctx, opts...)).ExptStats + exptStats, err := q.WithContext(ctx). + Where(q.ExptID.Eq(exptID), q.SpaceID.Eq(spaceID)). First() if err != nil { return nil, errorx.Wrapf(err, "get ExptStats fail, exptID: %v, spaceID: %v", exptID, spaceID) @@ -67,9 +67,10 @@ func (e *exptStatsDAOImpl) MGet(ctx context.Context, exptIDs []int64, spaceID in return exptStats, nil } -func (e *exptStatsDAOImpl) UpdateByExptID(ctx context.Context, exptID, spaceID int64, stats *model.ExptStats) error { - _, err := e.query.ExptStats.WithContext(ctx). - Where(e.query.ExptStats.ExptID.Eq(exptID), e.query.ExptStats.SpaceID.Eq(spaceID)). +func (e *exptStatsDAOImpl) UpdateByExptID(ctx context.Context, exptID, spaceID int64, stats *model.ExptStats, opts ...db.Option) error { + q := query.Use(e.db.NewSession(ctx, opts...)).ExptStats + _, err := q.WithContext(ctx). + Where(q.ExptID.Eq(exptID), q.SpaceID.Eq(spaceID)). Updates(map[string]interface{}{ "processing_cnt": stats.ProcessingCnt, "pending_cnt": stats.PendingCnt, @@ -91,7 +92,7 @@ func (e *exptStatsDAOImpl) Save(ctx context.Context, stats *model.ExptStats) err return nil } -func (e *exptStatsDAOImpl) ArithOperateCount(ctx context.Context, exptID, spaceID int64, cntArithOp *entity.StatsCntArithOp) error { +func (e *exptStatsDAOImpl) ArithOperateCount(ctx context.Context, exptID, spaceID int64, cntArithOp *entity.StatsCntArithOp, opts ...db.Option) error { if cntArithOp == nil { return nil } @@ -100,10 +101,7 @@ func (e *exptStatsDAOImpl) ArithOperateCount(ctx context.Context, exptID, spaceI return nil } - update := false - db := e.db.NewSession(ctx). - Model(&model.ExptStats{}). - Where("space_id = ? AND expt_id = ?", spaceID, exptID).Clauses(clause.Locking{Strength: clause.LockingStrengthUpdate}) + updates := make(map[string]any) for status, opCnt := range cntArithOp.OpStatusCnt { col := ItemRunStateStatsField(status) @@ -111,18 +109,18 @@ func (e *exptStatsDAOImpl) ArithOperateCount(ctx context.Context, exptID, spaceI continue } - update = true - db.Update(col, gorm.Expr(col+" + ?", opCnt)) + updates[col] = gorm.Expr(col+" + ?", opCnt) } - if !update { + if len(updates) == 0 { logs.CtxInfo(ctx, "ArithOperateCount without update, cntArithOp: %v", json.Jsonify(cntArithOp)) return nil } logs.CtxInfo(ctx, "[StatsCnt] ArithOperateCount expt_id: %v, op_cnt: %v", exptID, json.Jsonify(cntArithOp)) - if err := db.Error; err != nil { + if err := e.db.NewSession(ctx, opts...).Model(&model.ExptStats{}). + Where("space_id = ? AND expt_id = ?", spaceID, exptID).Updates(updates).Error; err != nil { return errorx.Wrapf(err, "update ExptStats cnt fail, expt_id: %v, ufields: %v", exptID, json.Jsonify(cntArithOp)) } return nil diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_turn_evaluator_result_ref.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_turn_evaluator_result_ref.go index b1445bd6c5..57dca33ebe 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_turn_evaluator_result_ref.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_turn_evaluator_result_ref.go @@ -19,6 +19,19 @@ type IExptTurnEvaluatorResultRefDAO interface { BatchGet(ctx context.Context, spaceID int64, exptTurnResultIDs []int64, opts ...db.Option) ([]*model.ExptTurnEvaluatorResultRef, error) GetByExptID(ctx context.Context, spaceID, exptID int64, opts ...db.Option) ([]*model.ExptTurnEvaluatorResultRef, error) GetByExptEvaluatorVersionID(ctx context.Context, spaceID, exptID, evaluatorVersionID int64, opts ...db.Option) ([]*model.ExptTurnEvaluatorResultRef, error) + DeleteByTurnResultIDs(ctx context.Context, spaceID, exptID int64, turnResultIDs, keepRefIDs []int64, opts ...db.Option) error +} + +func (dao *ExptTurnEvaluatorResultRefDAOImpl) DeleteByTurnResultIDs(ctx context.Context, spaceID, exptID int64, turnResultIDs, keepRefIDs []int64, opts ...db.Option) error { + if len(turnResultIDs) == 0 { + return nil + } + q := dao.provider.NewSession(ctx, opts...).Unscoped(). + Where("space_id = ? AND expt_id = ? AND expt_turn_result_id IN ?", spaceID, exptID, turnResultIDs) + if len(keepRefIDs) > 0 { + q = q.Where("id NOT IN ?", keepRefIDs) + } + return q.Delete(&model.ExptTurnEvaluatorResultRef{}).Error } type ExptTurnEvaluatorResultRefDAOImpl struct { diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt.go index 68ab468bf3..bf76ecf9f2 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt.go @@ -13,6 +13,7 @@ import ( context "context" reflect "reflect" + db "github.com/coze-dev/coze-loop/backend/infra/db" entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" model "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" gomock "go.uber.org/mock/gomock" @@ -22,6 +23,7 @@ import ( type MockIExptDAO struct { ctrl *gomock.Controller recorder *MockIExptDAOMockRecorder + isgomock struct{} } // MockIExptDAOMockRecorder is the mock recorder for MockIExptDAO. @@ -42,97 +44,87 @@ func (m *MockIExptDAO) EXPECT() *MockIExptDAOMockRecorder { } // Create mocks base method. -func (m *MockIExptDAO) Create(arg0 context.Context, arg1 *model.Experiment) error { +func (m *MockIExptDAO) Create(ctx context.Context, expt *model.Experiment) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1) + ret := m.ctrl.Call(m, "Create", ctx, expt) ret0, _ := ret[0].(error) return ret0 } // Create indicates an expected call of Create. -func (mr *MockIExptDAOMockRecorder) Create(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) Create(ctx, expt any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockIExptDAO)(nil).Create), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockIExptDAO)(nil).Create), ctx, expt) } // Delete mocks base method. -func (m *MockIExptDAO) Delete(arg0 context.Context, arg1 int64) error { +func (m *MockIExptDAO) Delete(ctx context.Context, id int64) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0, arg1) + ret := m.ctrl.Call(m, "Delete", ctx, id) ret0, _ := ret[0].(error) return ret0 } // Delete indicates an expected call of Delete. -func (mr *MockIExptDAOMockRecorder) Delete(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) Delete(ctx, id any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockIExptDAO)(nil).Delete), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockIExptDAO)(nil).Delete), ctx, id) } // ExistGroupKey mocks base method. -func (m *MockIExptDAO) ExistGroupKey(arg0 context.Context, arg1 string, arg2 int64) (bool, error) { +func (m *MockIExptDAO) ExistGroupKey(ctx context.Context, groupKey string, spaceID int64) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExistGroupKey", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "ExistGroupKey", ctx, groupKey, spaceID) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // ExistGroupKey indicates an expected call of ExistGroupKey. -func (mr *MockIExptDAOMockRecorder) ExistGroupKey(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) ExistGroupKey(ctx, groupKey, spaceID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExistGroupKey", reflect.TypeOf((*MockIExptDAO)(nil).ExistGroupKey), arg0, arg1, arg2) -} - -// ScanSchedulerQueue mocks base method. -func (m *MockIExptDAO) ScanSchedulerQueue(arg0 context.Context, arg1 *entity.SchedulerQueueScanParam) ([]*model.Experiment, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ScanSchedulerQueue", arg0, arg1) - ret0, _ := ret[0].([]*model.Experiment) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ScanSchedulerQueue indicates an expected call of ScanSchedulerQueue. -func (mr *MockIExptDAOMockRecorder) ScanSchedulerQueue(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanSchedulerQueue", reflect.TypeOf((*MockIExptDAO)(nil).ScanSchedulerQueue), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExistGroupKey", reflect.TypeOf((*MockIExptDAO)(nil).ExistGroupKey), ctx, groupKey, spaceID) } // GetByID mocks base method. -func (m *MockIExptDAO) GetByID(arg0 context.Context, arg1 int64) (*model.Experiment, error) { +func (m *MockIExptDAO) GetByID(ctx context.Context, id int64, opts ...db.Option) (*model.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetByID", arg0, arg1) + varargs := []any{ctx, id} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetByID", varargs...) ret0, _ := ret[0].(*model.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // GetByID indicates an expected call of GetByID. -func (mr *MockIExptDAOMockRecorder) GetByID(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) GetByID(ctx, id any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByID", reflect.TypeOf((*MockIExptDAO)(nil).GetByID), arg0, arg1) + varargs := append([]any{ctx, id}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByID", reflect.TypeOf((*MockIExptDAO)(nil).GetByID), varargs...) } // GetByName mocks base method. -func (m *MockIExptDAO) GetByName(arg0 context.Context, arg1 string, arg2 int64) (*model.Experiment, error) { +func (m *MockIExptDAO) GetByName(ctx context.Context, name string, spaceID int64) (*model.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetByName", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "GetByName", ctx, name, spaceID) ret0, _ := ret[0].(*model.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // GetByName indicates an expected call of GetByName. -func (mr *MockIExptDAOMockRecorder) GetByName(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) GetByName(ctx, name, spaceID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByName", reflect.TypeOf((*MockIExptDAO)(nil).GetByName), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByName", reflect.TypeOf((*MockIExptDAO)(nil).GetByName), ctx, name, spaceID) } // GetIDsByGroupKey mocks base method. -func (m *MockIExptDAO) GetIDsByGroupKey(arg0 context.Context, arg1 int64, arg2 string, arg3, arg4 int32) ([]int64, int64, error) { +func (m *MockIExptDAO) GetIDsByGroupKey(ctx context.Context, spaceID int64, groupKey string, page, pageSize int32) ([]int64, int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetIDsByGroupKey", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "GetIDsByGroupKey", ctx, spaceID, groupKey, page, pageSize) ret0, _ := ret[0].([]int64) ret1, _ := ret[1].(int64) ret2, _ := ret[2].(error) @@ -140,15 +132,15 @@ func (m *MockIExptDAO) GetIDsByGroupKey(arg0 context.Context, arg1 int64, arg2 s } // GetIDsByGroupKey indicates an expected call of GetIDsByGroupKey. -func (mr *MockIExptDAOMockRecorder) GetIDsByGroupKey(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) GetIDsByGroupKey(ctx, spaceID, groupKey, page, pageSize any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDsByGroupKey", reflect.TypeOf((*MockIExptDAO)(nil).GetIDsByGroupKey), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDsByGroupKey", reflect.TypeOf((*MockIExptDAO)(nil).GetIDsByGroupKey), ctx, spaceID, groupKey, page, pageSize) } // List mocks base method. -func (m *MockIExptDAO) List(arg0 context.Context, arg1, arg2 int32, arg3 *entity.ExptListFilter, arg4 []*entity.OrderBy, arg5 int64) ([]*model.Experiment, int64, error) { +func (m *MockIExptDAO) List(ctx context.Context, page, size int32, filter *entity.ExptListFilter, orders []*entity.OrderBy, spaceID int64) ([]*model.Experiment, int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5) + ret := m.ctrl.Call(m, "List", ctx, page, size, filter, orders, spaceID) ret0, _ := ret[0].([]*model.Experiment) ret1, _ := ret[1].(int64) ret2, _ := ret[2].(error) @@ -156,64 +148,84 @@ func (m *MockIExptDAO) List(arg0 context.Context, arg1, arg2 int32, arg3 *entity } // List indicates an expected call of List. -func (mr *MockIExptDAOMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) List(ctx, page, size, filter, orders, spaceID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIExptDAO)(nil).List), arg0, arg1, arg2, arg3, arg4, arg5) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIExptDAO)(nil).List), ctx, page, size, filter, orders, spaceID) } // MDelete mocks base method. -func (m *MockIExptDAO) MDelete(arg0 context.Context, arg1 []int64) error { +func (m *MockIExptDAO) MDelete(ctx context.Context, ids []int64) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MDelete", arg0, arg1) + ret := m.ctrl.Call(m, "MDelete", ctx, ids) ret0, _ := ret[0].(error) return ret0 } // MDelete indicates an expected call of MDelete. -func (mr *MockIExptDAOMockRecorder) MDelete(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) MDelete(ctx, ids any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MDelete", reflect.TypeOf((*MockIExptDAO)(nil).MDelete), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MDelete", reflect.TypeOf((*MockIExptDAO)(nil).MDelete), ctx, ids) } // MGetByID mocks base method. -func (m *MockIExptDAO) MGetByID(arg0 context.Context, arg1 []int64) ([]*model.Experiment, error) { +func (m *MockIExptDAO) MGetByID(ctx context.Context, ids []int64) ([]*model.Experiment, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MGetByID", arg0, arg1) + ret := m.ctrl.Call(m, "MGetByID", ctx, ids) ret0, _ := ret[0].([]*model.Experiment) ret1, _ := ret[1].(error) return ret0, ret1 } // MGetByID indicates an expected call of MGetByID. -func (mr *MockIExptDAOMockRecorder) MGetByID(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) MGetByID(ctx, ids any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetByID", reflect.TypeOf((*MockIExptDAO)(nil).MGetByID), ctx, ids) +} + +// ScanSchedulerQueue mocks base method. +func (m *MockIExptDAO) ScanSchedulerQueue(ctx context.Context, param *entity.SchedulerQueueScanParam) ([]*model.Experiment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanSchedulerQueue", ctx, param) + ret0, _ := ret[0].([]*model.Experiment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ScanSchedulerQueue indicates an expected call of ScanSchedulerQueue. +func (mr *MockIExptDAOMockRecorder) ScanSchedulerQueue(ctx, param any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetByID", reflect.TypeOf((*MockIExptDAO)(nil).MGetByID), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanSchedulerQueue", reflect.TypeOf((*MockIExptDAO)(nil).ScanSchedulerQueue), ctx, param) } // Update mocks base method. -func (m *MockIExptDAO) Update(arg0 context.Context, arg1 *model.Experiment) error { +func (m *MockIExptDAO) Update(ctx context.Context, expt *model.Experiment) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Update", arg0, arg1) + ret := m.ctrl.Call(m, "Update", ctx, expt) ret0, _ := ret[0].(error) return ret0 } // Update indicates an expected call of Update. -func (mr *MockIExptDAOMockRecorder) Update(arg0, arg1 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) Update(ctx, expt any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptDAO)(nil).Update), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptDAO)(nil).Update), ctx, expt) } // UpdateFields mocks base method. -func (m *MockIExptDAO) UpdateFields(arg0 context.Context, arg1 int64, arg2 map[string]any) error { +func (m *MockIExptDAO) UpdateFields(ctx context.Context, id int64, ufields map[string]any, opts ...db.Option) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateFields", arg0, arg1, arg2) + varargs := []any{ctx, id, ufields} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateFields", varargs...) ret0, _ := ret[0].(error) return ret0 } // UpdateFields indicates an expected call of UpdateFields. -func (mr *MockIExptDAOMockRecorder) UpdateFields(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockIExptDAOMockRecorder) UpdateFields(ctx, id, ufields any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateFields", reflect.TypeOf((*MockIExptDAO)(nil).UpdateFields), arg0, arg1, arg2) + varargs := append([]any{ctx, id, ufields}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateFields", reflect.TypeOf((*MockIExptDAO)(nil).UpdateFields), varargs...) } diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_dispatch.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_dispatch.go index 1a2c9e7cbb..53e2ae2b76 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_dispatch.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_dispatch.go @@ -42,18 +42,18 @@ func (m *MockIExptItemDispatchDAO) EXPECT() *MockIExptItemDispatchDAOMockRecorde } // ClaimQuotaReserved mocks base method. -func (m *MockIExptItemDispatchDAO) ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) { +func (m *MockIExptItemDispatchDAO) ClaimQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]repo.ClaimedDispatchItem, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClaimQuotaReserved", ctx, spaceID, exptID, exptRunID, itemIDs) - ret0, _ := ret[0].([]int64) + ret := m.ctrl.Call(m, "ClaimQuotaReserved", ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) + ret0, _ := ret[0].([]repo.ClaimedDispatchItem) ret1, _ := ret[1].(error) return ret0, ret1 } // ClaimQuotaReserved indicates an expected call of ClaimQuotaReserved. -func (mr *MockIExptItemDispatchDAOMockRecorder) ClaimQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs any) *gomock.Call { +func (mr *MockIExptItemDispatchDAOMockRecorder) ClaimQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClaimQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).ClaimQuotaReserved), ctx, spaceID, exptID, exptRunID, itemIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClaimQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).ClaimQuotaReserved), ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) } // LoadDispatchRuntime mocks base method. @@ -87,46 +87,46 @@ func (mr *MockIExptItemDispatchDAOMockRecorder) MGetDispatchObservations(ctx, sp } // RequeueProcessingItem mocks base method. -func (m *MockIExptItemDispatchDAO) RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) { +func (m *MockIExptItemDispatchDAO) RequeueProcessingItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RequeueProcessingItem", ctx, spaceID, exptID, exptRunID, itemID) + ret := m.ctrl.Call(m, "RequeueProcessingItem", ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // RequeueProcessingItem indicates an expected call of RequeueProcessingItem. -func (mr *MockIExptItemDispatchDAOMockRecorder) RequeueProcessingItem(ctx, spaceID, exptID, exptRunID, itemID any) *gomock.Call { +func (mr *MockIExptItemDispatchDAOMockRecorder) RequeueProcessingItem(ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequeueProcessingItem", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).RequeueProcessingItem), ctx, spaceID, exptID, exptRunID, itemID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequeueProcessingItem", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).RequeueProcessingItem), ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) } // ResetQuotaReserved mocks base method. -func (m *MockIExptItemDispatchDAO) ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64) ([]int64, error) { +func (m *MockIExptItemDispatchDAO) ResetQuotaReserved(ctx context.Context, spaceID, exptID, exptRunID int64, itemIDs []int64, itemRequestIDs map[int64]string) ([]int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResetQuotaReserved", ctx, spaceID, exptID, exptRunID, itemIDs) + ret := m.ctrl.Call(m, "ResetQuotaReserved", ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) ret0, _ := ret[0].([]int64) ret1, _ := ret[1].(error) return ret0, ret1 } // ResetQuotaReserved indicates an expected call of ResetQuotaReserved. -func (mr *MockIExptItemDispatchDAOMockRecorder) ResetQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs any) *gomock.Call { +func (mr *MockIExptItemDispatchDAOMockRecorder) ResetQuotaReserved(ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).ResetQuotaReserved), ctx, spaceID, exptID, exptRunID, itemIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetQuotaReserved", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).ResetQuotaReserved), ctx, spaceID, exptID, exptRunID, itemIDs, itemRequestIDs) } // StartReservedItem mocks base method. -func (m *MockIExptItemDispatchDAO) StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64) (bool, error) { +func (m *MockIExptItemDispatchDAO) StartReservedItem(ctx context.Context, spaceID, exptID, exptRunID, itemID int64, expectedRequestID string) (repo.StartReservedItemResult, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StartReservedItem", ctx, spaceID, exptID, exptRunID, itemID) - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "StartReservedItem", ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) + ret0, _ := ret[0].(repo.StartReservedItemResult) ret1, _ := ret[1].(error) return ret0, ret1 } // StartReservedItem indicates an expected call of StartReservedItem. -func (mr *MockIExptItemDispatchDAOMockRecorder) StartReservedItem(ctx, spaceID, exptID, exptRunID, itemID any) *gomock.Call { +func (mr *MockIExptItemDispatchDAOMockRecorder) StartReservedItem(ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartReservedItem", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).StartReservedItem), ctx, spaceID, exptID, exptRunID, itemID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartReservedItem", reflect.TypeOf((*MockIExptItemDispatchDAO)(nil).StartReservedItem), ctx, spaceID, exptID, exptRunID, itemID, expectedRequestID) } diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_ref.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_ref.go new file mode 100644 index 0000000000..21347739da --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_ref.go @@ -0,0 +1,144 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql (interfaces: IExptItemRefDAO) +// +// Generated by this command: +// +// mockgen -destination=mocks/expt_item_ref.go -package mocks . IExptItemRefDAO +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + db "github.com/coze-dev/coze-loop/backend/infra/db" + mysql "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql" + model "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptItemRefDAO is a mock of IExptItemRefDAO interface. +type MockIExptItemRefDAO struct { + ctrl *gomock.Controller + recorder *MockIExptItemRefDAOMockRecorder + isgomock struct{} +} + +// MockIExptItemRefDAOMockRecorder is the mock recorder for MockIExptItemRefDAO. +type MockIExptItemRefDAOMockRecorder struct { + mock *MockIExptItemRefDAO +} + +// NewMockIExptItemRefDAO creates a new mock instance. +func NewMockIExptItemRefDAO(ctrl *gomock.Controller) *MockIExptItemRefDAO { + mock := &MockIExptItemRefDAO{ctrl: ctrl} + mock.recorder = &MockIExptItemRefDAOMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptItemRefDAO) EXPECT() *MockIExptItemRefDAOMockRecorder { + return m.recorder +} + +// BatchCreateNX mocks base method. +func (m *MockIExptItemRefDAO) BatchCreateNX(ctx context.Context, items []*model.ExptItemRef, opts ...db.Option) error { + m.ctrl.T.Helper() + varargs := []any{ctx, items} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "BatchCreateNX", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchCreateNX indicates an expected call of BatchCreateNX. +func (mr *MockIExptItemRefDAOMockRecorder) BatchCreateNX(ctx, items any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, items}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchCreateNX", reflect.TypeOf((*MockIExptItemRefDAO)(nil).BatchCreateNX), varargs...) +} + +// CountByEvalSetGrouped mocks base method. +func (m *MockIExptItemRefDAO) CountByEvalSetGrouped(ctx context.Context, spaceID int64, exptIDs []int64, opts ...db.Option) ([]*mysql.ExptItemRefEvalSetCount, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptIDs} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CountByEvalSetGrouped", varargs...) + ret0, _ := ret[0].([]*mysql.ExptItemRefEvalSetCount) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountByEvalSetGrouped indicates an expected call of CountByEvalSetGrouped. +func (mr *MockIExptItemRefDAOMockRecorder) CountByEvalSetGrouped(ctx, spaceID, exptIDs any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptIDs}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountByEvalSetGrouped", reflect.TypeOf((*MockIExptItemRefDAO)(nil).CountByEvalSetGrouped), varargs...) +} + +// GetByExptIDAndItemID mocks base method. +func (m *MockIExptItemRefDAO) GetByExptIDAndItemID(ctx context.Context, spaceID, exptID, itemID int64, opts ...db.Option) (*model.ExptItemRef, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, itemID} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetByExptIDAndItemID", varargs...) + ret0, _ := ret[0].(*model.ExptItemRef) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetByExptIDAndItemID indicates an expected call of GetByExptIDAndItemID. +func (mr *MockIExptItemRefDAOMockRecorder) GetByExptIDAndItemID(ctx, spaceID, exptID, itemID any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, itemID}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByExptIDAndItemID", reflect.TypeOf((*MockIExptItemRefDAO)(nil).GetByExptIDAndItemID), varargs...) +} + +// ListByExptID mocks base method. +func (m *MockIExptItemRefDAO) ListByExptID(ctx context.Context, spaceID, exptID, cursor, limit int64, opts ...db.Option) ([]*model.ExptItemRef, int64, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, cursor, limit} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListByExptID", varargs...) + ret0, _ := ret[0].([]*model.ExptItemRef) + ret1, _ := ret[1].(int64) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ListByExptID indicates an expected call of ListByExptID. +func (mr *MockIExptItemRefDAOMockRecorder) ListByExptID(ctx, spaceID, exptID, cursor, limit any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, cursor, limit}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByExptID", reflect.TypeOf((*MockIExptItemRefDAO)(nil).ListByExptID), varargs...) +} + +// MGetByExptIDAndItemIDs mocks base method. +func (m *MockIExptItemRefDAO) MGetByExptIDAndItemIDs(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptItemRef, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, itemIDs} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "MGetByExptIDAndItemIDs", varargs...) + ret0, _ := ret[0].([]*model.ExptItemRef) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetByExptIDAndItemIDs indicates an expected call of MGetByExptIDAndItemIDs. +func (mr *MockIExptItemRefDAOMockRecorder) MGetByExptIDAndItemIDs(ctx, spaceID, exptID, itemIDs any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, itemIDs}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetByExptIDAndItemIDs", reflect.TypeOf((*MockIExptItemRefDAO)(nil).MGetByExptIDAndItemIDs), varargs...) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go index 74d876335d..7924628098 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -destination=mocks/expt_item_result.go -package mocks github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql IExptItemResultDAO +// mockgen -destination=mocks/expt_item_result.go -package mocks . IExptItemResultDAO // // Package mocks is a generated GoMock package. @@ -15,6 +15,7 @@ import ( db "github.com/coze-dev/coze-loop/backend/infra/db" entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + repo "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" model "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" gomock "go.uber.org/mock/gomock" ) @@ -141,6 +142,21 @@ func (mr *MockIExptItemResultDAOMockRecorder) CountItemsByStatus(ctx, spaceID, e return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountItemsByStatus", reflect.TypeOf((*MockIExptItemResultDAO)(nil).CountItemsByStatus), varargs...) } +// GetCurrentRequest mocks base method. +func (m *MockIExptItemResultDAO) GetCurrentRequest(ctx context.Context, spaceID, exptID, runID, itemID int64) (*entity.ExptItemCurrentRequest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCurrentRequest", ctx, spaceID, exptID, runID, itemID) + ret0, _ := ret[0].(*entity.ExptItemCurrentRequest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCurrentRequest indicates an expected call of GetCurrentRequest. +func (mr *MockIExptItemResultDAOMockRecorder) GetCurrentRequest(ctx, spaceID, exptID, runID, itemID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentRequest", reflect.TypeOf((*MockIExptItemResultDAO)(nil).GetCurrentRequest), ctx, spaceID, exptID, runID, itemID) +} + // GetItemIDListByExptID mocks base method. func (m *MockIExptItemResultDAO) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) ([]int64, error) { m.ctrl.T.Helper() @@ -312,6 +328,21 @@ func (mr *MockIExptItemResultDAOMockRecorder) MGetItemTurnResults(ctx, spaceID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemTurnResults", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MGetItemTurnResults), varargs...) } +// MarkProcessingIfCurrent mocks base method. +func (m *MockIExptItemResultDAO) MarkProcessingIfCurrent(ctx context.Context, spaceID, exptID, runID, itemID int64, requestID string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkProcessingIfCurrent", ctx, spaceID, exptID, runID, itemID, requestID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MarkProcessingIfCurrent indicates an expected call of MarkProcessingIfCurrent. +func (mr *MockIExptItemResultDAOMockRecorder) MarkProcessingIfCurrent(ctx, spaceID, exptID, runID, itemID, requestID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkProcessingIfCurrent", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MarkProcessingIfCurrent), ctx, spaceID, exptID, runID, itemID, requestID) +} + // SaveItemResults mocks base method. func (m *MockIExptItemResultDAO) SaveItemResults(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error { m.ctrl.T.Helper() @@ -331,6 +362,36 @@ func (mr *MockIExptItemResultDAOMockRecorder) SaveItemResults(ctx, itemResults a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveItemResults", reflect.TypeOf((*MockIExptItemResultDAO)(nil).SaveItemResults), varargs...) } +// ScanCurrentRequests mocks base method. +func (m *MockIExptItemResultDAO) ScanCurrentRequests(ctx context.Context, spaceID, exptID, runID int64, retryFirst bool) ([]*entity.ExptItemCurrentRequest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanCurrentRequests", ctx, spaceID, exptID, runID, retryFirst) + ret0, _ := ret[0].([]*entity.ExptItemCurrentRequest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ScanCurrentRequests indicates an expected call of ScanCurrentRequests. +func (mr *MockIExptItemResultDAOMockRecorder) ScanCurrentRequests(ctx, spaceID, exptID, runID, retryFirst any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanCurrentRequests", reflect.TypeOf((*MockIExptItemResultDAO)(nil).ScanCurrentRequests), ctx, spaceID, exptID, runID, retryFirst) +} + +// ScanIncompleteDispatchObservations mocks base method. +func (m *MockIExptItemResultDAO) ScanIncompleteDispatchObservations(ctx context.Context, spaceID, exptID, runID int64) ([]*repo.ExptDispatchObservation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanIncompleteDispatchObservations", ctx, spaceID, exptID, runID) + ret0, _ := ret[0].([]*repo.ExptDispatchObservation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ScanIncompleteDispatchObservations indicates an expected call of ScanIncompleteDispatchObservations. +func (mr *MockIExptItemResultDAOMockRecorder) ScanIncompleteDispatchObservations(ctx, spaceID, exptID, runID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanIncompleteDispatchObservations", reflect.TypeOf((*MockIExptItemResultDAO)(nil).ScanIncompleteDispatchObservations), ctx, spaceID, exptID, runID) +} + // ScanItemResults mocks base method. func (m *MockIExptItemResultDAO) ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64, opts ...db.Option) ([]*model.ExptItemResult, int64, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_run_log.go index db76f269b1..1f86e9d961 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_run_log.go @@ -42,6 +42,20 @@ func (m *MockIExptRunLogDAO) EXPECT() *MockIExptRunLogDAOMockRecorder { return m.recorder } +// AppendRunItemIDs mocks base method. +func (m *MockIExptRunLogDAO) AppendRunItemIDs(ctx context.Context, exptID, exptRunID, spaceID int64, itemIDs []int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendRunItemIDs", ctx, exptID, exptRunID, spaceID, itemIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// AppendRunItemIDs indicates an expected call of AppendRunItemIDs. +func (mr *MockIExptRunLogDAOMockRecorder) AppendRunItemIDs(ctx, exptID, exptRunID, spaceID, itemIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendRunItemIDs", reflect.TypeOf((*MockIExptRunLogDAO)(nil).AppendRunItemIDs), ctx, exptID, exptRunID, spaceID, itemIDs) +} + // Create mocks base method. func (m *MockIExptRunLogDAO) Create(ctx context.Context, exptRunLog *model.ExptRunLog, opts ...db.Option) error { m.ctrl.T.Helper() @@ -118,3 +132,17 @@ func (mr *MockIExptRunLogDAOMockRecorder) Update(ctx, exptID, exptRunID, spaceID varargs := append([]any{ctx, exptID, exptRunID, spaceID, ufields}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptRunLogDAO)(nil).Update), varargs...) } + +// UpdateRunStats mocks base method. +func (m *MockIExptRunLogDAO) UpdateRunStats(ctx context.Context, exptID, exptRunID, spaceID int64, fields map[string]any, stats *model.ExptStats) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateRunStats", ctx, exptID, exptRunID, spaceID, fields, stats) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateRunStats indicates an expected call of UpdateRunStats. +func (mr *MockIExptRunLogDAOMockRecorder) UpdateRunStats(ctx, exptID, exptRunID, spaceID, fields, stats any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRunStats", reflect.TypeOf((*MockIExptRunLogDAO)(nil).UpdateRunStats), ctx, exptID, exptRunID, spaceID, fields, stats) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_stats.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_stats.go index b2c61129ca..5bf27e0464 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_stats.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_stats.go @@ -13,6 +13,7 @@ import ( context "context" reflect "reflect" + db "github.com/coze-dev/coze-loop/backend/infra/db" entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" model "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" gomock "go.uber.org/mock/gomock" @@ -43,17 +44,22 @@ func (m *MockIExptStatsDAO) EXPECT() *MockIExptStatsDAOMockRecorder { } // ArithOperateCount mocks base method. -func (m *MockIExptStatsDAO) ArithOperateCount(ctx context.Context, exptID, spaceID int64, cntArithOp *entity.StatsCntArithOp) error { +func (m *MockIExptStatsDAO) ArithOperateCount(ctx context.Context, exptID, spaceID int64, cntArithOp *entity.StatsCntArithOp, opts ...db.Option) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ArithOperateCount", ctx, exptID, spaceID, cntArithOp) + varargs := []any{ctx, exptID, spaceID, cntArithOp} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ArithOperateCount", varargs...) ret0, _ := ret[0].(error) return ret0 } // ArithOperateCount indicates an expected call of ArithOperateCount. -func (mr *MockIExptStatsDAOMockRecorder) ArithOperateCount(ctx, exptID, spaceID, cntArithOp any) *gomock.Call { +func (mr *MockIExptStatsDAOMockRecorder) ArithOperateCount(ctx, exptID, spaceID, cntArithOp any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ArithOperateCount", reflect.TypeOf((*MockIExptStatsDAO)(nil).ArithOperateCount), ctx, exptID, spaceID, cntArithOp) + varargs := append([]any{ctx, exptID, spaceID, cntArithOp}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ArithOperateCount", reflect.TypeOf((*MockIExptStatsDAO)(nil).ArithOperateCount), varargs...) } // Create mocks base method. @@ -71,18 +77,23 @@ func (mr *MockIExptStatsDAOMockRecorder) Create(ctx, stats any) *gomock.Call { } // Get mocks base method. -func (m *MockIExptStatsDAO) Get(ctx context.Context, exptID, spaceID int64) (*model.ExptStats, error) { +func (m *MockIExptStatsDAO) Get(ctx context.Context, exptID, spaceID int64, opts ...db.Option) (*model.ExptStats, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", ctx, exptID, spaceID) + varargs := []any{ctx, exptID, spaceID} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Get", varargs...) ret0, _ := ret[0].(*model.ExptStats) ret1, _ := ret[1].(error) return ret0, ret1 } // Get indicates an expected call of Get. -func (mr *MockIExptStatsDAOMockRecorder) Get(ctx, exptID, spaceID any) *gomock.Call { +func (mr *MockIExptStatsDAOMockRecorder) Get(ctx, exptID, spaceID any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIExptStatsDAO)(nil).Get), ctx, exptID, spaceID) + varargs := append([]any{ctx, exptID, spaceID}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIExptStatsDAO)(nil).Get), varargs...) } // MGet mocks base method. @@ -115,15 +126,20 @@ func (mr *MockIExptStatsDAOMockRecorder) Save(ctx, stats any) *gomock.Call { } // UpdateByExptID mocks base method. -func (m *MockIExptStatsDAO) UpdateByExptID(ctx context.Context, exptID, spaceID int64, stats *model.ExptStats) error { +func (m *MockIExptStatsDAO) UpdateByExptID(ctx context.Context, exptID, spaceID int64, stats *model.ExptStats, opts ...db.Option) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateByExptID", ctx, exptID, spaceID, stats) + varargs := []any{ctx, exptID, spaceID, stats} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateByExptID", varargs...) ret0, _ := ret[0].(error) return ret0 } // UpdateByExptID indicates an expected call of UpdateByExptID. -func (mr *MockIExptStatsDAOMockRecorder) UpdateByExptID(ctx, exptID, spaceID, stats any) *gomock.Call { +func (mr *MockIExptStatsDAOMockRecorder) UpdateByExptID(ctx, exptID, spaceID, stats any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateByExptID", reflect.TypeOf((*MockIExptStatsDAO)(nil).UpdateByExptID), ctx, exptID, spaceID, stats) + varargs := append([]any{ctx, exptID, spaceID, stats}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateByExptID", reflect.TypeOf((*MockIExptStatsDAO)(nil).UpdateByExptID), varargs...) } diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_turn_evaluator_result_ref.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_turn_evaluator_result_ref.go index ff61bfd9f1..d36774307a 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_turn_evaluator_result_ref.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_turn_evaluator_result_ref.go @@ -62,6 +62,25 @@ func (mr *MockIExptTurnEvaluatorResultRefDAOMockRecorder) BatchGet(ctx, spaceID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchGet", reflect.TypeOf((*MockIExptTurnEvaluatorResultRefDAO)(nil).BatchGet), varargs...) } +// DeleteByTurnResultIDs mocks base method. +func (m *MockIExptTurnEvaluatorResultRefDAO) DeleteByTurnResultIDs(ctx context.Context, spaceID, exptID int64, turnResultIDs, keepRefIDs []int64, opts ...db.Option) error { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, turnResultIDs, keepRefIDs} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteByTurnResultIDs", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteByTurnResultIDs indicates an expected call of DeleteByTurnResultIDs. +func (mr *MockIExptTurnEvaluatorResultRefDAOMockRecorder) DeleteByTurnResultIDs(ctx, spaceID, exptID, turnResultIDs, keepRefIDs any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, turnResultIDs, keepRefIDs}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteByTurnResultIDs", reflect.TypeOf((*MockIExptTurnEvaluatorResultRefDAO)(nil).DeleteByTurnResultIDs), varargs...) +} + // GetByExptEvaluatorVersionID mocks base method. func (m *MockIExptTurnEvaluatorResultRefDAO) GetByExptEvaluatorVersionID(ctx context.Context, spaceID, exptID, evaluatorVersionID int64, opts ...db.Option) ([]*model.ExptTurnEvaluatorResultRef, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/run_log_master_read_test.go b/backend/modules/evaluation/infra/repo/experiment/mysql/run_log_master_read_test.go index 2d574fec44..61eb5cf422 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/run_log_master_read_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/run_log_master_read_test.go @@ -15,6 +15,7 @@ import ( "gorm.io/plugin/dbresolver" dbmock "github.com/coze-dev/coze-loop/backend/infra/db/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" ) @@ -128,3 +129,29 @@ func TestExptTurnResultDAO_GetItemTurnRunLogs_ReadContextReadsReplica(t *testing require.NoError(t, err) require.NoError(t, replicaMock.ExpectationsWereMet()) } + +// Projection candidates and per-item snapshot refs must not use lagging replicas. +func TestFilterProjection_WriteContextReadsPrimary(t *testing.T) { + for _, kind := range []string{"item_candidates", "item_ref"} { + t.Run(kind, func(t *testing.T) { + ctrl := gomock.NewController(t) + conn, master, _ := newRunLogResolverTestDB(t) + provider := dbmock.NewMockProvider(ctrl) + provider.EXPECT().NewSession(gomock.Any()).Return(conn).AnyTimes() + ctx := contexts.WithCtxWriteDB(context.Background()) + if kind == "item_candidates" { + master.ExpectQuery("SELECT count\\(\\*\\) FROM `expt_item_result`").WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + master.ExpectQuery("SELECT .* FROM `expt_item_result`").WillReturnRows(sqlmock.NewRows([]string{"id", "item_id"}).AddRow(1, 7384603622000000011)) + rows, _, err := (&exptItemResultDAOImpl{provider: provider}).ListItemResultsByExptID(ctx, 1, 3, entity.Page{}, false) + require.NoError(t, err) + require.Len(t, rows, 1) + } else { + master.ExpectQuery("SELECT .* FROM `expt_item_ref`").WillReturnRows(sqlmock.NewRows([]string{"id", "item_id", "eval_set_version_id"}).AddRow(1, 7384603622000000011, 7384603622000000099)) + rows, err := (&exptItemRefDAOImpl{provider: provider}).MGetByExptIDAndItemIDs(ctx, 3, 1, []int64{7384603622000000011}) + require.NoError(t, err) + require.Len(t, rows, 1) + } + require.NoError(t, master.ExpectationsWereMet()) + }) + } +} diff --git a/backend/modules/evaluation/infra/repo/experiment/wire.go b/backend/modules/evaluation/infra/repo/experiment/wire.go index 0634300190..1d8a6090c7 100644 --- a/backend/modules/evaluation/infra/repo/experiment/wire.go +++ b/backend/modules/evaluation/infra/repo/experiment/wire.go @@ -15,6 +15,8 @@ import ( var ExperimentRepoSet = wire.NewSet( NewExptRepo, + NewExptManualRetryRepo, + NewExptResultPublishRepo, NewExptSchedulerQueueRepo, NewExptStatsRepo, NewExptAggrResultRepo, diff --git a/backend/modules/evaluation/infra/rpc/data/convert.go b/backend/modules/evaluation/infra/rpc/data/convert.go index 573df5c4c6..a64b4093d8 100644 --- a/backend/modules/evaluation/infra/rpc/data/convert.go +++ b/backend/modules/evaluation/infra/rpc/data/convert.go @@ -646,6 +646,26 @@ func convertStorageProvider(provider *dataset.StorageProvider) *entity.StoragePr } func convert2EvaluationSetTurn(ctx context.Context, item *dataset.DatasetItem) (turns []*entity.Turn) { + if len(item.RepeatedData) > 0 { + turns = make([]*entity.Turn, 0, len(item.RepeatedData)) + for _, itemData := range item.RepeatedData { + if itemData == nil { + continue + } + turn := &entity.Turn{ + ID: gptr.Indirect(itemData.ID), + ItemID: gptr.Indirect(item.ItemID), + EvalSetID: item.GetDatasetID(), + FieldDataList: make([]*entity.FieldData, 0, len(itemData.Data)), + } + for _, e := range itemData.Data { + turn.FieldDataList = append(turn.FieldDataList, convert2EvaluationSetFieldData(ctx, e)) + } + turns = append(turns, turn) + } + return turns + } + data := item.Data if len(data) == 0 { return nil diff --git a/backend/modules/evaluation/infra/rpc/data/convert_test.go b/backend/modules/evaluation/infra/rpc/data/convert_test.go index aae030ef11..359dbf1a1f 100755 --- a/backend/modules/evaluation/infra/rpc/data/convert_test.go +++ b/backend/modules/evaluation/infra/rpc/data/convert_test.go @@ -15,6 +15,123 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" ) +func TestConvert2EvaluationSetItemReadTurns(t *testing.T) { + ctx := context.Background() + singular := []*dataset.FieldData{{ + Key: gptr.Of("case"), Name: gptr.Of("case"), + ContentType: gptr.Of(dataset.ContentType_Text), Format: gptr.Of(dataset.FieldDisplayFormat_PlainText), + Content: gptr.Of("legacy"), + }} + repeated := []*dataset.ItemData{ + {ID: gptr.Of(int64(1)), Data: []*dataset.FieldData{{ + Key: gptr.Of("case"), Name: gptr.Of("case"), + ContentType: gptr.Of(dataset.ContentType_Text), Format: gptr.Of(dataset.FieldDisplayFormat_PlainText), + Content: gptr.Of("S"), TraceID: gptr.Of("trace-1"), + }}}, + {ID: gptr.Of(int64(2)), Data: []*dataset.FieldData{{ + Key: gptr.Of("case"), Name: gptr.Of("case"), + ContentType: gptr.Of(dataset.ContentType_Text), Format: gptr.Of(dataset.FieldDisplayFormat_PlainText), + Content: gptr.Of("A"), TraceID: gptr.Of("trace-2"), + }}}, + } + wantRepeated := []*entity.Turn{ + { + ID: 1, ItemID: 7590130073438429442, EvalSetID: 7590130228007518722, + FieldDataList: []*entity.FieldData{{ + Key: "case", Name: "case", TraceID: "trace-1", + Content: &entity.Content{ + ContentType: gptr.Of(entity.ContentTypeText), + Format: gptr.Of(entity.PlainText), Text: gptr.Of("S"), + }, + }}, + }, + { + ID: 2, ItemID: 7590130073438429442, EvalSetID: 7590130228007518722, + FieldDataList: []*entity.FieldData{{ + Key: "case", Name: "case", TraceID: "trace-2", + Content: &entity.Content{ + ContentType: gptr.Of(entity.ContentTypeText), + Format: gptr.Of(entity.PlainText), Text: gptr.Of("A"), + }, + }}, + }, + } + wantSingular := []*entity.Turn{{ + ID: 0, ItemID: 7590130073438429442, EvalSetID: 7590130228007518722, + FieldDataList: []*entity.FieldData{{ + Key: "case", Name: "case", + Content: &entity.Content{ + ContentType: gptr.Of(entity.ContentTypeText), + Format: gptr.Of(entity.PlainText), Text: gptr.Of("legacy"), + }, + }}, + }} + tests := []struct { + name string + data []*dataset.FieldData + repeated []*dataset.ItemData + want []*entity.Turn + }{ + {name: "repeated_data_only", repeated: repeated, want: wantRepeated}, + {name: "repeated_data_takes_precedence", data: singular, repeated: repeated, want: wantRepeated}, + {name: "nil_turn_is_skipped", repeated: []*dataset.ItemData{repeated[0], nil, repeated[1]}, want: wantRepeated}, + {name: "all_nil_turns_do_not_fall_back", data: singular, repeated: []*dataset.ItemData{nil}, want: []*entity.Turn{}}, + { + name: "empty_turn_keeps_original_id", repeated: []*dataset.ItemData{{ID: gptr.Of(int64(7))}}, + want: []*entity.Turn{{ID: 7, ItemID: 7590130073438429442, EvalSetID: 7590130228007518722, FieldDataList: []*entity.FieldData{}}}, + }, + { + name: "nil_field_is_preserved", repeated: []*dataset.ItemData{{ID: gptr.Of(int64(9)), Data: []*dataset.FieldData{nil}}}, + want: []*entity.Turn{{ID: 9, ItemID: 7590130073438429442, EvalSetID: 7590130228007518722, FieldDataList: []*entity.FieldData{nil}}}, + }, + {name: "single_turn_id_zero_is_unchanged", data: singular, want: wantSingular}, + {name: "empty_repeated_data_falls_back", data: singular, repeated: []*dataset.ItemData{}, want: wantSingular}, + {name: "no_data_returns_nil"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + item := &dataset.DatasetItem{ + ID: gptr.Of(int64(9001)), AppID: gptr.Of(int32(566307)), + SpaceID: gptr.Of(int64(7372107779483369483)), DatasetID: gptr.Of(int64(7590130228007518722)), + SchemaID: gptr.Of(int64(9002)), ItemID: gptr.Of(int64(7590130073438429442)), + ItemKey: gptr.Of("mt-A"), Data: tt.data, RepeatedData: tt.repeated, + CreatedBy: gptr.Of("creator"), UpdatedBy: gptr.Of("updater"), + CreatedAt: gptr.Of(int64(1792718993000)), UpdatedAt: gptr.Of(int64(1792718994000)), + } + got := convert2EvaluationSetItem(ctx, item) + if got == nil { + t.Fatal("item conversion returned nil") + } + if len(got.Turns) != len(tt.want) { + t.Fatalf("turn count: want %d, got %d", len(tt.want), len(got.Turns)) + } + assert.Equal(t, &entity.EvaluationSetItem{ + ID: 9001, AppID: 566307, SpaceID: 7372107779483369483, + EvaluationSetID: 7590130228007518722, SchemaID: 9002, + ItemID: 7590130073438429442, ItemKey: "mt-A", Turns: tt.want, + BaseInfo: &entity.BaseInfo{ + CreatedBy: &entity.UserInfo{UserID: gptr.Of("creator")}, + UpdatedBy: &entity.UserInfo{UserID: gptr.Of("updater")}, + CreatedAt: gptr.Of(int64(1792718993000)), UpdatedAt: gptr.Of(int64(1792718994000)), + }, + }, got) + }) + } + t.Run("nil_item_and_batch_entry_remain_nil", func(t *testing.T) { + assert.Nil(t, convert2EvaluationSetItem(ctx, nil)) + assert.Nil(t, convert2EvaluationSetItems(ctx, nil)) + got := convert2EvaluationSetItems(ctx, []*dataset.DatasetItem{nil, { + ItemID: gptr.Of(int64(7590130073438429442)), + DatasetID: gptr.Of(int64(7590130228007518722)), RepeatedData: repeated, + }}) + if len(got) != 2 || got[1] == nil { + t.Fatal("batch conversion did not preserve both entries") + } + assert.Nil(t, got[0]) + assert.Equal(t, wantRepeated, got[1].Turns) + }) +} + func TestConvert2EvaluationSetFieldData(t *testing.T) { ctx := context.Background() diff --git a/backend/modules/evaluation/pkg/conf/expt_running_retry_test.go b/backend/modules/evaluation/pkg/conf/expt_running_retry_test.go new file mode 100644 index 0000000000..b6526efe1c --- /dev/null +++ b/backend/modules/evaluation/pkg/conf/expt_running_retry_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package conf + +import ( + "context" + "encoding/json" + "testing" + + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/pkg/conf" + confmocks "github.com/coze-dev/coze-loop/backend/pkg/conf/mocks" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestRunningRetryConfigPerSpaceReload(t *testing.T) { + loader := confmocks.NewMockIConfigLoader(gomock.NewController(t)) + raw := `{"expt_exec_conf":{"running_retry_enabled":true},"space_expt_exec_conf":{"9":{},"10":{"running_retry_enabled":false},"11":{"running_retry_enabled":true}}}` + loader.EXPECT().UnmarshalKey(gomock.Any(), "expt_consumer_conf", gomock.Any()).DoAndReturn(func(_ context.Context, _ string, value any, _ ...conf.DecodeOptionFn) error { + return json.Unmarshal([]byte(raw), value) + }).AnyTimes() + c := &configer{loader: loader} + require.False(t, c.GetExptExecConf(context.Background(), 9).IsRunningRetryEnabled()) + require.False(t, c.GetExptExecConf(context.Background(), 10).IsRunningRetryEnabled()) + require.True(t, c.GetExptExecConf(context.Background(), 11).IsRunningRetryEnabled()) + raw = `{}` + require.False(t, c.GetExptExecConf(context.Background(), 11).IsRunningRetryEnabled()) + var unset *entity.ExptExecConf + require.False(t, unset.IsRunningRetryEnabled()) +} diff --git a/backend/modules/evaluation/pkg/errno/evaluation.go b/backend/modules/evaluation/pkg/errno/evaluation.go index 01d60c882e..3cd1071de5 100644 --- a/backend/modules/evaluation/pkg/errno/evaluation.go +++ b/backend/modules/evaluation/pkg/errno/evaluation.go @@ -424,9 +424,7 @@ const ( exptMultiSetSpaceNotAllowedMessage = "the space is not allowed to create multi-set experiments" exptMultiSetSpaceNotAllowedNoAffectStability = true - // 沙箱 agent 评测对象上报结果错误码 (5073-5083)。沙箱侧上报时用下列细分码替代通用 CallTargetFail， - // 便于 metrics classifier / 前端展示按 reason 分类。触发要点见每条 comment。 - SandboxAgentModelNotFoundCode = 601205073 // sandbox agent target reported model_not_found (trigger: {code:10003, msg:"model X not found"}) + SandboxAgentModelNotFoundCode = 601205073 // sandbox agent target reported model_not_found (trigger: {code:10003, msg:"model X not found, allow:..."}) sandboxAgentModelNotFoundMessage = "sandbox agent: model not found" sandboxAgentModelNotFoundNoAffectStability = false @@ -454,7 +452,7 @@ const ( sandboxAgentSessionNotWaitingInputMessage = "sandbox agent: session not waiting input" sandboxAgentSessionNotWaitingInputNoAffectStability = true - SandboxAgentCmdForwardPortNotReadyCode = 601205080 // sandbox agent target reported cmd_forward_port_not_ready (trigger: port 16259 timeout, "Failed to connect to localhost port 16259", VSCode cmd exec failure, "[Errno 111] Connection refused") + SandboxAgentCmdForwardPortNotReadyCode = 601205080 // sandbox agent target reported cmd_forward_port_not_ready (trigger: port 16259 timeout / "Failed to connect to localhost port 16259" / VSCode cmd exec failure / "[Errno 111] Connection refused") sandboxAgentCmdForwardPortNotReadyMessage = "sandbox agent: cmd forward port not ready" sandboxAgentCmdForwardPortNotReadyNoAffectStability = false @@ -462,7 +460,7 @@ const ( sandboxAgentNetworkMessage = "sandbox agent: network error" sandboxAgentNetworkNoAffectStability = false - SandboxAgentIDEHostMismatchCode = 601205082 // sandbox agent target reported ide_host_mismatch (trigger: (a) "CN 环境机器不支持评测海外 IDE"; (b) /Applications/Trae.app/.../product.json missing and combined does not contain "Trae CN.app") + SandboxAgentIDEHostMismatchCode = 601205082 // sandbox agent target reported ide_host_mismatch (trigger: (a) explicit "CN 环境机器不支持评测海外 IDE"; (b) /Applications/Trae.app/.../product.json missing and combined does not contain "Trae CN.app") sandboxAgentIDEHostMismatchMessage = "sandbox agent: IDE host mismatch" sandboxAgentIDEHostMismatchNoAffectStability = true @@ -470,7 +468,6 @@ const ( sandboxAgentConfigNameMismatchMessage = "sandbox agent: config_name mismatch" sandboxAgentConfigNameMismatchNoAffectStability = true - // 实验/行级超时错误码（用于把超时原因暴露给用户，替代原来落库为空 err_msg 或英文 raw string 的实现） ExptZombieTimeoutCode = 601205084 // 实验整体运行时长超过阈值被超时终止（HandleEventCheck 触发） exptZombieTimeoutMessage = "实验已超过最大执行时长被终止" exptZombieTimeoutNoAffectStability = false @@ -485,17 +482,12 @@ const ( ItemQuotaImpossibleCode = 601205087 // 实验行申报的资源量超过调度域登记上限，任何配置下都无法调度（central scheduler 每拍授予点触发） itemQuotaImpossibleMessage = "实验行申报的资源量超过调度域上限，任何配置下都无法调度，已被置为失败" - itemQuotaImpossibleNoAffectStability = true // 结构性配置错误（申报>上限），非系统稳定性问题 + itemQuotaImpossibleNoAffectStability = true - // ItemManuallyTerminatedCode 行级终止（TerminateExperimentItems 触发）。用户主动放弃该行，不是系统故障， - // 故 noAffectStability = true，不计入稳定性口径。message 会经 err_msg 落库并由 ItemSystemInfo.Error 透给前端展示。 - // ⚠️ 原取 601205087，与 main 上新增的 ItemQuotaImpossibleCode 撞号（两边并行开发各自取了下一个空位）， - // 合并时本码顺延到 601205088；601205087 归 main 先到者。 - ItemManuallyTerminatedCode = 601205088 + ItemManuallyTerminatedCode = 601205088 // 行级终止（TerminateExperimentItems 触发）；用户主动终止，不计入稳定性口径 itemManuallyTerminatedMessage = "该行被用户主动终止" itemManuallyTerminatedNoAffectStability = true - // SandboxAgent 评测对象阶段性错误码 (601206xxx)：按沙箱内执行阶段划分，便于按阶段做 metrics 分类与用户前端展示。 SandboxAgentSetupErrorCode = 601206001 // sandbox agent target setup phase error: agent 初始化 / 环境依赖装载失败 sandboxAgentSetupErrorMessage = "sandbox agent: agent setup failed" sandboxAgentSetupErrorNoAffectStability = false @@ -523,6 +515,34 @@ const ( SandboxAgentUnknownErrorCode = 601206205 // sandbox agent target unknown error: 未能归类到上述阶段的兜底错误 sandboxAgentUnknownErrorMessage = "sandbox agent: unknown error" sandboxAgentUnknownErrorNoAffectStability = false + + ManualRetryRunConflictCode = 601204101 // same-run manual retry run_conflict + manualRetryRunConflictMessage = "run_conflict" + manualRetryRunConflictNoAffectStability = true + + ManualRetryRunFinishedCode = 601204102 // same-run manual retry run_finished + manualRetryRunFinishedMessage = "run_finished" + manualRetryRunFinishedNoAffectStability = true + + ManualRetryRequestConflictCode = 601204103 // same-run manual retry request_conflict + manualRetryRequestConflictMessage = "request_conflict" + manualRetryRequestConflictNoAffectStability = true + + ManualRetryCapacityCode = 601204104 // same-run manual retry receipt_capacity + manualRetryCapacityMessage = "receipt_capacity" + manualRetryCapacityNoAffectStability = true + + ManualRetryBusyCode = 601204105 // same-run manual retry retry_busy + manualRetryBusyMessage = "retry_busy" + manualRetryBusyNoAffectStability = true + + ManualRetryDisabledCode = 601204106 // same-run manual retry running retry disabled + manualRetryDisabledMessage = "running retry disabled" + manualRetryDisabledNoAffectStability = true + + ManualRetryIdentityRequiredCode = 601204107 // same-run manual retry identity required + manualRetryIdentityRequiredMessage = "identity_required" + manualRetryIdentityRequiredNoAffectStability = true ) func init() { @@ -1289,4 +1309,46 @@ func init() { code.WithAffectStability(!sandboxAgentUnknownErrorNoAffectStability), ) + code.Register( + ManualRetryRunConflictCode, + manualRetryRunConflictMessage, + code.WithAffectStability(!manualRetryRunConflictNoAffectStability), + ) + + code.Register( + ManualRetryRunFinishedCode, + manualRetryRunFinishedMessage, + code.WithAffectStability(!manualRetryRunFinishedNoAffectStability), + ) + + code.Register( + ManualRetryRequestConflictCode, + manualRetryRequestConflictMessage, + code.WithAffectStability(!manualRetryRequestConflictNoAffectStability), + ) + + code.Register( + ManualRetryCapacityCode, + manualRetryCapacityMessage, + code.WithAffectStability(!manualRetryCapacityNoAffectStability), + ) + + code.Register( + ManualRetryBusyCode, + manualRetryBusyMessage, + code.WithAffectStability(!manualRetryBusyNoAffectStability), + ) + + code.Register( + ManualRetryDisabledCode, + manualRetryDisabledMessage, + code.WithAffectStability(!manualRetryDisabledNoAffectStability), + ) + + code.Register( + ManualRetryIdentityRequiredCode, + manualRetryIdentityRequiredMessage, + code.WithAffectStability(!manualRetryIdentityRequiredNoAffectStability), + ) + } diff --git a/backend/script/errorx/evaluation.yaml b/backend/script/errorx/evaluation.yaml index 4798b0f94a..56015bb30d 100644 --- a/backend/script/errorx/evaluation.yaml +++ b/backend/script/errorx/evaluation.yaml @@ -562,6 +562,12 @@ error_code: description: the async eval target was terminated because the experiment was manually cancelled no_affect_stability: true + - name: ExptMultiSetSpaceNotAllowed + code: 5072 + message: "the space is not allowed to create multi-set experiments" + description: the space is not in the gray-release whitelist for multi-set experiments + no_affect_stability: true + # 沙箱 agent 评测对象上报结果错误码 (5073-5083)。5072 由既有 gray-release 白名单占用， # 沙箱侧上报时用下列细分码替代通用 CallTargetFail，便于 metrics classifier / 前端展示分类。 - name: SandboxAgentModelNotFound @@ -649,6 +655,18 @@ error_code: description: 'sandbox agent target sandbox execute reached terminal state (Failed/Canceled) before the async result was reported (sweep triggered by ExptSchedulerImpl.sweepTerminatedSandboxItems)' no_affect_stability: false + - name: ItemQuotaImpossible + code: 5087 + message: "实验行申报的资源量超过调度域上限，任何配置下都无法调度，已被置为失败" + description: '实验行申报的资源量超过调度域登记上限，任何配置下都无法调度（central scheduler 每拍授予点触发）' + no_affect_stability: true + + - name: ItemManuallyTerminated + code: 5088 + message: "该行被用户主动终止" + description: '行级终止（TerminateExperimentItems 触发）；用户主动终止，不计入稳定性口径' + no_affect_stability: true + # SandboxAgent 评测对象阶段性错误码 (6xxx)：按沙箱内执行阶段划分，便于按阶段做 metrics 分类与用户前端展示。 - name: SandboxAgentSetupError code: 6001 @@ -691,3 +709,46 @@ error_code: message: "sandbox agent: unknown error" description: 'sandbox agent target unknown error: 未能归类到上述阶段的兜底错误' no_affect_stability: false + + # Same-run manual retry admission refusals (7384603622). + - name: ManualRetryRunConflict + code: 4101 + message: run_conflict + description: same-run manual retry run_conflict + no_affect_stability: true + + - name: ManualRetryRunFinished + code: 4102 + message: run_finished + description: same-run manual retry run_finished + no_affect_stability: true + + - name: ManualRetryRequestConflict + code: 4103 + message: request_conflict + description: same-run manual retry request_conflict + no_affect_stability: true + + - name: ManualRetryCapacity + code: 4104 + message: receipt_capacity + description: same-run manual retry receipt_capacity + no_affect_stability: true + + - name: ManualRetryBusy + code: 4105 + message: retry_busy + description: same-run manual retry retry_busy + no_affect_stability: true + + - name: ManualRetryDisabled + code: 4106 + message: running retry disabled + description: same-run manual retry running retry disabled + no_affect_stability: true + + - name: ManualRetryIdentityRequired + code: 4107 + message: identity_required + description: same-run manual retry identity required + no_affect_stability: true diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift index bd81ce62c0..ef848360f0 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift @@ -320,14 +320,27 @@ struct RetryExperimentRequest { 3: optional i64 expt_id (api.path = 'expt_id', api.js_conv = 'true', go.tag = 'json:"expt_id"') 4: optional list item_ids (api.body = 'item_ids', api.js_conv = 'true', go.tag = 'json:"item_ids"') + 5: optional i64 expected_run_id (api.body = 'expected_run_id', api.js_conv = 'true', go.tag = 'json:"expected_run_id,omitempty"') + 6: optional string request_id (api.body = 'request_id') + 100: optional map ext (api.body = 'ext') 255: optional base.Base Base } +struct RetryRejectedItem { + 1: optional i64 item_id (api.js_conv = 'true', go.tag = 'json:"item_id,omitempty"') + 2: optional string reason +} + struct RetryExperimentResponse { 1: optional i64 run_id (api.body = 'run_id', api.js_conv = 'true', go.tag = 'json:"run_id"') + 2: optional string request_id + 3: optional i32 accepted_count + 4: optional list rejected_items + 5: optional i64 snapshot_at_ms (api.js_conv = 'true', go.tag = 'json:"snapshot_at_ms,omitempty"') + 255: base.BaseResp BaseResp } diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift index 1fe9d86fd1..3d03886ed2 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift @@ -880,6 +880,9 @@ struct RetryExperimentOApiRequest { 3: optional experiment.ExptRetryMode retry_mode (api.body = 'retry_mode') 4: optional list item_ids (api.body = 'item_ids', api.js_conv = 'true', go.tag = 'json:"item_ids"') + 5: optional i64 expected_run_id (api.body = 'expected_run_id', api.js_conv = 'true', go.tag = 'json:"expected_run_id,omitempty"') + 6: optional string request_id (api.body = 'request_id') + 100: optional map ext (api.body = 'ext') 255: optional base.Base Base @@ -894,7 +897,12 @@ struct RetryExperimentOApiResponse { } struct RetryExperimentOpenAPIData { - 3: optional i64 run_id + 3: optional i64 run_id (api.js_conv = 'true', go.tag = 'json:"run_id,omitempty"') + 4: optional string request_id + 5: optional i32 accepted_count + // Keep an explicit empty list in a current-run receipt. + 6: optional list rejected_items (go.tag = 'json:"rejected_items"') + 7: optional i64 snapshot_at_ms (api.js_conv = 'true', go.tag = 'json:"snapshot_at_ms,omitempty"') } // KillExperimentOApiRequest 通过 OpenAPI 终止一个进行中的实验。 diff --git a/idl/thrift/coze/loop/evaluation/domain/expt.thrift b/idl/thrift/coze/loop/evaluation/domain/expt.thrift index a18495d9e7..1aac934d1a 100644 --- a/idl/thrift/coze/loop/evaluation/domain/expt.thrift +++ b/idl/thrift/coze/loop/evaluation/domain/expt.thrift @@ -272,6 +272,7 @@ struct Experiment { 118: optional ExpectedQuotaConsumption expected_quota_consumption // 从 experiment.eval_conf 回显；缺省表示普通评测。 119: optional VerificationConfig verification_config + 120: optional i64 latest_run_id (api.js_conv='true', go.tag='json:"latest_run_id,omitempty"') // 注: scheduler_scope **刻意不进读视图**。它是不透明调度域 ID (形如 fornax_cn_prod), // 对调用方没有可用语义却泄露部署拓扑; 业务代码本就不允许解析该字符串, 回显只会诱使 diff --git a/idl/thrift/coze/loop/evaluation/domain_openapi/experiment.thrift b/idl/thrift/coze/loop/evaluation/domain_openapi/experiment.thrift index 7fb4a93361..e3ade60d4e 100644 --- a/idl/thrift/coze/loop/evaluation/domain_openapi/experiment.thrift +++ b/idl/thrift/coze/loop/evaluation/domain_openapi/experiment.thrift @@ -355,6 +355,7 @@ struct Experiment { // 单 item 预期资源消耗向量; "有则回显、无则省略"(legacy 实验确实没申报)。 118: optional ExpectedQuotaConsumption expected_quota_consumption 119: optional VerificationConfig verification_config + 120: optional i64 latest_run_id (api.js_conv = 'true', go.tag = 'json:"latest_run_id,omitempty"') // 注: scheduler_scope **不进读模型**。它是不透明调度域 ID, 对调用方无可用语义却泄露部署拓扑; // 内部运维需要时直接查 experiment.scheduler_scope 列。 @@ -363,6 +364,11 @@ struct Experiment { 100: optional common.BaseInfo base_info } +struct RetryRejectedItem { + 1: optional i64 item_id (api.js_conv = 'true', go.tag = 'json:"item_id,omitempty"') + 2: optional string reason +} + // 单资源预期消耗。与 domain/expt.thrift 的 ExpectedResourceConsumption 同构 —— // 本文件另立一份而非 include, 与 RunModeConfig / ExptEvalSetSourceType 同套模式, 避免符号冲突。 struct ExpectedResourceConsumption {