feat: finalize admin UI and routing improvements

Author: nextroad-dev

.github/workflows/ci.yml

@@ -47,6 +47,10 @@ jobs:
         working-directory: webui
       - run: npm test
         working-directory: webui
+      - run: npx playwright install --with-deps chromium
+        working-directory: webui
+      - run: npm run test:e2e
+        working-directory: webui
       - run: npm run typecheck
         working-directory: webui
       - run: npm run build

README.md

@@ -30,7 +30,9 @@ docker rm auto-router
 
 ## 自动路由
 
-将请求中的 `model` 设为 `auto`，即可请求自动路由。启用 Jev 时，它推荐简单、中等或复杂路由组；本地策略根据请求特征、模型能力和上下文窗口过滤候选，再从所选模型组中决定具体上游。Jev 不可用或未启用时，服务仍按本地路由策略处理请求。
+将请求中的 `model` 设为 `auto`，即可请求自动路由。启用 Jev 时，它只在当前有合格成员的 `simple`、`medium`、`complex` 组中选择任务组；本地策略先按请求特征、模型能力、上下文窗口及硬限制过滤成员，再严格按组内配置顺序选择主模型。不会跨组重试，也不会由本地评分改变组内主备顺序。
+
+Jev 不可用、未启用或推荐置信度低于 `routing.policy.low_confidence` 时，使用 `routing.auto.default_group`（默认 `medium`）；所选组没有合格成员时明确失败，不自动切换任务等级。故障转移只尝试同组后续成员。`routing.auto.failover.max_attempts` 统计首次请求，范围为 1–8，默认 2；请求发送前可确认未送达的失败默认允许重试，超时与 HTTP 状态码默认不重试。启用超时或状态码重试可能导致重复计费或重复执行，状态码仅支持 408、425、429、500、502、503、504。以上设置可在管理台“系统设置 → 全局路由行为”调整。
 
 `GET /v1/models` 始终首先列出虚拟模型 `auto`，默认声明：
 
@@ -82,7 +84,7 @@ Anthropic 和 Gemini 原生接口要求所选上游支持相同协议。OpenAI C
 
 ## 发布前验证
 
-CI 运行 Go vet/test/race/build 与 WebUI 的 OpenAPI 类型生成一致性检查、Vitest、类型检查和生产构建；Playwright E2E 不在 CI 中运行，发布前需本地执行并检查失败 trace：
+CI 运行 Go vet/test/race/build 与 WebUI 的 OpenAPI 类型生成一致性检查、Vitest、Playwright E2E、类型检查、生产构建和嵌入资源一致性检查。发布前仍建议本地复跑 E2E，并检查失败 trace：
 
 ```sh
 cd webui

cmd/auto-router/logs_check.go

@@ -172,6 +172,8 @@ func writeStoredTrace(output io.Writer, trace *logging.JevTrace) {
 		trace.Status, orNone(trace.FailureReason), trace.InputMode, formatStoredOptionalInt(trace.LatencyMS))
 	fmt.Fprintf(output, "  jev model_count=%d candidate_count=%d selected=%s\n",
 		trace.ModelCount, trace.CandidateCount, orNone(trace.Selected))
+	fmt.Fprintf(output, "  jev group_count=%d recommended_group=%s selected_group=%s candidate_groups=%s\n",
+		trace.GroupCount, orNone(trace.RecommendedGroup), orNone(trace.SelectedGroup), formatStringsOrNone(trace.CandidateGroups))
 	fmt.Fprintf(output, "  jev confidence=%s band=%s fallback=%s evidence_hash=%s\n",
 		formatStoredOptionalFloat(trace.Confidence), orNone(trace.ConfidenceBand),
 		orNone(trace.FallbackReason), orNone(trace.EvidenceHash))
@@ -181,6 +183,11 @@ func writeStoredTrace(output io.Writer, trace *logging.JevTrace) {
 		distribution = append(distribution, fmt.Sprintf("%s=%.6f", probability.Model, probability.Probability))
 	}
 	fmt.Fprintf(output, "  jev distribution=%s\n", formatStringsOrNone(distribution))
+	groupDistribution := make([]string, 0, len(trace.GroupProbabilities))
+	for _, probability := range trace.GroupProbabilities {
+		groupDistribution = append(groupDistribution, fmt.Sprintf("%s=%.6f", probability.Group, probability.Probability))
+	}
+	fmt.Fprintf(output, "  jev group_distribution=%s\n", formatStringsOrNone(groupDistribution))
 }
 
 // logsCheckJSONEntry is the JSON shape of one report line. The keys match the text
@@ -232,19 +239,24 @@ type logsCheckJSONEvent struct {
 }
 
 type logsCheckJSONJev struct {
-	Status          string    `json:"status"`
-	FailureReason   string    `json:"failure_reason,omitempty"`
-	InputMode       string    `json:"input_mode"`
-	Selected        string    `json:"selected,omitempty"`
-	ConfidenceBand  string    `json:"confidence_band,omitempty"`
-	FallbackReason  string    `json:"fallback_reason,omitempty"`
-	EvidenceHash    string    `json:"evidence_hash,omitempty"`
-	LatencyMS       *int64    `json:"latency_ms,omitempty"`
-	CandidateModels []string  `json:"candidate_models"`
-	ModelCount      int       `json:"model_count"`
-	CandidateCount  int       `json:"candidate_count"`
-	Confidence      *float64  `json:"confidence,omitempty"`
-	Probabilities   []float64 `json:"probabilities,omitempty"`
+	Status             string                     `json:"status"`
+	FailureReason      string                     `json:"failure_reason,omitempty"`
+	InputMode          string                     `json:"input_mode"`
+	Selected           string                     `json:"selected,omitempty"`
+	ConfidenceBand     string                     `json:"confidence_band,omitempty"`
+	FallbackReason     string                     `json:"fallback_reason,omitempty"`
+	EvidenceHash       string                     `json:"evidence_hash,omitempty"`
+	LatencyMS          *int64                     `json:"latency_ms,omitempty"`
+	CandidateModels    []string                   `json:"candidate_models"`
+	ModelCount         int                        `json:"model_count"`
+	CandidateCount     int                        `json:"candidate_count"`
+	CandidateGroups    []string                   `json:"candidate_groups"`
+	GroupCount         int                        `json:"group_count"`
+	RecommendedGroup   string                     `json:"recommended_group,omitempty"`
+	SelectedGroup      string                     `json:"selected_group,omitempty"`
+	Confidence         *float64                   `json:"confidence,omitempty"`
+	Probabilities      []float64                  `json:"probabilities,omitempty"`
+	GroupProbabilities []logging.GroupProbability `json:"group_probabilities,omitempty"`
 }
 
 // writeLogsJSON prints one JSON object per entry. The values are the stored ones:
@@ -296,18 +308,23 @@ func writeLogsJSON(output io.Writer, events []storage.StoredEvent) error {
 		if entry.JevTrace != nil {
 			trace := entry.JevTrace
 			rendered := &logsCheckJSONJev{
-				Status:          trace.Status,
-				FailureReason:   trace.FailureReason,
-				InputMode:       trace.InputMode,
-				Selected:        trace.Selected,
-				ConfidenceBand:  trace.ConfidenceBand,
-				FallbackReason:  trace.FallbackReason,
-				EvidenceHash:    trace.EvidenceHash,
-				LatencyMS:       trace.LatencyMS,
-				CandidateModels: trace.CandidateModels,
-				ModelCount:      trace.ModelCount,
-				CandidateCount:  trace.CandidateCount,
-				Confidence:      trace.Confidence,
+				Status:             trace.Status,
+				FailureReason:      trace.FailureReason,
+				InputMode:          trace.InputMode,
+				Selected:           trace.Selected,
+				ConfidenceBand:     trace.ConfidenceBand,
+				FallbackReason:     trace.FallbackReason,
+				EvidenceHash:       trace.EvidenceHash,
+				LatencyMS:          trace.LatencyMS,
+				CandidateModels:    trace.CandidateModels,
+				ModelCount:         trace.ModelCount,
+				CandidateCount:     trace.CandidateCount,
+				CandidateGroups:    trace.CandidateGroups,
+				GroupCount:         trace.GroupCount,
+				RecommendedGroup:   trace.RecommendedGroup,
+				SelectedGroup:      trace.SelectedGroup,
+				Confidence:         trace.Confidence,
+				GroupProbabilities: trace.GroupProbabilities,
 			}
 			if rendered.CandidateModels == nil {
 				rendered.CandidateModels = []string{}

cmd/auto-router/main.go

@@ -297,9 +297,6 @@ func run(ctx context.Context, args []string, output io.Writer) error {
 		SyncRegistry: func(syncCtx context.Context) (any, error) {
 			return syncGate.Do(syncCtx, func(syncCtx context.Context) (any, error) {
 				current := settingsStore.Config()
-				if len(current.Registry.Sync.Include) == 0 {
-					return nil, api.ErrSyncAllowlistEmpty
-				}
 				if err := syncRegistry(syncCtx, current, db, logger); err != nil {
 					return nil, err
 				}
@@ -452,7 +449,10 @@ func newAutoRouter(cfg config.Config, store *models.Store, engine *policy.Engine
 		Engine:            engine,
 		DefaultPreference: cfg.Routing.DefaultPreference,
 		InputMode:         cfg.Jev.DomainInputMode(),
-		FailoverEnabled:   cfg.Routing.Auto.Failover.Enabled,
+		Failover:          autoFailoverPolicy(cfg.Routing.Auto.Failover),
+		DefaultGroup:      auto.GroupName(cfg.Routing.Auto.DefaultGroup),
+		LowConfidence:     cfg.Routing.Policy.LowConfidence,
+		HighConfidence:    cfg.Routing.Policy.HighConfidence,
 		GroupConfig:       groupConfigFromStorage(groups),
 	}
 	if settingsStore != nil {
@@ -493,12 +493,29 @@ func groupConfigFromStorage(groups *atomic.Pointer[storage.ModelGroups]) auto.Gr
 	return auto.GroupConfig{Simple: convert(stored.Simple), Medium: convert(stored.Medium), Complex: convert(stored.Complex)}
 }
 
+func autoFailoverPolicy(cfg config.RoutingAutoFailoverConfig) auto.FailoverPolicy {
+	return auto.FailoverPolicy{
+		Enabled:                cfg.Enabled,
+		MaxAttempts:            cfg.MaxAttempts,
+		RetryPreRequestFailure: cfg.RetryOn.PreRequestFailure,
+		RetryTimeout:           cfg.RetryOn.Timeout,
+		RetryStatusCodes:       append([]int(nil), cfg.RetryOn.StatusCodes...),
+	}
+}
+
 func autoRuntimeSnapshot(snapshot *settings.Snapshot, groups *atomic.Pointer[storage.ModelGroups]) auto.RuntimeSettings {
 	var client auto.JevCaller
 	if prepared, ok := snapshot.Runtime.(*jev.Client); ok && prepared != nil {
 		client = prepared
 	}
-	return auto.RuntimeSettings{Engine: snapshot.Engine, JevEnabled: snapshot.Config.Jev.Enabled, Jev: client, InputMode: snapshot.Config.Jev.DomainInputMode(), FailoverEnabled: snapshot.Config.Routing.Auto.Failover.Enabled, DefaultPreference: snapshot.Config.Routing.DefaultPreference, GroupConfig: groupConfigFromStorage(groups)}
+	cfg := snapshot.Config
+	return auto.RuntimeSettings{
+		Engine: snapshot.Engine, JevEnabled: cfg.Jev.Enabled, Jev: client,
+		InputMode: cfg.Jev.DomainInputMode(), Failover: autoFailoverPolicy(cfg.Routing.Auto.Failover),
+		DefaultGroup:  auto.GroupName(cfg.Routing.Auto.DefaultGroup),
+		LowConfidence: cfg.Routing.Policy.LowConfidence, HighConfidence: cfg.Routing.Policy.HighConfidence,
+		DefaultPreference: cfg.Routing.DefaultPreference, GroupConfig: groupConfigFromStorage(groups),
+	}
 }
 func (v settingsValues) RuntimeSnapshot() auto.RuntimeSettings {
 	return autoRuntimeSnapshot(v.store.Current(), v.groups)

cmd/auto-router/policy_check.go

@@ -96,7 +96,6 @@ type policyCheckCandidate struct {
 	SupportsVision     bool   `json:"supports_vision"`
 	SupportsReasoning  bool   `json:"supports_reasoning"`
 	SupportsAudioInput bool   `json:"supports_audio_input"`
-	PairPriority       int    `json:"pair_priority"`
 	ProviderPriority   int    `json:"provider_priority"`
 }
 
@@ -248,7 +247,6 @@ func policyCandidates(path string) ([]decision.Candidate, string, error) {
 			SupportsVision:     entry.SupportsVision,
 			SupportsReasoning:  entry.SupportsReasoning,
 			SupportsAudioInput: entry.SupportsAudioInput,
-			PairPriority:       entry.PairPriority,
 			ProviderPriority:   entry.ProviderPriority,
 		})
 	}
@@ -349,22 +347,22 @@ func builtinPolicyCandidates() []decision.Candidate {
 			ModelID: "gpt-4o", ProviderKey: "openai", GatewayModel: "gpt-4o",
 			ContextWindow: 128000, MaxOutput: &output,
 			SupportsTools: true, SupportsVision: true,
-			PairPriority: 0, ProviderPriority: 0,
+			ProviderPriority: 0,
 		}, {
 			ModelID: "gpt-4o-mini", ProviderKey: "openai", GatewayModel: "gpt-4o-mini",
 			ContextWindow: 128000, MaxOutput: &output,
 			SupportsTools: true, SupportsVision: true,
-			PairPriority: 1, ProviderPriority: 0,
+			ProviderPriority: 0,
 		}, {
 			ModelID: "o4-mini", ProviderKey: "openai", GatewayModel: "o4-mini",
 			ContextWindow: 200000, MaxOutput: &output,
 			SupportsTools: true, SupportsReasoning: true,
-			PairPriority: 2, ProviderPriority: 0,
+			ProviderPriority: 0,
 		}, {
 			ModelID: "llama-3.3-70b", ProviderKey: "groq", GatewayModel: "llama-3.3-70b-versatile",
 			ContextWindow: 131072, MaxOutput: &output,
-			SupportsTools: true,
-			PairPriority:  3, ProviderPriority: 1,
+			SupportsTools:    true,
+			ProviderPriority: 1,
 		},
 	}
 }

cmd/auto-router/registry.go

@@ -43,19 +43,22 @@ func (g *registrySyncGate) Do(ctx context.Context, run func(context.Context) (an
 	return run(ctx)
 }
 
-// syncRegistry fetches the models.dev catalog, imports the allowlisted models
-// and records the outcome. It never rewrites local overrides and never deletes
-// rows. On any error the transaction is rolled back and the database keeps its
-// previous registry.
+var errNoMappableRegistryModels = errors.New("models.dev catalog contains no models with a usable context window")
+
+// syncRegistry fetches models.dev and imports every mappable catalog entry when
+// no explicit include is configured. A non-empty include remains an explicit
+// scope override. Entries without usable context metadata are skipped with
+// warnings; a full-catalog sync with no mappable pairs fails before storage can
+// disable rows from the previous catalog. Local overrides and rows are never
+// deleted. On any error the transaction is rolled back and the registry is kept.
 func syncRegistry(ctx context.Context, cfg config.Config, db *sql.DB, logger *slog.Logger) error {
-	if len(cfg.Registry.Sync.Include) == 0 {
-		return errors.New("registry.sync.include is empty; refusing to synchronize an empty allowlist")
-	}
+	include := append([]string(nil), cfg.Registry.Sync.Include...)
+	fullCatalogScope := len(include) == 0
 	client, err := modelsdev.NewClient(cfg.Registry.Sync.URL, time.Duration(cfg.Registry.Sync.Timeout))
 	if err != nil {
 		return err
 	}
-	logger.Info("model registry sync starting", "url", client.URL(), "allowlisted_entries", len(cfg.Registry.Sync.Include))
+
 	payload, err := client.Fetch(ctx)
 	if err != nil {
 		return err
@@ -64,18 +67,31 @@ func syncRegistry(ctx context.Context, cfg config.Config, db *sql.DB, logger *sl
 	if err != nil {
 		return err
 	}
-	result, err := modelsdev.Map(document, cfg.Registry.Sync.Include)
+	var result *modelsdev.Result
+	if fullCatalogScope {
+		result, include, err = modelsdev.MapAll(document)
+		if err == nil && len(result.Pairs) == 0 {
+			return errNoMappableRegistryModels
+		}
+	} else {
+		result, err = modelsdev.Map(document, include)
+	}
 	if err != nil {
 		return err
 	}
+	scopeSource := "explicit_include"
+	if fullCatalogScope {
+		scopeSource = "modelsdev_catalog"
+	}
+	logger.Info("model registry sync starting", "url", client.URL(), "scope_entries", len(include), "scope_source", scopeSource)
 	summary, err := storage.ApplyRegistryImport(ctx, db, models.SourceModelsDev, storage.Import{
 		Providers: result.Providers,
 		Models:    result.Models,
 		Pairs:     result.Pairs,
 		Sync: &storage.SyncState{
 			URL:             client.URL(),
 			FetchedAt:       time.Now().UTC(),
-			AllowlistDigest: modelsdev.IncludeDigest(cfg.Registry.Sync.Include),
+			AllowlistDigest: modelsdev.IncludeDigest(include),
 			ImportedPairs:   len(result.Pairs),
 			SkippedPairs:    result.Skipped,
 			Warnings:        result.Warnings,
@@ -103,7 +119,7 @@ func logImportSummary(logger *slog.Logger, message string, summary storage.Impor
 	logger.Info(message,
 		slog.Group("providers", "added", summary.ProvidersAdded, "updated", summary.ProvidersUpdated, "disabled", summary.ProvidersDisabled, "admin_owned", summary.ProvidersAdminOwned),
 		slog.Group("models", "added", summary.ModelsAdded, "updated", summary.ModelsUpdated, "disabled", summary.ModelsDisabled, "admin_owned", summary.ModelsAdminOwned),
-		slog.Group("pairs", "added", summary.PairsAdded, "updated", summary.PairsUpdated, "disabled", summary.PairsDisabled, "admin_owned", summary.PairsAdminOwned),
+		slog.Group("pairs", "added", summary.PairsAdded, "updated", summary.PairsUpdated, "disabled", summary.PairsDisabled, "excluded", summary.PairsExcluded, "admin_owned", summary.PairsAdminOwned),
 	)
 	if summary.ProvidersAdminOwned+summary.ModelsAdminOwned+summary.PairsAdminOwned > 0 {
 		// This is not a warning: skipping administrator-owned rows is the documented