diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index c901adc6f..ffe31aba3 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -62,6 +62,9 @@ jobs: id: build run: | go build -ldflags "-X github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/cli.Version=$(git rev-parse HEAD)" -o "$RUNNER_TEMP/oac-daemon${{ runner.os == 'Windows' && '.exe' || '' }}" ./apps/daemon/cmd/oac-daemon + if [[ "$RUNNER_OS" == Linux ]]; then + CGO_ENABLED=0 go build -mod=readonly -trimpath -o "$RUNNER_TEMP/oac-sandbox-io" ./apps/sandboxio/cmd/oac-sandbox-io + fi node --test scripts/build-native-installer.test.mjs - name: Build and test the shared Core installer run: | @@ -128,6 +131,7 @@ jobs: node scripts/build-native-installer-ci.mjs - name: Bootstrap, authenticate, install and connect natively id: onboarding + if: runner.os == 'Linux' run: node scripts/native-onboarding-smoke.mjs - name: Verify native Harness protocols without model requests id: protocol diff --git a/apps/daemon/internal/cli/connect_environment.go b/apps/daemon/internal/cli/connect_environment.go index 0b1c3b4b1..cd2c0813b 100644 --- a/apps/daemon/internal/cli/connect_environment.go +++ b/apps/daemon/internal/cli/connect_environment.go @@ -85,32 +85,45 @@ func decodeEnvironmentJSON(raw []byte, value any) error { func enrollEnvironment(ctx context.Context, client *http.Client, base, environment, credential string) (environmentEnrollment, error) { var out environmentEnrollment + raw, err := requestEnrollment(ctx, client, base, environment, credential) + if err != nil { + return out, err + } + if decodeEnvironmentJSON(raw, &out) != nil || !environmentUUID(out.DeviceID) || !environmentUUID(out.SessionID) || out.EnvironmentID != environment || out.WorkspaceDirectory == "/" || agentcapabilities.ValidateLocalDirectories([]string{out.WorkspaceDirectory}) != nil { + return environmentEnrollment{}, errors.New("connect: invalid Environment enrollment response") + } + return out, nil +} + +// requestEnrollment returns the body of a successful enrollment; each caller +// decodes the response it expects. +func requestEnrollment(ctx context.Context, client *http.Client, base, environment, credential string) ([]byte, error) { body, _ := json.Marshal(map[string]string{"environment_id": environment}) req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/agent-daemon/enroll", bytes.NewReader(body)) if err != nil { - return out, errors.New("connect: invalid enrollment request") + return nil, errors.New("connect: invalid enrollment request") } req.Header.Set("Authorization", "Bearer "+credential) req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { - return out, errors.New("connect: Environment enrollment transport failed") + return nil, errors.New("connect: Environment enrollment transport failed") } defer resp.Body.Close() switch resp.StatusCode { case http.StatusOK: case http.StatusUnauthorized: - return out, errEnvironmentCredentialRejected + return nil, errEnvironmentCredentialRejected case http.StatusConflict: - return out, errEnvironmentBindingConflict + return nil, errEnvironmentBindingConflict default: - return out, fmt.Errorf("connect: Environment enrollment rejected (HTTP %d)", resp.StatusCode) + return nil, fmt.Errorf("connect: Environment enrollment rejected (HTTP %d)", resp.StatusCode) } raw, err := io.ReadAll(io.LimitReader(resp.Body, 16*1024+1)) - if err != nil || len(raw) > 16*1024 || decodeEnvironmentJSON(raw, &out) != nil || !environmentUUID(out.DeviceID) || !environmentUUID(out.SessionID) || out.EnvironmentID != environment || out.WorkspaceDirectory == "/" || agentcapabilities.ValidateLocalDirectories([]string{out.WorkspaceDirectory}) != nil { - return environmentEnrollment{}, errors.New("connect: invalid Environment enrollment response") + if err != nil || len(raw) > 16*1024 { + return nil, errors.New("connect: invalid Environment enrollment response") } - return out, nil + return raw, nil } func environmentBootstrap(ctx context.Context, prof auth.Profile, remote string) (*transport.BootstrapResponse, error) { diff --git a/apps/daemon/internal/cli/native_harness.go b/apps/daemon/internal/cli/native_harness.go index 69aa94b91..cf831a29a 100644 --- a/apps/daemon/internal/cli/native_harness.go +++ b/apps/daemon/internal/cli/native_harness.go @@ -91,13 +91,3 @@ func checkNativeInstallation(ctx context.Context, root string, selected []string } return nil } - -// Installed discovery is confined to verified adapters; ordinary tool PATH -// remains available to the selected Harness and its tools. -func nativeInstallationKinds(selected []string) map[string]bool { - kinds := make(map[string]bool, len(selected)) - for _, name := range selected { - kinds[nativeHarnesses[name].AgentKind] = true - } - return kinds -} diff --git a/apps/daemon/internal/cli/native_install.go b/apps/daemon/internal/cli/native_install.go index 9e0f87695..5347875fb 100644 --- a/apps/daemon/internal/cli/native_install.go +++ b/apps/daemon/internal/cli/native_install.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "slices" + "strings" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/daemonize" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" @@ -178,7 +179,7 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO return err } } - if err = nativeInstallPhase(rc.stdout, "Installing Runtime", func() error { return installNativeBinary(ctx, o.Directory, len(previous.Harnesses) > 0) }); err != nil { + if err = nativeInstallPhase(rc.stdout, "Installing Runtime", func() error { return installNativeBinary(ctx, o.Bundle, o.Directory, len(previous.Harnesses) > 0) }); err != nil { return err } for _, name := range append([]string{"node"}, selected...) { @@ -199,7 +200,7 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO } fmt.Fprintln(rc.stdout, "Installation: ready; verified Harnesses:", all) if o.OnboardURL == "" { - fmt.Fprintln(rc.stdout, "Daemon connection: not checked by install; run the installed oac-daemon start, then check Host connection in Core.") + fmt.Fprintln(rc.stdout, "Host connection: not checked by install; run the installed oac-daemon start, then check Host connection in Core.") } fmt.Fprintln(rc.stdout, "Model configuration: not checked; configure the Session model provider in Core and send a Turn.") return nil @@ -224,7 +225,9 @@ func verifyNativeComponents(ctx context.Context, root string, selected []string) return nil } -func installNativeBinary(ctx context.Context, root string, existing bool) error { +// installNativeBinary installs the running oac-daemon and the distribution's +// other programs (nativeBundlePrograms) into bin. +func installNativeBinary(ctx context.Context, bundle, root string, existing bool) error { exe, err := os.Executable() if err != nil { return err @@ -233,7 +236,19 @@ func installNativeBinary(ctx context.Context, root string, existing bool) error if err = os.MkdirAll(dir, 0700); err != nil { return err } - dest := filepath.Join(dir, nativeExe("oac-daemon")) + if err = installNativeProgram(ctx, exe, dir, nativeExe("oac-daemon"), existing); err != nil { + return err + } + for _, name := range nativeBundlePrograms { + if err = installNativeProgram(ctx, filepath.Join(bundle, name), dir, name, existing); err != nil { + return err + } + } + return nil +} + +func installNativeProgram(ctx context.Context, source, dir, name string, existing bool) error { + dest := filepath.Join(dir, name) digest := func(name string) (string, error) { f, e := os.Open(name) if e != nil { @@ -244,22 +259,25 @@ func installNativeBinary(ctx context.Context, root string, existing bool) error _, e = nativeCopy(ctx, h, f) return hex.EncodeToString(h.Sum(nil)), e } - want, err := digest(exe) + want, err := digest(source) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("install: the distribution has no %s; use the matching native distribution", name) + } if err != nil { return err } if got, e := digest(dest); e == nil { if got != want { - return errors.New("install: existing daemon binary differs; in-place upgrades are unsupported") + return fmt.Errorf("install: existing %s differs; in-place upgrades are unsupported", name) } return nil } else if !errors.Is(e, os.ErrNotExist) { return e } if existing { - return errors.New("install: existing daemon binary is missing; preserve the installation and reinstall separately") + return fmt.Errorf("install: existing %s is missing; preserve the installation and reinstall separately", name) } - in, err := os.Open(exe) + in, err := os.Open(source) if err != nil { return err } @@ -271,7 +289,7 @@ func installNativeBinary(ctx context.Context, root string, existing bool) error if err = requireNativeSpace(dir, uint64(info.Size())); err != nil { return err } - out, err := os.CreateTemp(dir, ".oac-daemon-") + out, err := os.CreateTemp(dir, "."+strings.TrimSuffix(name, ".exe")+"-") if err != nil { return err } @@ -320,32 +338,11 @@ func runStart(rc *runContext, args []string) error { if err == nil { err = validateNativeInstallation(config) } - if err == nil && len(config.Harnesses) == 0 { - err = errors.New("start: no installed Harnesses; rerun install with --harness") - } - if err == nil { - err = verifyNativeComponents(ctx, root, config.Harnesses) - } - if err == nil { - err = probeNativeInstallation(ctx, root, config.Harnesses) - } unlock() if err != nil { return fmt.Errorf("start: installation unavailable or incompatible: %w", err) } - previousKinds := rc.installedKinds - rc.installedKinds = nativeInstallationKinds(config.Harnesses) - defer func() { rc.installedKinds = previousKinds }() - values := nativeHarnessEnvironment(root, config.Harnesses) - values["OAC_RUNTIME_WORKSPACE"] = config.Workspace - values["OAC_RUNTIME_CAPABILITY_DIRECTORY"] = config.CapabilityDirectory - values["OAC_RUNTIME_TOOL_ENV_FILE"] = config.ToolEnvironmentFile - for key, value := range values { - if err = os.Setenv(key, value); err != nil { - return err - } - } - return runEnvironmentConnect(ctx, rc, paths.DefaultProfile, !*foreground, config.Remote, config.Environment, config.Credential) + return runSandboxLauncher(ctx, rc, !*foreground, root, config) } func useInstalledNativeHome() { diff --git a/apps/daemon/internal/cli/native_install_io.go b/apps/daemon/internal/cli/native_install_io.go index aa2060641..a3441b468 100644 --- a/apps/daemon/internal/cli/native_install_io.go +++ b/apps/daemon/internal/cli/native_install_io.go @@ -14,8 +14,8 @@ import ( // These exact temporary names are reserved by the installer, never workspace data. var nativeTemporaryNames = map[string]*regexp.Regexp{ "components": regexp.MustCompile(`^\.install-(node|codex|claude|minimax)-[0-9]+$`), - "bin": regexp.MustCompile(`^\.oac-daemon-[0-9]+$`), - "daemon": regexp.MustCompile(`^\.(installation\.json|executor-credential\.json)-[0-9a-f]{24}\.tmp$`), + "bin": regexp.MustCompile(`^\.(oac-daemon|oac-sandbox-io)-[0-9]+$`), + "daemon": regexp.MustCompile(`^\.(installation\.json|executor-credential\.json|sandbox-io-bootstrap\.json)-[0-9a-f]{24}\.tmp$`), } // The caller holds the installation lock, including while recovering a failed copy. diff --git a/apps/daemon/internal/cli/native_install_test.go b/apps/daemon/internal/cli/native_install_test.go index 2a8453d1f..1ab045f64 100644 --- a/apps/daemon/internal/cli/native_install_test.go +++ b/apps/daemon/internal/cli/native_install_test.go @@ -41,6 +41,11 @@ func nativeInstallFixture(t *testing.T) (*runContext, []string, string, string) t.Fatal(err) } } + for _, name := range nativeBundlePrograms { + if err := os.WriteFile(filepath.Join(bundle, name), []byte("#!/bin/sh\nexit 0\n"), 0700); err != nil { + t.Fatal(err) + } + } raw, _ := json.Marshal(b) if err := os.WriteFile(filepath.Join(bundle, "bundle.json"), raw, 0600); err != nil { t.Fatal(err) diff --git a/apps/daemon/internal/cli/native_onboarding.go b/apps/daemon/internal/cli/native_onboarding.go index cfecf5cb7..66aec60c7 100644 --- a/apps/daemon/internal/cli/native_onboarding.go +++ b/apps/daemon/internal/cli/native_onboarding.go @@ -163,7 +163,7 @@ func finishOnboarding(ctx context.Context, rc *runContext, o nativeInstallOption command.Env = withNativeEnv(map[string]string{"OAC_RUNTIME_HOME": o.Directory, daemonize.BackgroundSentinelEnv: ""}) command.Stdout, command.Stderr = rc.stdout, rc.stderr if err := command.Run(); err != nil { - fmt.Fprintln(rc.stderr, "Daemon connection: start failed. Rerun this command to resume, or run the installed oac-daemon start.") + fmt.Fprintln(rc.stderr, "Host connection: start failed. Rerun this command to resume, or run the installed oac-daemon start.") return errors.New("install: daemon startup failed") } } @@ -182,12 +182,12 @@ func finishOnboarding(ctx context.Context, rc *runContext, o nativeInstallOption return err } if connected { - fmt.Fprintln(rc.stdout, "Daemon connection: connected to Core.") + fmt.Fprintln(rc.stdout, "Host connection: connected to Core.") return nil } select { case <-deadline.Done(): - fmt.Fprintf(rc.stderr, "Daemon connection: not confirmed. The installed daemon will keep reconnecting. Check %s and Core connectivity, then rerun this command.\n", filepath.Join(o.Directory, "daemon", paths.DefaultProfile, "connect.log")) + fmt.Fprintf(rc.stderr, "Host connection: not confirmed. The installed daemon will keep reconnecting. Check %s and Core connectivity, then rerun this command.\n", filepath.Join(o.Directory, "daemon", paths.DefaultProfile, "connect.log")) return errors.New("install: connection verification timed out") case <-ticker.C: } @@ -206,7 +206,7 @@ func installedEnvironmentConnected(ctx context.Context, base, environment, secre } defer response.Body.Close() if response.StatusCode == 401 || response.StatusCode == 409 { - return false, errors.New("Daemon connection: credential rejected; check the Environment credential in Core") + return false, errors.New("Host connection: credential rejected; check the Environment credential in Core") } if response.StatusCode != http.StatusOK { return false, nil @@ -216,7 +216,7 @@ func installedEnvironmentConnected(ctx context.Context, base, environment, secre Status string `json:"status"` } if json.NewDecoder(io.LimitReader(response.Body, 4096)).Decode(&result) != nil || result.Environment != environment { - return false, errors.New("Daemon connection: invalid status returned by Core") + return false, errors.New("Host connection: invalid status returned by Core") } return result.Status == "connected", nil } diff --git a/apps/daemon/internal/cli/native_start_interrupt_unix_test.go b/apps/daemon/internal/cli/native_start_interrupt_linux_test.go similarity index 75% rename from apps/daemon/internal/cli/native_start_interrupt_unix_test.go rename to apps/daemon/internal/cli/native_start_interrupt_linux_test.go index 316498fda..5c23c53bf 100644 --- a/apps/daemon/internal/cli/native_start_interrupt_unix_test.go +++ b/apps/daemon/internal/cli/native_start_interrupt_linux_test.go @@ -1,11 +1,9 @@ -//go:build unix +//go:build linux package cli import ( "bytes" - "context" - "encoding/json" "errors" "net/http" "net/http/httptest" @@ -21,7 +19,6 @@ func TestNativeStartInterruptHelper(t *testing.T) { if os.Getenv("OAC_TEST_START_INTERRUPT") != "1" { t.Skip("subprocess helper") } - probeNativeInstallation = func(context.Context, string, []string) error { return nil } if err := runStart(&runContext{stdout: os.Stdout, stderr: os.Stderr}, nil); err != nil { os.Exit(2) } @@ -32,19 +29,10 @@ func TestNativeStartInterruptCancelsEnrollment(t *testing.T) { rc, args, root, _ := nativeInstallFixture(t) requested := make(chan struct{}) release := make(chan struct{}) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The enrollment never answers before the interrupt. + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { close(requested) <-release - var config nativeInstallation - if err := readNativeJSON(filepath.Join(root, "daemon", "installation.json"), &config); err != nil { - return - } - _ = json.NewEncoder(w).Encode(environmentEnrollment{ - DeviceID: "aaaaaaaa-1111-4111-8111-aaaaaaaaaaaa", - SessionID: "bbbbbbbb-1111-4111-8111-bbbbbbbbbbbb", - EnvironmentID: config.Environment, - WorkspaceDirectory: config.Workspace, - }) })) defer server.Close() defer close(release) diff --git a/apps/daemon/internal/cli/native_start_linux.go b/apps/daemon/internal/cli/native_start_linux.go new file mode 100644 index 000000000..3dd86d1a8 --- /dev/null +++ b/apps/daemon/internal/cli/native_start_linux.go @@ -0,0 +1,151 @@ +//go:build linux + +package cli + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "syscall" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/daemonize" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimefs" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" +) + +// nativeBundlePrograms are the distribution's programs besides oac-daemon. +var nativeBundlePrograms = []string{"oac-sandbox-io"} + +const sandboxBootstrapFile = "sandbox-io-bootstrap.json" + +// sandboxRestartDelay is the first and the longest wait before an exited +// oac-sandbox-io is replaced. +var sandboxRestartDelay = [2]time.Duration{time.Second, 30 * time.Second} + +// sandboxStopGrace covers oac-sandbox-io's SIGTERM shutdown, at most its cancel +// grace limit plus five seconds (docs/sandbox-bootstrap.md). +const sandboxStopGrace = 40 * time.Second + +// runSandboxLauncher is the Sandbox Provider for this machine's enrollment. It +// enrolls, hands oac-sandbox-io its bootstrap file and replaces the process +// whenever it exits, enrolling again first because a rotation advances the +// generation. Serve reconnects within one process, so this is the only restart +// loop; a native installation has no other supervisor. +func runSandboxLauncher(parent context.Context, rc *runContext, background bool, root string, c nativeInstallation) error { + base, err := environmentBase(c.Remote) + if err != nil { + return err + } + keyID := "" + // The -b parent reports a rejection to its terminal; the process that owns + // the service parks instead. + parks := !background || daemonize.IsBackgroundChild() + rejected := func(err error) error { + if message := environmentRejection(err, keyID, c.Environment); parks && message != "" { + return parkEnvironment(parent, rc.stderr, message) + } + return err + } + // Each enrollment reads the credential file, so a replaced credential + // takes effect without a restart. + enroll := func() (sandboxbootstrap.Input, error) { + id, token, err := executorCredential(c.Credential, c.Environment) + if err != nil { + return sandboxbootstrap.Input{}, err + } + keyID = id + ctx, cancel := context.WithTimeout(parent, bootstrapTimeout) + defer cancel() + return enrollSandbox(ctx, environmentClient(), base, c.Environment, token) + } + input, err := enroll() + if err != nil { + return rejected(err) + } + if err = parent.Err(); err != nil { + return err + } + if background && !daemonize.IsBackgroundChild() { + return spawnBackground(parent, rc, paths.DefaultProfile, os.Args) + } + dir := filepath.Join(root, "daemon") + held, err := os.OpenRoot(dir) + if err != nil { + return err + } + defer held.Close() + program := filepath.Join(root, "bin", "oac-sandbox-io") + delay := sandboxRestartDelay[0] + for { + started := time.Now() + err = runSandboxIO(parent, rc, held, program, filepath.Join(dir, sandboxBootstrapFile), input) + if parent.Err() != nil { + return nil + } + if time.Since(started) >= sandboxRestartDelay[1] { + delay = sandboxRestartDelay[0] + } + fmt.Fprintf(rc.stderr, "oac-daemon: oac-sandbox-io stopped (%v); enrolling again\n", err) + for { + select { + case <-parent.Done(): + return nil + case <-time.After(delay): + } + delay = min(2*delay, sandboxRestartDelay[1]) + if input, err = enroll(); err == nil { + break + } + if environmentRejection(err, keyID, c.Environment) != "" { + return rejected(err) + } + fmt.Fprintf(rc.stderr, "oac-daemon: %v; retrying\n", err) + } + } +} + +// enrollSandbox enrolls this machine and returns the bootstrap input of its +// enrollment resource, which the executor token serves. +func enrollSandbox(ctx context.Context, client *http.Client, base, environment, credential string) (sandboxbootstrap.Input, error) { + raw, err := requestEnrollment(ctx, client, base, environment, credential) + if err != nil { + return sandboxbootstrap.Input{}, err + } + var out struct { + LinkURL string `json:"link_url"` + Resource sandboxbootstrap.Resource `json:"resource"` + } + in := sandboxbootstrap.Input{Version: sandboxbootstrap.Version, Credential: credential} + if decodeEnvironmentJSON(raw, &out) == nil { + in.LinkURL, in.Resource = out.LinkURL, out.Resource + if in.Validate() == nil && in.Resource.Kind == "enrollment" && in.Resource.EnvironmentID == environment { + return in, nil + } + } + return sandboxbootstrap.Input{}, errors.New("connect: invalid Environment enrollment response") +} + +// runSandboxIO writes the bootstrap file and runs oac-sandbox-io until it +// exits. Cancelling ctx sends it SIGTERM and waits for its shutdown. +func runSandboxIO(ctx context.Context, rc *runContext, held *os.Root, program, bootstrap string, input sandboxbootstrap.Input) error { + raw, err := input.Marshal() + if err != nil { + return err + } + if err = runtimefs.WritePrivateAtomic(held, sandboxBootstrapFile, raw); err != nil { + return err + } + cmd := exec.CommandContext(ctx, program, "--bootstrap-file", bootstrap) + cmd.Stdout, cmd.Stderr = rc.stderr, rc.stderr + // The service must not keep serving this machine after its launcher dies. + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGTERM} + cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) } + cmd.WaitDelay = sandboxStopGrace + return cmd.Run() +} diff --git a/apps/daemon/internal/cli/native_start_linux_test.go b/apps/daemon/internal/cli/native_start_linux_test.go new file mode 100644 index 000000000..fe2cd67e8 --- /dev/null +++ b/apps/daemon/internal/cli/native_start_linux_test.go @@ -0,0 +1,94 @@ +//go:build linux + +package cli + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" + "github.com/google/uuid" +) + +// The launcher hands oac-sandbox-io a private bootstrap file for the enrolled +// resource and, after the process exits, enrolls again and starts it with the +// new generation. +func TestNativeStartRestartsSandboxIOAfterEnrolling(t *testing.T) { + rc, args, root, _ := nativeInstallFixture(t) + tenant, resource := uuid.NewString(), uuid.NewString() + var enrolls atomic.Uint64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]string + if r.URL.Path != "/api/v1/agent-daemon/enroll" || r.Header.Get("Authorization") != "Bearer private-test-credential" || json.NewDecoder(r.Body).Decode(&body) != nil { + w.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"link_url": "wss://core.example/api/v1/sandbox-link", "resource": map[string]any{ + "tenant_id": tenant, "environment_id": body["environment_id"], "kind": "enrollment", "id": resource, "generation": enrolls.Add(1)}}) + })) + defer server.Close() + for i := range args { + if args[i] == "--remote" { + args[i+1] = "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/agent-daemon/ws" + } + } + if err := runInstall(rc, args); err != nil { + t.Fatal(err) + } + var config nativeInstallation + if err := readNativeJSON(filepath.Join(root, "daemon", "installation.json"), &config); err != nil { + t.Fatal(err) + } + // The fake service records its bootstrap file and mode, then exits. + runs := filepath.Join(t.TempDir(), "runs") + script := "#!/bin/sh\n[ \"$1\" = --bootstrap-file ] || exit 9\necho \"$(stat -c %a \"$2\") $(cat \"$2\")\" >> '" + runs + "'\nexit 3\n" + if err := os.WriteFile(filepath.Join(root, "bin", "oac-sandbox-io"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + previous := sandboxRestartDelay + sandboxRestartDelay = [2]time.Duration{time.Millisecond, 2 * time.Millisecond} + defer func() { sandboxRestartDelay = previous }() + output := new(lockedBuffer) + rc.stdout, rc.stderr = output, output + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- runSandboxLauncher(ctx, rc, false, root, config) }() + deadline := time.Now().Add(10 * time.Second) + for { + if raw, _ := os.ReadFile(runs); strings.Count(string(raw), "\n") >= 2 { + break + } + if time.Now().After(deadline) { + t.Fatalf("oac-sandbox-io was not restarted: %s", output.String()) + } + time.Sleep(5 * time.Millisecond) + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(runs) + for i, line := range strings.Split(string(raw), "\n")[:2] { + mode, document, _ := strings.Cut(line, " ") + in, err := sandboxbootstrap.Decode([]byte(document)) + if err != nil || mode != "600" { + t.Fatalf("run %d: mode %s: %v", i, mode, err) + } + want := sandboxbootstrap.Resource{TenantID: tenant, EnvironmentID: config.Environment, Kind: "enrollment", ID: resource, Generation: uint64(i + 1)} + if in.LinkURL != "wss://core.example/api/v1/sandbox-link" || in.Credential != "private-test-credential" || in.Resource != want { + t.Fatalf("run %d: unexpected bootstrap input %+v", i, in.Resource) + } + } + if strings.Contains(output.String(), "private-test-credential") { + t.Fatal("credential leaked") + } +} diff --git a/apps/daemon/internal/cli/native_start_other.go b/apps/daemon/internal/cli/native_start_other.go new file mode 100644 index 000000000..5732cc434 --- /dev/null +++ b/apps/daemon/internal/cli/native_start_other.go @@ -0,0 +1,19 @@ +//go:build !linux + +package cli + +import ( + "context" + "errors" +) + +// nativeBundlePrograms are the distribution's programs besides oac-daemon. +var nativeBundlePrograms []string + +// errSandboxPlatform rejects start off Linux: oac-sandbox-io, which serves a +// self-hosted machine, runs only on Linux. +var errSandboxPlatform = errors.New("start: self-hosted Environments run only on Linux") + +func runSandboxLauncher(context.Context, *runContext, bool, string, nativeInstallation) error { + return errSandboxPlatform +} diff --git a/apps/web/e2e/diagnostics.spec.ts b/apps/web/e2e/diagnostics.spec.ts index 962380260..5ce24a761 100644 --- a/apps/web/e2e/diagnostics.spec.ts +++ b/apps/web/e2e/diagnostics.spec.ts @@ -68,7 +68,7 @@ test("connection reads govern host completion and bound-key guidance", async ({ if (unavailable) return route.fulfill({ status: 503, json: { error: { code: "internal_error", message: "Unreadable connection", type: "server_error", param: null } } }); await route.fulfill({ json: { data: [{ key_id: keyId, created_at: "2026-09-28T08:00:00Z", revoked_at: revoked ? "2026-09-28T09:01:00Z" : null }], - connection: { status: connected ? "connected" : "disconnected", bound_key_id: keyId, enrolled_at: "2026-09-28T08:00:01Z", last_seen_at: "2026-09-28T08:59:00Z" }, + connection: { status: connected ? "connected" : "disconnected", bound_key_id: keyId, enrolled_at: "2026-09-28T08:00:01Z" }, } }); }); await openConsole(page, request, "sessions"); diff --git a/apps/web/e2e/fixture-console.mjs b/apps/web/e2e/fixture-console.mjs index aeff30833..108f1da9e 100644 --- a/apps/web/e2e/fixture-console.mjs +++ b/apps/web/e2e/fixture-console.mjs @@ -476,7 +476,7 @@ async function executorCredentialRoute(request, response, projectId, environment const credentials = state.executorCredentials.get(environmentId); if (!keyId && request.method === "GET") return send(response, 200, { data: credentials.map((entry) => ({ ...entry })), - connection: { status: "never_enrolled", bound_key_id: null, enrolled_at: null, last_seen_at: null }, + connection: { status: "never_enrolled", bound_key_id: null, enrolled_at: null }, }); if (!keyId && request.method === "POST") { // As Core, a body that is not JSON is invalid input like any other: 400 with one message. diff --git a/apps/web/src/features/sessions/ExecutorConnectionPanel.tsx b/apps/web/src/features/sessions/ExecutorConnectionPanel.tsx index f676b207b..2ee427aa0 100644 --- a/apps/web/src/features/sessions/ExecutorConnectionPanel.tsx +++ b/apps/web/src/features/sessions/ExecutorConnectionPanel.tsx @@ -3,7 +3,6 @@ import { useId } from "react"; import { useTranslation } from "react-i18next"; import { HelpTip, RefreshButton, StatusDot } from "../../components/console-ui"; -import { formatDateTime } from "../../lib/format"; import { executorConnectionState } from "./executor-connection"; import "./executor-connection.css"; @@ -14,20 +13,16 @@ export function ExecutorConnectionPanel({ read, stale, failed, refreshing, onRef refreshing: boolean; onRefresh: () => void; }) { - const { t, i18n } = useTranslation("sessions"); + const { t } = useTranslation("sessions"); const { t: tCommon } = useTranslation("common"); const headingId = useId(); const state = executorConnectionState(read, stale); - const connection = read?.connection; return (

{t("executor.connection.title")}

- - {t("executor.connection.help")}
- {t("executor.connection.lastSeen")}: {connection?.last_seen_at ? formatDateTime(Date.parse(connection.last_seen_at) / 1000, i18n.resolvedLanguage) : t(connection ? "executor.connection.noHeartbeat" : "executor.connection.status.unknown")} - + {t("executor.connection.help")}
{stale && read ?

{t(failed ? "executor.connection.stale" : "executor.connection.refreshing")}

: null} diff --git a/apps/web/src/features/sessions/executor-connection.test.ts b/apps/web/src/features/sessions/executor-connection.test.ts index 3a9bd0e1c..2eed78d98 100644 --- a/apps/web/src/features/sessions/executor-connection.test.ts +++ b/apps/web/src/features/sessions/executor-connection.test.ts @@ -10,19 +10,17 @@ vi.mock("../../lib/projects", () => ({ admin: { listExecutorCredentials: list } const active = { key_id: "bound-key", created_at: "2026-09-28T01:00:00Z", revoked_at: null }; function read(status: ExecutorCredentialList["connection"]["status"] = "disconnected"): ExecutorCredentialList { - return { data: [active], connection: { status, bound_key_id: status === "never_enrolled" ? null : "BOUND-KEY", enrolled_at: status === "never_enrolled" ? null : "2026-09-28T01:00:00Z", last_seen_at: null } }; + return { data: [active], connection: { status, bound_key_id: status === "never_enrolled" ? null : "BOUND-KEY", enrolled_at: status === "never_enrolled" ? null : "2026-09-28T01:00:00Z" } }; } afterEach(() => vi.clearAllMocks()); describe("executor connection evidence", () => { - it("never treats an active credential or a recent heartbeat as a connection", () => { + it("never treats an active credential or an enrollment as a connection", () => { expect(executorConnectionState(read("never_enrolled"), false)).toBe("never_enrolled"); - const disconnected = read(); - disconnected.connection.last_seen_at = new Date().toISOString(); - expect(executorConnectionState(disconnected, false)).toBe("disconnected"); + expect(executorConnectionState(read(), false)).toBe("disconnected"); }); - it("accepts Core connectivity even without a recorded heartbeat", () => { + it("accepts Core connectivity", () => { expect(executorConnectionState(read("connected"), false)).toBe("connected"); }); it("distinguishes bound-key revocation from unrelated revoked keys", () => { diff --git a/apps/web/src/features/sessions/executor-credential-ownership.test.tsx b/apps/web/src/features/sessions/executor-credential-ownership.test.tsx index e10fa04cc..3a28975f7 100644 --- a/apps/web/src/features/sessions/executor-credential-ownership.test.tsx +++ b/apps/web/src/features/sessions/executor-credential-ownership.test.tsx @@ -24,7 +24,7 @@ function read(revoked = false): ExecutorCredentialList { { key_id: boundId, created_at: "2026-09-28T01:00:00Z", revoked_at: revoked ? "2026-09-28T02:00:00Z" : null }, { key_id: newId, created_at: "2026-09-28T03:00:00Z", revoked_at: null }, ], - connection: { status: "disconnected", bound_key_id: boundId, enrolled_at: "2026-09-28T01:00:00Z", last_seen_at: "2026-09-28T01:30:00Z" }, + connection: { status: "disconnected", bound_key_id: boundId, enrolled_at: "2026-09-28T01:00:00Z" }, }; } function render(value: ExecutorCredentialList | null, failed = false) { diff --git a/apps/web/src/i18n/locales/en/sessions.ts b/apps/web/src/i18n/locales/en/sessions.ts index fdf8078f3..c7f715adf 100644 --- a/apps/web/src/i18n/locales/en/sessions.ts +++ b/apps/web/src/i18n/locales/en/sessions.ts @@ -150,12 +150,10 @@ export const sessions = { executor: { connection: { title: "Host connection", - help: "Core reports a connection only while the executor has current authority and a live connection. Last seen is the last recorded heartbeat and may lag. This does not confirm model or harness readiness.", + help: "Core reports a connection only while the bound credential has current authority and the host is serving this Environment.", status: { never_enrolled: "Never connected", connected: "Connected", disconnected: "Disconnected", revoked: "Bound credential revoked", unknown: "Unknown" }, boundKey: "Bound credential", - lastSeen: "Last seen", notBound: "Not bound", - noHeartbeat: "No heartbeat recorded", stale: "Refresh failed. Showing the last loaded binding; the current connection is unknown.", refreshing: "Refreshing the last loaded binding; the current connection is not yet confirmed.", failed: "The connection could not be read. Refresh to try again.", diff --git a/apps/web/src/i18n/locales/zh-CN/sessions.ts b/apps/web/src/i18n/locales/zh-CN/sessions.ts index 82569748e..cef6b77d0 100644 --- a/apps/web/src/i18n/locales/zh-CN/sessions.ts +++ b/apps/web/src/i18n/locales/zh-CN/sessions.ts @@ -147,12 +147,10 @@ export const sessions = { executor: { connection: { title: "主机连接", - help: "只有执行器仍具备有效权限且连接存活时，Core 才报告已连接。最后在线时间来自最近一次心跳，可能有延迟；它不代表模型或运行引擎已就绪。", + help: "只有绑定凭证仍具备有效权限、且主机正在为此 Environment 提供服务时，Core 才报告已连接。", status: { never_enrolled: "尚未连接", connected: "已连接", disconnected: "已断开", revoked: "绑定凭证已撤销", unknown: "未知" }, boundKey: "绑定凭证", - lastSeen: "最后在线", notBound: "尚未绑定", - noHeartbeat: "尚无心跳记录", stale: "刷新失败。当前显示上次读取的绑定信息，连接状态未知。", refreshing: "正在刷新上次读取的绑定信息，尚未确认当前连接。", failed: "无法读取连接状态，请刷新重试。", diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index a764c0d86..1eec67515 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -342,10 +342,6 @@ definitions: format: date-time type: string x-nullable: true - last_seen_at: - format: date-time - type: string - x-nullable: true status: enum: - never_enrolled @@ -355,7 +351,6 @@ definitions: required: - bound_key_id - enrolled_at - - last_seen_at - status type: object api.ExecutorCredentialList: @@ -4112,7 +4107,7 @@ paths: - Environment Templates /core/v1/projects/{project_id}/environments/{environment_id}/executor-credentials: get: - description: Core key only. Returns metadata of the credentials restricted to this Environment, oldest first; secrets are never listed. Connection combines current credential authority and an open matching gateway peer; timestamps are historical observations, not readiness. Without a gateway it is never connected. The Environment must be a self_hosted Environment of the Project whose Session exists; otherwise 404. + description: Core key only. Returns metadata of the credentials restricted to this Environment, oldest first; secrets are never listed. Connection is connected while the enrolled credential has authority and the sandbox it enrolled serves the Environment's Link resource; enrolled_at is when the Environment was first enrolled, not readiness. The Environment must be a self_hosted Environment of the Project whose Session exists; otherwise 404. parameters: - description: Project UUID in: path diff --git a/contracts/agents-api/environment-executor-credentials.md b/contracts/agents-api/environment-executor-credentials.md index 46631231b..2bdf90a9b 100644 --- a/contracts/agents-api/environment-executor-credentials.md +++ b/contracts/agents-api/environment-executor-credentials.md @@ -2,7 +2,7 @@ title: "Environment executor credentials" --- -An executor credential lets `oac-daemon` enroll and connect for one `self_hosted` Environment. It authorizes only the private daemon transport (`/api/v1/agent-daemon/*`) for that Environment, never `/v1`, `/core/v1`, sandbox-node enrollment or Project resources. The Project's principal is its execution principal. Core stores only a digest of the secret. +An executor credential lets `oac-daemon` enroll one `self_hosted` Environment and serve it over the [sandbox Link](../../docs/sandbox-link-protocol.md). It authorizes only the private daemon routes (`/api/v1/agent-daemon/*`) for that Environment and, once enrolled, the Serve of the Environment's enrollment resource on the Link, never `/v1`, `/core/v1`, sandbox-node enrollment or Project resources. The Project's principal is its execution principal. Core stores only a digest of the secret. A credential comes from one of two places: @@ -61,7 +61,7 @@ Writes have two conflicts, both 409. `executor_credential_exists`: an issuance w An issuance or rotation is checked in this order, and the first failure is returned: the request body (400); the target Environment (404); an archived Project (409 `project_archived`); then the key itself (409 `executor_credential_exists` without `rotate`, or 404 when rotating a `key_id` that was never issued). -Rotation replaces the secret of an existing key restricted to this Environment, keeps that Environment, invalidates the previous secret at once and restores a revoked key. Revocation is idempotent and returns 204 each time. It denies further enrollment and connection. +Rotation replaces the secret of an existing key restricted to this Environment, keeps that Environment and `key_id`, invalidates the previous secret at once and restores a revoked key. It also advances the generation of the Environment's enrollment and the epoch of its Session's assignment, so the machine serves again only after it enrolls with the new secret, and a running Turn loses the Environment ([Session assignments](../../docs/runtime-protocol.md#session-assignments)). Revocation is idempotent and returns 204 each time. It denies further enrollment and closes the machine's Serve. After an uncertain result, such as a timeout, do not retry automatically. List the credentials, then either rotate the same `key_id` (it was issued but its secret was lost) or issue it again (it was not issued). @@ -73,26 +73,22 @@ Issue, rotate and revoke each record an administrator audit entry (`resource_typ ## Connection status -The list's required `connection` object contains `status` (`never_enrolled`, `connected` or `disconnected`), `bound_key_id`, `enrolled_at` and `last_seen_at`. All three binding fields are null before enrollment. Once enrolled, the bound key and enrollment time describe the existing device; a null `last_seen_at` means no authenticated heartbeat has been recorded. Issuing another key does not change the binding. Rotation or revocation can make the binding disconnected while its history remains visible. Expired Environments remain readable under the existing list rules but cannot have current executor authority. +The list's required `connection` object contains `status` (`never_enrolled`, `connected` or `disconnected`), `bound_key_id` and `enrolled_at`. Both binding fields are null before enrollment. Once enrolled, `bound_key_id` is the key that enrolled the Environment and `enrolled_at` is when it first enrolled; neither is readiness. Issuing another key does not change the binding. Rotation or revocation can make the binding disconnected while its history remains visible. Expired Environments remain readable under the existing list rules but cannot have current executor authority. -Connected means the Environment is connected, its device and executor key still have current Core authority, and the process-local gateway has an open peer that authenticated with that current key. Core rechecks authority after observing the peer. A former key's live socket, a device timestamp or a ready-looking Environment alone is insufficient; without a gateway, Core never returns connected. These facts are an observation, not a reservation of connectivity or of native or model readiness. `last_seen_at` may lag by a heartbeat interval. +Connected means the enrolled key still has current Core authority and the Environment's live Link resource is that key's enrollment, Serving at its current generation: the relay holds the serve peer of the machine's Sandbox I/O service. Core rechecks authority after observing the serve peer, so a peer that still serves with a former secret is not connected. This is the same rule as for a hosted Environment ([readiness facts](../../docs/sandbox-provider.md#four-distinct-readiness-facts)). It is an observation, not a reservation of connectivity or of native or model readiness. -List metadata and binding facts use one read-only database snapshot. That snapshot ends before the live authority checks, so a committed rotation or revocation is not hidden by snapshot isolation. Known authority loss projects as disconnected; observation and storage failures remain errors. Device IDs and credential digests are internal and never serialized. The public `/v1` Environment shape is unchanged. +List metadata and binding facts use one read-only database snapshot. That snapshot ends before the live authority checks, so a committed rotation or revocation is not hidden by snapshot isolation. Known authority loss projects as disconnected; observation and storage failures remain errors. Enrollment IDs and credential digests are internal and never serialized. The public `/v1` Environment shape is unchanged. ### Private connection confirmation -`GET /api/v1/agent-daemon/connection?environment_id=UUID` uses the executor bearer, sent directly to Core (the reverse proxy routes `/api/v1` to Core; the console does not serve it). It is part of the private daemon transport, not the public Agents API. It reads existing authorization and binding only; it never enrolls a device, starts execution or changes resources. The no-store response contains only the requested `environment_id` and `status` (`connected` or `disconnected`). Connected requires the Environment observation, its exact Session and device binding, current executor authority and a live gateway socket authenticated with that same credential. A stale observation or a socket carrying a rotated key cannot confirm connection. +`GET /api/v1/agent-daemon/connection?environment_id=UUID` uses the executor bearer, sent directly to Core (the reverse proxy routes `/api/v1` to Core; the console does not serve it). It is part of the private daemon transport, not the public Agents API. It reads existing authorization and enrollment only; it never enrolls, starts execution or changes resources. The no-store response contains only the requested `environment_id` and `status` (`connected` or `disconnected`). `connected` follows the [connection status](#connection-status) rule for the presented credential: the Environment's live Link resource is the enrollment of that same credential and is Serving. A stale observation or a serve peer of a rotated secret cannot confirm connection. -Invalid, revoked, foreign or deleted-Session authority returns 401; a different key for an already bound Environment returns 409. Responses do not expose the actual binding or database diagnostics. +Invalid, revoked, foreign or deleted-Session authority, or a failed or expired Environment, returns 401; a different key for an already enrolled Environment returns 409. Responses do not expose the actual binding or database diagnostics. -The installer derives this route from the returned `remote_url` and does not follow redirects. After starting the daemon it polls once a second for up to 45 seconds. A 401 or 409 fails at once. On timeout it prints the path of the daemon's `connect.log` and asks you to rerun the same command; the daemon keeps reconnecting, and the installation, credential and history stay in place. A confirmed connection proves authentication only, not model access, Harness capability or completed execution. +The installer derives this route from the returned `remote_url` and does not follow redirects. After starting the daemon it polls once a second for up to 45 seconds. A 401 or 409 fails at once. On timeout it prints the path of the daemon's `connect.log` and asks you to rerun the same command; the daemon keeps reconnecting, and the installation, credential and history stay in place. A confirmed connection proves that the machine serves the Environment, not model access, Harness capability or completed execution. ## Revoked or rotated credential -When Core permanently rejects the daemon (enrollment 401 or 409, a permanent WebSocket rejection, or a daemon version from another Core distribution), the daemon prints the reason once and makes no further requests until it is stopped; it then exits successfully, so a supervisor that restarts on exit does not loop. When started again, it tries enrollment once and parks again. Transient failures keep the normal reconnect behavior and never replay execution. +When Core permanently rejects the machine's enrollment (401 or 409), the daemon prints the reason once and makes no further requests until it is stopped; it then exits successfully, so a supervisor that restarts on exit does not loop. When started again, it tries enrollment once and parks again. Transient failures, including an exited Sandbox I/O service, are retried with backoff and never replay execution. -A machine reconnects only with its bound `key_id`, rotated to a new secret that replaces the credential file at its configured path; the [self-hosted guide](../../docs/getting-started/self-hosted.md#rotate-or-revoke) gives the steps. A new `key_id` cannot reconnect an Environment that is already bound: issuing it succeeds, but enrollment with it returns 409. Rotation does not reinstall Harnesses, change the workspace or replace native history; never create a new Session history to recover a credential. - -## Model provider - -[Model execution](./model-execution.md) owns provider resolution and delivery. A saved Agent's provider key is delivered to the executor of every `self_hosted` Session created with that Agent in the Project, and a deployment default's key to the executor of every `self_hosted` Session that falls back to it, so anyone who can create such a Session and run an executor can read that key. +A machine reconnects only with its bound `key_id`, rotated to a new secret that replaces the credential file at its configured path; the [self-hosted guide](../../docs/getting-started/self-hosted.md#rotate-or-revoke) gives the steps. A new `key_id` cannot reconnect an Environment that is already bound: issuing it succeeds, but enrollment with it returns 409. Rotation does not change the workspace or the Session's native history; never create a new Session to recover a credential. diff --git a/contracts/agents-api/environments.md b/contracts/agents-api/environments.md index bf60e712d..a1fcd2752 100644 --- a/contracts/agents-api/environments.md +++ b/contracts/agents-api/environments.md @@ -51,7 +51,7 @@ Both placements run the same Runtime: the daemon, the selected Harness, native t | Provider allocation | Compute and filesystem lifetime: Core's Sandbox Provider for `openai_hosted`, the application for `self_hosted` | | Device and daemon connection | Authenticated Runtime identity and the replaceable dispatch transport | | Harness process and native session | The native model and tool loop, its execution state and native history | -| Enrollment | The exact Environment, device and executor-key binding of a `self_hosted` machine | +| Enrollment | The executor key with which a `self_hosted` machine serves the Environment's [Link](../../docs/sandbox-link-protocol.md) resource | ### Hosted (`openai_hosted`) @@ -69,18 +69,18 @@ The deployment's configured Sandbox Provider (E2B, Docker or microsandbox, see [ The application owns the machine. It creates the Session with a clean absolute `workspace_directory` and optional absolute local `capability_directories`. Core returns the Environment ID, the `remote_url` and an install command in `x_agents_core.installation`; running that command on the machine installs the daemon and enrolls it ([self-hosted guide](../../docs/getting-started/self-hosted.md), [executor credentials](./environment-executor-credentials.md)). - `remote_url` is the daemon WebSocket URL derived from Core's public URL, never from request headers or a daemon address. It names Core's private daemon transport. -- Enrollment binds the exact Session, Environment, device and executor key. It creates no allocation and cannot move a Session to another device. +- Enrollment records the executor key that serves the Environment's Link resource; the first key keeps it. It creates no allocation and binds no Session: the Session runs on the deployment's agent host ([Session assignments](../../docs/runtime-protocol.md#session-assignments)). - The Session's workspace must equal the `/workspace` alias or the exact canonical directory the Runtime is bound to. Naming a path grants no access to it. - Session reads, lists and events return the `self_hosted` output with the Environment ID, workspace and capability directories, never private configuration. `capability_directories` lists the caller's selections; the Runtime's installation locations stay private. - Compute, workspace and files stay the application's. Deleting the Session or revoking the credential denies further access but does not stop native processes; the machine owner stops and cleans up. -- The workspace and native history must survive a daemon restart. Losing them never authorizes silent replacement or replay. +- The workspace must survive a daemon restart. Losing it never authorizes silent replacement or replay. **Application-managed E2B.** An application can run the Runtime in an E2B sandbox it creates, renews and destroys with the E2B SDK, then enroll that Runtime as a `self_hosted` Environment ([E2B Runtime guide](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/e2b/README.md)). Core keeps no E2B allocation for it and never renews or kills it. ### Ownership rules - Keep Environment identity, ownership, configuration and lifecycle in Core, separate from Provider compute, device identity, daemon sockets and native sessions. Keep mutable connection state out of immutable configuration; a replacement owner fences stale observations. -- Callers, devices and Environment connections use distinct credentials. The daemon gateway authenticates the enrolled executor key and the exact device. Connection observations keep generation and revision fencing. Registration and connection do not establish readiness. +- Callers, devices and Environment connections use distinct credentials. The relay authenticates a `self_hosted` machine's Serve with the enrolled executor key. Connection observations keep generation and revision fencing. Registration and connection do not establish readiness. - Rotation, revocation, Session deletion and loss of ownership deny further access; they do not promise that native effects stop at once. - Native history stays on the bound Runtime. Preserve it, or demonstrably restore it, across compute replacement; never silently move a bound Session or replay unknown work. - Durable metadata reads need no live Runtime. Live file reads need an authorized view of the exact workspace and bounded operation ownership. Operations that write, replace or retire a workspace owner also apply mutation fencing. diff --git a/contracts/agents-api/machine-api.md b/contracts/agents-api/machine-api.md index f6dfa22ff..65ef4200c 100644 --- a/contracts/agents-api/machine-api.md +++ b/contracts/agents-api/machine-api.md @@ -32,7 +32,7 @@ The generated [`runtime.openapi.yaml`](./runtime.openapi.yaml) describes only th | Enrollment token | `POST /core/v1/sandbox/enrollment-tokens` (Web **Add node**), with the node's approved capacity. One use; it expires at the response's `expires_at` | `sandbox-node/configuration` without a node ID, `sandbox-node/enroll` | | Node credential | The node itself: it generates a secret of 32 to 256 characters without whitespace and registers it at enrollment | `sandbox-node/configuration` with `X-OAC-Node-ID`, `sandbox-node/identity`, `sandbox-node/connect` | | Installation grant | The `x_agents_core.installation` command of a `self_hosted` Session; short-lived | `agent-daemon/installation` and its `claim` | -| Executor credential | The installation claim, or the Core-key [executor credential routes](./environment-executor-credentials.md) | `agent-daemon/enroll` and `agent-daemon/connection`; after enrollment it is also the daemon credential of the bound device | +| Executor credential | The installation claim, or the Core-key [executor credential routes](./environment-executor-credentials.md) | `agent-daemon/enroll` and `agent-daemon/connection`; after enrollment it is also the Serve credential of the Environment's enrollment on `sandbox-link` | | Daemon credential of a hosted sandbox | Core, for each managed allocation, delivered in the [bootstrap file](../../docs/runtime-bootstrap.md) | `agent-daemon/bootstrap`, `device-status` and `ws` | | Operator device profile | `oac-core-device`, run by an operator with database access | `agent-daemon/bootstrap`, `device-status` and `ws` | @@ -40,7 +40,7 @@ Core keeps only a SHA-256 digest of each token and credential it stores; install ### Operator device profile -An engine host for `environment: none` Sessions connects with a device profile that an operator provisions directly in the database: +`oac-core-device` provisions a Runtime device profile directly in the database: ```sh umask 077 @@ -49,7 +49,7 @@ OAC_DATABASE_URL=... oac-core-device --tenant --name 'engine host' oac-daemon connect --profile default ``` -`--tenant` is the Project's execution tenant UUID and `--url` Core's origin without a path. The command prints the profile once: `server_url` (the origin plus `/api/v1`), `runtime_id` (the device ID), `runner_credential` and `device_name`. Use a new profile rather than overwriting another device's file, and copy it privately to the same path on a remote host. `oac-core-device --tenant --revoke ` revokes the device: new connections are refused at once, and an open connection closes at its next heartbeat. The Worker binds each `none` Session to one connected device of its tenant that declares the required capabilities and keeps that binding across retries and restarts; self-hosted Sessions never use this path. +`--tenant` is the Project's execution tenant UUID and `--url` Core's origin without a path. The command prints the profile once: `server_url` (the origin plus `/api/v1`), `runtime_id` (the device ID), `runner_credential` and `device_name`. Use a new profile rather than overwriting another device's file, and copy it privately to the same path on a remote host. `oac-core-device --tenant --revoke ` revokes the device: new connections are refused at once, and an open connection closes at its next heartbeat. Core binds Sessions only to the deployment's agent host ([Session assignments](../../docs/runtime-protocol.md#session-assignments)), so a device of this profile runs no Session. ## Node routes @@ -111,13 +111,13 @@ The bootstrap, device-status and WebSocket routes share one error body, `{"error ### Enroll a self-hosted daemon -`POST /api/v1/agent-daemon/enroll` with the executor credential and exactly `{"environment_id": "…"}` (no query) binds one dedicated device to the Environment's Session and returns `device_id`, `session_id`, `environment_id` and `workspace_directory`. It never returns another credential: the executor credential becomes the daemon credential of that device. A retry with the same credential returns the same binding. A successful response carries `Cache-Control: no-store`. +`POST /api/v1/agent-daemon/enroll` with the executor credential and exactly `{"environment_id": "…"}` (no query) enrolls the machine as the Environment's [Link](../../docs/sandbox-link-protocol.md) resource and returns the [launch input](../../docs/sandbox-bootstrap.md#launch-input) fields Core owns: `link_url` and `resource` (`tenant_id`, `environment_id`, `kind` `enrollment`, `id` and `generation`). It never returns another credential: the executor credential is the resource's Serve credential. The first key to enroll the Environment keeps it, and a retry with that key returns the same resource at its current generation. A successful response carries `Cache-Control: no-store`. | HTTP | When | | --- | --- | | 400 | A malformed body or any query | | 401 | An invalid, revoked or foreign credential, a deleted Session, or an Environment without current executor authority | -| 409 | The Environment is already bound to a different key or device | -| 503 | Storage is unavailable | +| 409 | Another executor key already enrolled the Environment | +| 503 | `{"error": "no_sandbox_link", "detail": "a self_hosted sandbox needs an https public URL"}`, before the credential is checked, when the [public URL](../../docs/configuration.md#changing-the-public-url) is not https; otherwise storage is unavailable | -Enrollment creates no managed allocation and grants no Session API access. The daemon keeps the binding beside its credential and refuses another Environment's native history. The gateway and Worker recheck the credential's authority on every connection and dispatch, so rotation, revocation and Session deletion end further use. The [self-hosted guide](../../docs/getting-started/self-hosted.md) gives the operator steps, and the [executor credential contract](./environment-executor-credentials.md#revoked-or-rotated-credential) describes how the daemon handles a permanent rejection. +Enrollment creates no managed allocation, binds no Session and grants no Session API access. Core binds the Session to the deployment's agent host while the machine serves the resource ([Session assignments](../../docs/runtime-protocol.md#session-assignments)). The relay rechecks the credential's authority on every Serve and Open, so rotation, revocation and Session deletion end further use. The [self-hosted guide](../../docs/getting-started/self-hosted.md) gives the operator steps, and the [executor credential contract](./environment-executor-credentials.md#revoked-or-rotated-credential) describes how the daemon handles a permanent rejection. diff --git a/contracts/agents-api/model-execution.md b/contracts/agents-api/model-execution.md index 206b61d5f..e34e3bc4a 100644 --- a/contracts/agents-api/model-execution.md +++ b/contracts/agents-api/model-execution.md @@ -36,7 +36,7 @@ Provider precedence is: a complete Session bundle, then a complete saved bundle, MiniMax Code requires positive context and output limits. Core validates the resolved combination before writing the Session. Core and Runtime read the same ordered `protocols` declaration in `internal/harnessconfig`. Nothing converts between protocols, including inside a Harness. Unsupported saved configurations and Session snapshots fail when used; they are never rewritten, aliased or migrated. -Every source applies to every Environment type, because the key reaches only the [credential gateway](#credential-gateway) on the agent host, never the Harness or the sandbox. For `self_hosted`, that agent host is the application's executor, so a deployment default there hands the operator's key to that executor's gateway. Every Session must resolve a bundle: without one, creation is rejected before any write with 400 `model_provider_required`, param `x_agents_core.model_provider` and a message saying what to configure. A saved Agent may omit its bundle and leave it to the Session or the deployment default. +Every source applies to every Environment type, because the key reaches only the [credential gateway](#credential-gateway) on the deployment's agent host, never the Harness, the sandbox or a `self_hosted` machine. Every Session must resolve a bundle: without one, creation is rejected before any write with 400 `model_provider_required`, param `x_agents_core.model_provider` and a message saying what to configure. A saved Agent may omit its bundle and leave it to the Session or the deployment default. | Operation | Omitted | Explicit null | | --- | --- | --- | @@ -82,7 +82,7 @@ Unsupported protocol, Harness or Environment combinations are rejected before th The resolved provider configuration is frozen and encrypted in the Session creation transaction, with its own encryption purpose and Project and Session binding. Creation retries include a Session's own bundle in their request hash, so a changed key or endpoint under the same Idempotency-Key conflicts; a key enters any stored hash only as a fingerprint keyed by the deployment credential key. No public Session, Agent, Environment, event or ordinary configuration contains the key. The top-level extension is write-only and cannot be updated. -At dispatch, Core sends the snapshot as one confidential provider bundle over the daemon connection bound to the Session, and the adapter points the Harness at the [credential gateway](#credential-gateway) through native configuration. Core never falls back to other credentials when a snapshot is missing or cannot be decrypted. For `self_hosted`, the receiving daemon is the executor enrolled for the Session's own Environment with a current executor credential of the Session creator's principal; rotation or revocation closes the socket before further dispatch. The gateway holds the key in memory for the Session, and the key never enters the Harness's environment, configuration or home, or the sandbox. +At dispatch, Core sends the snapshot as one confidential provider bundle over the daemon connection bound to the Session, and the adapter points the Harness at the [credential gateway](#credential-gateway) through native configuration. Core never falls back to other credentials when a snapshot is missing or cannot be decrypted. The gateway holds the key in memory for the Session, and the key never enters the Harness's environment, configuration or home, or the sandbox. ## Credential gateway diff --git a/contracts/agents-api/runtime-observability.md b/contracts/agents-api/runtime-observability.md index c91351dac..2d9fff66a 100644 --- a/contracts/agents-api/runtime-observability.md +++ b/contracts/agents-api/runtime-observability.md @@ -12,7 +12,7 @@ Core attributes every observation to durable Core identity before it reads a pro ```text managed: tenant_id -> session_id -> environment_id -> runtime_allocation_id -self-hosted: tenant_id -> session_id -> environment_id -> device_id + connection_generation +self-hosted: tenant_id -> session_id -> environment_id none: tenant_id -> session_id (no Session-owned Runtime instance) ``` diff --git a/contracts/agents-api/zh/environment-executor-credentials.md b/contracts/agents-api/zh/environment-executor-credentials.md index 1516c7468..fe5666e12 100644 --- a/contracts/agents-api/zh/environment-executor-credentials.md +++ b/contracts/agents-api/zh/environment-executor-credentials.md @@ -1,10 +1,10 @@ --- title: "Environment 执行器凭证" source: contracts/agents-api/environment-executor-credentials.md -source_hash: 8f87d545cb1b9b944b29e0c2a2c1b5bcd160280771b3ec39a196da96dad93766 +source_hash: b1e07476ee5307bb57f58a94547a9ebde1cf377aeb456117c150632108435f7e --- -执行器凭证允许 `oac-daemon` 为一个 `self_hosted` Environment 注册并连接。它只授权该 Environment 的私有 daemon 传输（`/api/v1/agent-daemon/*`），不授权 `/v1`、`/core/v1`、sandbox node 注册或 Project 资源。Project 的 principal 是其执行 principal。Core 只保存密钥摘要。 +执行器凭证允许 `oac-daemon` 为一个 `self_hosted` Environment 注册，并通过 [sandbox Link](../../../docs/zh/sandbox-link-protocol.md) 为它提供服务。它只授权该 Environment 的私有 daemon 路由（`/api/v1/agent-daemon/*`），以及注册后在 Link 上 Serve 该 Environment 的 enrollment resource，不授权 `/v1`、`/core/v1`、sandbox node 注册或 Project 资源。Project 的 principal 是其执行 principal。Core 只保存密钥摘要。 凭证有两个来源： @@ -63,7 +63,7 @@ grant 绑定 Environment、Session 创建者的 principal 和 Core 构建版本 签发或轮换按以下顺序检查，返回第一个失败：请求体（400）；目标 Environment（404）；已归档 Project（409 `project_archived`）；key 本身（未设置 `rotate` 时为 409 `executor_credential_exists`，轮换从未签发的 `key_id` 时为 404）。 -轮换替换限定于该 Environment 的现有 key 密钥，保留 Environment，立即使旧密钥失效，并恢复已撤销的 key。撤销是幂等操作，每次返回 204，禁止后续注册和连接。 +轮换替换限定于该 Environment 的现有 key 密钥，保留 Environment 和 `key_id`，立即使旧密钥失效，并恢复已撤销的 key。它还会推进该 Environment 的 enrollment generation 及其 Session 分配的 epoch，因此机器必须用新密钥重新注册后才能再次提供服务，正在运行的 Turn 也会失去该 Environment（[Session 分配](../../../docs/zh/runtime-protocol.md#session-assignments)）。撤销是幂等操作，每次返回 204，禁止后续注册，并关闭机器的 Serve。 超时等结果不确定的情况下，不要自动重试。先列出凭证，再轮换同一 `key_id`（已签发但密钥丢失），或重新签发（尚未签发）。 @@ -75,26 +75,22 @@ Core API 不可用时，`oac-core-environment-key` 直接在数据库中签发 ## 连接状态 {#connection-status} -列表必需的 `connection` 对象包含 `status`（`never_enrolled`、`connected` 或 `disconnected`）、`bound_key_id`、`enrolled_at` 和 `last_seen_at`。注册前，三个绑定字段均为 null。注册后，绑定 key 和注册时间描述已有设备；`last_seen_at` 为 null 表示尚无经过认证的心跳。签发另一 key 不改变绑定。轮换或撤销可能使绑定断开，但历史仍可见。过期 Environment 仍按现有列表规则可读，但不能拥有当前执行器权限。 +列表必需的 `connection` 对象包含 `status`（`never_enrolled`、`connected` 或 `disconnected`）、`bound_key_id` 和 `enrolled_at`。注册前，两个绑定字段均为 null。注册后，`bound_key_id` 是为该 Environment 注册的 key，`enrolled_at` 是首次注册的时间；两者都不表示就绪。签发另一 key 不改变绑定。轮换或撤销可能使绑定断开，但历史仍可见。过期 Environment 仍按现有列表规则可读，但不能拥有当前执行器权限。 -Connected 表示 Environment 已连接，其设备和执行器 key 仍有当前 Core 权限，且进程内 gateway 有一个用当前 key 认证的开放 peer。Core 观察 peer 后重新检查权限。旧 key 的存活 socket、设备时间戳或看似就绪的 Environment 均不充分；没有 gateway 时 Core 不返回 connected。这些事实是观测结果，不预留连接，也不保证原生执行或模型就绪。`last_seen_at` 可能滞后一个心跳间隔。 +Connected 表示已注册的 key 仍有当前 Core 权限，且 Environment 的 live Link resource 正是该 key 的 enrollment，并以当前 generation Serve：relay 持有机器上 Sandbox I/O 服务的 serve peer。Core 观察 serve peer 后重新检查权限，因此仍用旧密钥提供服务的 peer 不算已连接。该规则与托管 Environment 相同（[就绪事实](../../../docs/zh/sandbox-provider.md#four-distinct-readiness-facts)）。它是观测结果，不预留连接，也不保证原生执行或模型就绪。 -列表元数据和绑定事实使用同一个只读数据库快照。快照在实时权限检查前结束，因此已提交的轮换或撤销不会被快照隔离隐藏。已知权限丢失映射为 disconnected；观测和存储失败仍返回错误。设备 ID 和凭证摘要为内部数据，不序列化。公开 `/v1` Environment 形状不变。 +列表元数据和绑定事实使用同一个只读数据库快照。快照在实时权限检查前结束，因此已提交的轮换或撤销不会被快照隔离隐藏。已知权限丢失映射为 disconnected；观测和存储失败仍返回错误。Enrollment ID 和凭证摘要为内部数据，不序列化。公开 `/v1` Environment 形状不变。 ### 私有连接确认 {#private-connection-confirmation} -`GET /api/v1/agent-daemon/connection?environment_id=UUID` 使用执行器 bearer，直接发往 Core（反向代理将 `/api/v1` 路由到 Core，console 不提供该接口）。它属于私有 daemon 传输，不属于公开 Agents API。它只读取现有授权和绑定，不注册设备、启动执行或修改资源。no-store 响应只包含请求的 `environment_id` 和 `status`（`connected` 或 `disconnected`）。Connected 要求 Environment 观测、确切的 Session 和设备绑定、当前执行器权限，以及用相同凭证认证的存活 gateway socket。过期观测或携带已轮换 key 的 socket 不能确认连接。 +`GET /api/v1/agent-daemon/connection?environment_id=UUID` 使用执行器 bearer，直接发往 Core（反向代理将 `/api/v1` 路由到 Core，console 不提供该接口）。它属于私有 daemon 传输，不属于公开 Agents API。它只读取现有授权和注册，不注册、不启动执行，也不修改资源。no-store 响应只包含请求的 `environment_id` 和 `status`（`connected` 或 `disconnected`）。`connected` 对所出示的凭证遵循[连接状态](#connection-status)规则：Environment 的 live Link resource 是同一凭证的 enrollment，且正在 Serve。过期观测或使用已轮换密钥的 serve peer 不能确认连接。 -无效、已撤销、其他范围或已删除 Session 的权限返回 401；已绑定 Environment 使用不同 key 返回 409。响应不暴露实际绑定或数据库诊断。 +无效、已撤销、其他范围或已删除 Session 的权限，以及失败或已过期的 Environment，返回 401；已注册 Environment 使用不同 key 返回 409。响应不暴露实际绑定或数据库诊断。 -安装器从返回的 `remote_url` 推导此路由，不跟随重定向。启动 daemon 后，每秒轮询一次，最多 45 秒。401 或 409 立即失败。超时后打印 daemon 的 `connect.log` 路径，请求重新执行同一命令；daemon 继续重连，安装、凭证和历史保留。连接确认只证明认证成功，不证明模型访问、Harness 能力或执行完成。 +安装器从返回的 `remote_url` 推导此路由，不跟随重定向。启动 daemon 后，每秒轮询一次，最多 45 秒。401 或 409 立即失败。超时后打印 daemon 的 `connect.log` 路径，请求重新执行同一命令；daemon 继续重连，安装、凭证和历史保留。连接确认证明机器正在为该 Environment 提供服务，不证明模型访问、Harness 能力或执行完成。 ## 已撤销或轮换的凭证 {#revoked-or-rotated-credential} -Core 永久拒绝 daemon 时（注册 401/409、永久 WebSocket 拒绝，或 daemon 版本来自其他 Core 分发），daemon 只打印一次原因，停止发请求直到被停止；随后成功退出，避免按退出重启的 supervisor 循环。再次启动时只尝试一次注册，然后再次停驻。临时故障保持正常重连行为，不重放执行。 +Core 永久拒绝机器注册时（401 或 409），daemon 只打印一次原因，停止发请求直到被停止；随后成功退出，避免按退出重启的 supervisor 循环。再次启动时只尝试一次注册，然后再次停驻。临时故障（包括 Sandbox I/O 服务退出）按退避重试，不重放执行。 -机器只能使用绑定的 `key_id` 重连，轮换新密钥后替换配置路径的凭证文件；[自托管指南](../../../docs/zh/getting-started/self-hosted.md#rotate-or-revoke)提供步骤。新 `key_id` 无法重新连接已绑定的 Environment：签发成功，但用它注册返回 409。轮换不重装 Harness、不改变工作区、不替换原生历史；不要为恢复凭证创建新 Session 历史。 - -## 模型服务 {#model-provider} - -[模型执行](model-execution.md)负责模型服务的解析和交付。保存的 Agent 的服务 key 会交给 Project 中用该 Agent 创建的每个 `self_hosted` Session 的执行器，部署默认值的 key 会交给回退到它的每个 `self_hosted` Session 的执行器，因此任何能创建这类 Session 并运行执行器的人都能读取该 key。 +机器只能使用绑定的 `key_id` 重连，轮换新密钥后替换配置路径的凭证文件；[自托管指南](../../../docs/zh/getting-started/self-hosted.md#rotate-or-revoke)提供步骤。新 `key_id` 无法重新连接已绑定的 Environment：签发成功，但用它注册返回 409。轮换不改变工作区或 Session 的原生历史；不要为恢复凭证创建新 Session。 diff --git a/contracts/agents-api/zh/environments.md b/contracts/agents-api/zh/environments.md index 025c769ef..98a676688 100644 --- a/contracts/agents-api/zh/environments.md +++ b/contracts/agents-api/zh/environments.md @@ -1,7 +1,7 @@ --- title: "环境与模板" source: contracts/agents-api/environments.md -source_hash: 6bfd865756dbe0d358c687ce83f23b610ab9f6db9902e86ef775741982920ae5 +source_hash: 91a4b01924b65e48003f3a66a76a840272b5f672d0e0b667e6afd3626b8e4564 --- Environment 是 Session 的执行资源，包括 Harness 运行所在的机器、工作区以及已完成准备的能力。Session 通过其 `environment` 配置创建 Environment；不存在独立的 create 调用。Environment Template 是 Session 创建时解析的可复用准备配置。本契约涵盖这两类资源、两种放置方式、输入接纳、能力准备、Skills、Plugins 和 MCP 连接来源。 @@ -53,7 +53,7 @@ Core 在 Session 创建事务中创建 Environment 记录；Session upsert 会 | Provider 分配 | 计算资源和文件系统的生命周期：`openai_hosted` 使用 Core 的 Sandbox Provider，`self_hosted` 使用应用程序 | | 设备与 daemon 连接 | 经认证的 Runtime 身份和可替换的分派传输 | | Harness 进程与原生会话 | 原生模型和工具循环、其执行状态及原生历史 | -| 注册 | `self_hosted` 机器的 Environment、设备和 executor key 的精确绑定 | +| 注册 | `self_hosted` 机器用来 Serve 该 Environment 的 [Link](../../../docs/zh/sandbox-link-protocol.md) resource 的 executor key | ### 托管（`openai_hosted`） {#hosted-openai-hosted} @@ -71,18 +71,18 @@ Core 在 Session 创建事务中创建 Environment 记录；Session upsert 会 应用程序拥有机器。它使用干净的绝对路径 `workspace_directory` 和可选的绝对本地 `capability_directories` 创建 Session。Core 会返回 Environment ID、`remote_url` 以及 `x_agents_core.installation` 中的一条安装命令；在该机器上运行此命令会安装 daemon 并为其注册（[self-hosted guide](../../../docs/zh/getting-started/self-hosted.md)、[executor credentials](environment-executor-credentials.md)）。 - `remote_url` 是根据 Core 的公共 URL 推导出的 daemon WebSocket URL，绝不根据请求头或 daemon 地址生成。它指定 Core 的私有 daemon 传输通道。 -- 注册会将精确的 Session、Environment、设备和 executor key 绑定在一起。它不会创建任何分配，也无法将 Session 迁移到另一台设备。 +- 注册记录为该 Environment 的 Link resource 提供服务的 executor key；最先注册的 key 保有它。注册不会创建任何分配，也不绑定 Session：Session 运行在部署的 agent host 上（[Session 分配](../../../docs/zh/runtime-protocol.md#session-assignments)）。 - Session 的工作区必须等于 `/workspace` 别名，或等于 Runtime 绑定到的精确规范目录。指定某个路径并不会授予对它的访问权限。 - Session 读取、列表和事件会返回带有 Environment ID、工作区及能力目录的 `self_hosted` 输出，但绝不返回私有配置。`capability_directories` 列出调用方选择的内容；Runtime 的安装位置保持私有。 - 计算资源、工作区和文件仍归应用程序所有。删除 Session 或撤销凭据会拒绝后续访问，但不会停止原生进程；机器所有者负责停止和清理。 -- 工作区和原生历史必须能在 daemon 重启后继续存在。丢失它们绝不授权进行静默替换或重播。 +- 工作区必须能在 daemon 重启后继续存在。丢失它绝不授权进行静默替换或重播。 **应用管理的 E2B。** 应用程序可以在由其使用 E2B SDK 创建、续期和销毁的 E2B sandbox 中运行 Runtime，然后将该 Runtime 注册为 `self_hosted` Environment（[E2B Runtime guide](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/e2b/README.md)）。Core 不为其保留 E2B 分配，也绝不续期或终止它。 ### 所有权规则 {#ownership-rules} - 将 Environment 身份、所有权、配置和生命周期保留在 Core 中，并使其与 Provider 计算资源、设备身份、daemon 套接字和原生会话相分离。将可变连接状态排除在不可变配置之外；替换后的所有者会使过期观察值失效。 -- 调用方、设备和 Environment 连接使用彼此不同的凭据。daemon gateway 会对已注册的 executor key 和精确设备进行认证。连接观察保留 generation 和 revision 栅栏。注册和连接都不表示已就绪。 +- 调用方、设备和 Environment 连接使用彼此不同的凭据。relay 使用已注册的 executor key 认证 `self_hosted` 机器的 Serve。连接观察保留 generation 和 revision 栅栏。注册和连接都不表示已就绪。 - 轮换、撤销、Session 删除和所有权丧失都会拒绝后续访问；但它们不保证原生效果会立即停止。 - 原生历史保留在绑定的 Runtime 上。替换计算资源时必须保留或以可证明的方式恢复原生历史；绝不能静默移动已绑定的 Session 或重播未知工作。 - 持久元数据读取不需要活跃的 Runtime。实时文件读取需要获得对精确工作区的授权视图和有界操作所有权。写入、替换或撤销工作区所有者的操作也适用变更栅栏。 diff --git a/contracts/agents-api/zh/machine-api.md b/contracts/agents-api/zh/machine-api.md index 59b0ebe6a..70ce5314b 100644 --- a/contracts/agents-api/zh/machine-api.md +++ b/contracts/agents-api/zh/machine-api.md @@ -1,7 +1,7 @@ --- title: "机器连接 API" source: contracts/agents-api/machine-api.md -source_hash: ede6031c1e4761dbbf27dd19829642ff892c2e12f26ec972da46dcb76c43c4fe +source_hash: 1540a3b84eb1cfa977654b46d407e1a4de7f502bab9c5264e21a8cc57e96d238 --- 机器通过 `/api/v1` 调用 Core：包括沙箱节点、Runtime daemon、Sandbox I/O 服务和自托管安装器。各路由仅接受所列凭据，不接受 Core 密钥或 Project API 密钥；控制台登录也不授予此处权限。反向代理将 `/api/v1` 直接发送给 Core；Web 不提供这些路由。 @@ -34,7 +34,7 @@ source_hash: ede6031c1e4761dbbf27dd19829642ff892c2e12f26ec972da46dcb76c43c4fe | 登记 token | `POST /core/v1/sandbox/enrollment-tokens`（Web **Add node**），带节点批准容量。使用一次；在响应 `expires_at` 过期 | 无节点 ID 的 `sandbox-node/configuration`、`sandbox-node/enroll` | | 节点凭据 | 节点自身：生成 32 至 256 个无空白字符的密钥，在登记时注册 | 带 `X-OAC-Node-ID` 的 `sandbox-node/configuration`、`sandbox-node/identity`、`sandbox-node/connect` | | 安装授权 | `self_hosted` Session 的 `x_agents_core.installation` 命令；短期有效 | `agent-daemon/installation` 及其 `claim` | -| 执行器凭据 | 安装领取，或 Core 密钥[执行器凭据路由](environment-executor-credentials.md) | `agent-daemon/enroll` 和 `agent-daemon/connection`；登记后也作为绑定设备的 daemon 凭据 | +| 执行器凭据 | 安装领取，或 Core 密钥[执行器凭据路由](environment-executor-credentials.md) | `agent-daemon/enroll` 和 `agent-daemon/connection`；登记后也作为该 Environment 的 enrollment 在 `sandbox-link` 上的 Serve 凭据 | | 托管沙箱 daemon 凭据 | Core 为每个受管分配签发，通过[引导文件](../../../docs/zh/runtime-bootstrap.md)交付 | `agent-daemon/bootstrap`、`device-status` 和 `ws` | | 操作者设备配置 | 具有数据库访问权限的操作者运行 `oac-core-device` | `agent-daemon/bootstrap`、`device-status` 和 `ws` | @@ -42,7 +42,7 @@ Core 对存储的每个 token 和凭据仅保留 SHA-256 摘要；安装授权 ### 操作者设备配置 {#operator-device-profile} -`environment: none` Session 的引擎主机使用操作者直接在数据库创建的设备配置连接： +`oac-core-device` 直接在数据库中创建 Runtime 设备配置： ```sh umask 077 @@ -51,7 +51,7 @@ OAC_DATABASE_URL=... oac-core-device --tenant --name 'engine host' oac-daemon connect --profile default ``` -`--tenant` 为 Project 执行租户 UUID，`--url` 为不带路径的 Core origin。命令打印配置一次：`server_url`（origin 加 `/api/v1`）、`runtime_id`（设备 ID）、`runner_credential` 和 `device_name`。使用新配置，不覆盖其他设备文件；私密复制到远程主机相同路径。`oac-core-device --tenant --revoke ` 撤销设备：立即拒绝新连接，已有连接在下一次心跳关闭。Worker 将每个 `none` Session 绑定到其租户内声明所需能力的已连接设备，重试和重启保留绑定；自托管 Session 不使用此路径。 +`--tenant` 为 Project 执行租户 UUID，`--url` 为不带路径的 Core origin。命令打印配置一次：`server_url`（origin 加 `/api/v1`）、`runtime_id`（设备 ID）、`runner_credential` 和 `device_name`。使用新配置，不覆盖其他设备文件；私密复制到远程主机相同路径。`oac-core-device --tenant --revoke ` 撤销设备：立即拒绝新连接，已有连接在下一次心跳关闭。Core 只把 Session 绑定到部署的 agent host（[Session 分配](../../../docs/zh/runtime-protocol.md#session-assignments)），因此此配置的设备不运行任何 Session。 ## 节点路由 {#node-routes} @@ -113,13 +113,13 @@ Core 在一个事务中检查 token 有效、部署已初始化且为节点型 ### 登记自托管 daemon {#enroll-a-self-hosted-daemon} -`POST /api/v1/agent-daemon/enroll` 携带执行器凭据及精确正文 `{"environment_id": "…"}`（无查询），将一个专用设备绑定到 Environment 的 Session，返回 `device_id`、`session_id`、`environment_id` 和 `workspace_directory`。不返回其他凭据：执行器凭据成为该设备的 daemon 凭据。相同凭据重试返回相同绑定。成功响应包含 `Cache-Control: no-store`。 +`POST /api/v1/agent-daemon/enroll` 携带执行器凭据及精确正文 `{"environment_id": "…"}`（无查询），把机器登记为 Environment 的 [Link](../../../docs/zh/sandbox-link-protocol.md) resource，并返回由 Core 提供的[启动输入](../../../docs/zh/sandbox-bootstrap.md#launch-input)字段：`link_url` 和 `resource`（`tenant_id`、`environment_id`、`kind` 为 `enrollment`、`id` 和 `generation`）。不返回其他凭据：执行器凭据就是该 resource 的 Serve 凭据。最先登记该 Environment 的密钥保有它，用该密钥重试返回同一 resource 及其当前 generation。成功响应包含 `Cache-Control: no-store`。 | HTTP | 时机 | | --- | --- | | 400 | 正文格式错误或存在任何查询 | | 401 | 凭据无效、撤销、属于其他范围，Session 已删除，或 Environment 无当前执行器权限 | -| 409 | Environment 已绑定到不同密钥或设备 | -| 503 | 存储不可用 | +| 409 | 其他执行器密钥已登记该 Environment | +| 503 | [公共 URL](../../../docs/zh/configuration.md#changing-the-public-url) 不是 https 时，在检查凭据前返回 `{"error": "no_sandbox_link", "detail": "a self_hosted sandbox needs an https public URL"}`；否则表示存储不可用 | -登记不创建受管分配，也不授予 Session API 访问权限。daemon 在凭据旁保存绑定，拒绝其他 Environment 的原生历史。网关与 Worker 在每次连接和分发时重查凭据权限，因此轮换、撤销和删除 Session 终止后续使用。[自托管指南](../../../docs/zh/getting-started/self-hosted.md)提供操作步骤，[执行器凭据契约](environment-executor-credentials.md#revoked-or-rotated-credential)描述 daemon 如何处理永久拒绝。 +登记不创建受管分配，不绑定 Session，也不授予 Session API 访问权限。机器为该 resource 提供服务期间，Core 把 Session 绑定到部署的 agent host（[Session 分配](../../../docs/zh/runtime-protocol.md#session-assignments)）。relay 在每次 Serve 和 Open 时重查凭据权限，因此轮换、撤销和删除 Session 终止后续使用。[自托管指南](../../../docs/zh/getting-started/self-hosted.md)提供操作步骤，[执行器凭据契约](environment-executor-credentials.md#revoked-or-rotated-credential)描述 daemon 如何处理永久拒绝。 diff --git a/contracts/agents-api/zh/model-execution.md b/contracts/agents-api/zh/model-execution.md index 173fd3262..e8ca07076 100644 --- a/contracts/agents-api/zh/model-execution.md +++ b/contracts/agents-api/zh/model-execution.md @@ -1,7 +1,7 @@ --- title: "模型执行" source: contracts/agents-api/model-execution.md -source_hash: 4a59ada66fa31f92d3775175281316854b00255a68a4c25ac1959f7e05863825 +source_hash: e8abe9e40a54a35935d08e2d2c26e1269b8139e9972fb5359d483db525d0862e --- 每个 Session 都运行一个 Harness，并使用一个模型提供商。Core 通过三个固定版本上游协议未定义的 Core 扩展来选择它们：`x_agents_core.harness` 选择 Harness，`x_agents_core.model_provider` 提供端点和密钥，`x_agents_core.harness_config` 携带原生模型参数。Core 没有提供商目录、模型别名解析或产品权限模型；除 Session 和已保存 Agent 配置包外，唯一存储的配置包是每个 Harness 的一个 [deployment default](#deployment-defaults)。本文档定义 Harness—模型提供商协议：[`internal/modelprovider/config.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/modelprovider/config.go) 定义 Core 和 Runtime 共同应用的[提供商规则](#session-override)，并声明[凭据网关](#credential-gateway)转发的内容，每个 Harness 则通过 [`internal/harnessconfig/harness.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/harnessconfig/harness.go) 声明其协议和原生参数。 @@ -38,7 +38,7 @@ Session 的 `environment` 和 Environment Templates 用于选择准备流程， MiniMax Code 要求上下文限制和输出限制均为正数。Core 会在写入 Session 前验证解析后的组合。Core 和 Runtime 读取 `internal/harnessconfig` 中相同的有序 `protocols` 声明。任何地方都不在协议之间转换，Harness 内部也不例外。不受支持的已保存配置和 Session 快照一旦使用便会失败；它们绝不会在何处被重写、创建别名或迁移。 -每种 Environment 类型都接受所有来源，因为密钥只会到达 agent host 上的[凭据网关](#credential-gateway)，绝不会进入 Harness 或沙箱。对于 `self_hosted`，agent host 就是应用程序的执行器，因此在那里使用部署默认值会把运营方的密钥交给该执行器的网关。每个 Session 都必须解析到一个配置包：否则创建会在发生任何写入之前以 400 `model_provider_required` 被拒绝，错误参数为 `x_agents_core.model_provider`，并会返回说明应配置内容的消息。已保存 Agent 可以省略配置包，交由 Session 或部署默认值提供。 +每种 Environment 类型都接受所有来源，因为密钥只会到达部署的 agent host 上的[凭据网关](#credential-gateway)，绝不会进入 Harness、沙箱或 `self_hosted` 机器。每个 Session 都必须解析到一个配置包：否则创建会在发生任何写入之前以 400 `model_provider_required` 被拒绝，错误参数为 `x_agents_core.model_provider`，并会返回说明应配置内容的消息。已保存 Agent 可以省略配置包，交由 Session 或部署默认值提供。 | 操作 | 省略 | 显式 null | | --- | --- | --- | @@ -84,7 +84,7 @@ Core 从同一个数据库快照读取 Agent 配置和加密配置包；显式 解析后的提供商配置会在 Session 创建事务中被冻结并加密，使用自己的加密用途，并绑定到 Project 和 Session。创建重试会将 Session 自身的配置包纳入请求哈希，因此使用相同 Idempotency-Key 时，更改密钥或端点会产生冲突；密钥进入任何存储哈希时，只会表现为由部署凭据密钥加键控的指纹。任何公开的 Session、Agent、Environment、事件或常规配置均不包含该密钥。顶层扩展仅可写入，无法更新。 -在分派时，Core 会通过绑定到 Session 的 daemon 连接，将快照作为一个机密提供商配置包发送出去；适配器通过原生配置让 Harness 指向[凭据网关](#credential-gateway)。快照缺失或无法解密时，Core 绝不会回退到其他凭据。对于 `self_hosted`，接收方 daemon 是为该 Session 自身 Environment 注册的执行器，并持有 Session 创建者主体的当前执行器凭据；凭据轮换或吊销会在继续分派前关闭套接字。网关在 Session 期间将密钥保存在内存中，密钥从不进入 Harness 的环境、配置或 home，也不进入沙箱。 +在分派时，Core 会通过绑定到 Session 的 daemon 连接，将快照作为一个机密提供商配置包发送出去；适配器通过原生配置让 Harness 指向[凭据网关](#credential-gateway)。快照缺失或无法解密时，Core 绝不会回退到其他凭据。网关在 Session 期间将密钥保存在内存中，密钥从不进入 Harness 的环境、配置或 home，也不进入沙箱。 ## 凭据网关 {#credential-gateway} diff --git a/contracts/agents-api/zh/runtime-observability.md b/contracts/agents-api/zh/runtime-observability.md index 223c463c9..5602d054a 100644 --- a/contracts/agents-api/zh/runtime-observability.md +++ b/contracts/agents-api/zh/runtime-observability.md @@ -1,7 +1,7 @@ --- title: "运行时可观测性" source: contracts/agents-api/runtime-observability.md -source_hash: 31fa597224d654f347c7f438bea78d8e548e7588849d4346c599aac8a8f8054c +source_hash: f77670c854cb175c2a30a8428e75c5c382c28b94b56479dfbee9fd82d2a57785 --- 这是面向贡献者的契约，规定 Core 如何观测 Runtime 并保留其历史。路由和响应字段见 [Runtime telemetry API](runtime-observability-api.md)。代码位于 `services/core/internal/runtimeobs`（解析、源、采样器和导出）、`internal/runtimehistory`（历史查询和 PostgreSQL 存储）以及 `internal/runtimeobs/otlpexporter`。 @@ -14,7 +14,7 @@ source_hash: 31fa597224d654f347c7f438bea78d8e548e7588849d4346c599aac8a8f8054c ```text managed: tenant_id -> session_id -> environment_id -> runtime_allocation_id -self-hosted: tenant_id -> session_id -> environment_id -> device_id + connection_generation +self-hosted: tenant_id -> session_id -> environment_id none: tenant_id -> session_id (no Session-owned Runtime instance) ``` diff --git a/docs/api/public-agent-api.md b/docs/api/public-agent-api.md index 5c3d1a8c9..b6aabd446 100644 --- a/docs/api/public-agent-api.md +++ b/docs/api/public-agent-api.md @@ -248,7 +248,7 @@ Returns 201 with the Session: | `environment.type` | Runs on | Notes | | --- | --- | --- | | `openai_hosted` | A sandbox Core creates on a node or E2B; the administrator provides the capacity | Optional `network`, `packages`, `files`, `skills`, `plugins`, `env`, `capability_directories`, `setup_commands`, or a template | -| `self_hosted` | Your own Linux, macOS or Windows machine | Requires an absolute `workspace_directory`. Skills, packages, files or a template go in `x_agents_core.environment`. The response carries install commands in `x_agents_core.installation`; see [self-hosted execution](../getting-started/self-hosted.md) | +| `self_hosted` | Your own Linux machine | Requires an absolute `workspace_directory`. Skills, packages, files or a template go in `x_agents_core.environment`. The response carries install commands in `x_agents_core.installation`; see [self-hosted execution](../getting-started/self-hosted.md) | | `none` | A device connection an operator registered, with no workspace | `input` required | A new `openai_hosted` Session reads `idle` while Core prepares its sandbox; its first Turn starts when the Environment is ready. The [Environment contract](../../contracts/agents-api/environments.md) owns placement, expiry and preparation. diff --git a/docs/configuration.md b/docs/configuration.md index 48baf73da..75e5d45c1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,7 +31,7 @@ Use `docker compose ps` to check the services. See [stop and restart](./getting- `OAC_PUBLIC_URL` is the one origin that applications, nodes, sandboxes and self-hosted executors use. Core derives the daemon WebSocket URL, the sandbox Link URL, the self-hosted `remote_url` and each sandbox's connection address from it. It is an http or https origin: the address browsers and nodes use. The installation serves Web over HTTP on `OAC_WEB_PORT`; a reverse proxy or hosting platform terminates HTTPS when you put one in front. -Sandboxes and agent hosts dial the [sandbox Link](./sandbox-link-protocol.md) at `wss:///api/v1/sandbox-link` when the origin is https. An http origin on `localhost` or a loopback address gives `ws:///api/v1/sandbox-link`, which only peers in Core's own network namespace, such as a colocated agent host, can reach. An http origin on any other host gives no Link URL. Hosted sandboxes run outside Core's network namespace, so Core selects and admits them only while the origin is https on a host that is not loopback. +Sandboxes and agent hosts dial the [sandbox Link](./sandbox-link-protocol.md) at `wss:///api/v1/sandbox-link` when the origin is https. An http origin on `localhost` or a loopback address gives `ws:///api/v1/sandbox-link`, which only peers in Core's own network namespace, such as a colocated agent host, can reach. An http origin on any other host gives no Link URL. Hosted sandboxes run outside Core's network namespace, so Core selects and admits them only while the origin is https on a host that is not loopback. A self-hosted machine dials the Link from its own host, so Core [enrolls](../contracts/agents-api/machine-api.md#enroll-a-self-hosted-daemon) one only while the origin is https. To change it, point the reverse proxy at the new address first, then edit `OAC_PUBLIC_URL` and run `oac apply`. Afterwards: diff --git a/docs/getting-started/self-hosted.md b/docs/getting-started/self-hosted.md index e5b8382a0..8a4d53c59 100644 --- a/docs/getting-started/self-hosted.md +++ b/docs/getting-started/self-hosted.md @@ -6,26 +6,22 @@ A `self_hosted` Session runs on a machine your application owns: a workstation, **The daemon is not a sandbox.** Tools run with the permissions of the account that starts it and can reach whatever that account can. Use a container or VM when you need isolation; see [Runtime and outer isolation](../concepts.md#runtime-and-outer-isolation). The daemon does not restrict network access, so a Template that requires a network policy is rejected for a self-hosted Session. -The Session's model provider resolves as for any other Session, so the installation default applies when neither the Session nor its Agent supplies one; its key then reaches this machine ([model execution](../../contracts/agents-api/model-execution.md#saved-defaults-and-precedence)). The machine gets an executor credential that works for this one Environment and nothing else. +The Session's Harness runs on the deployment's agent host and reads files and runs tools on this machine through the machine's [Sandbox I/O service](../sandbox-bootstrap.md), so no model key reaches the machine. The machine gets an executor credential that works for this one Environment and nothing else. ## Platforms -| Platform | Codex | Claude Code | MiniMax Code | -| --- | --- | --- | --- | -| Linux amd64 | Supported | Supported | Supported | -| macOS arm64 | Supported | Supported | Supported | -| Windows amd64 | Supported | Supported | Not supported | +Self-hosted machines run Linux amd64. On macOS and Windows, `oac-daemon start` refuses to start. The installer brings its own pinned Node.js and Harness versions (listed in [`scripts/build-native-installer.mjs`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/scripts/build-native-installer.mjs)) and leaves other installations of those tools untouched. On a platform without a matching installer, the command fails. The machine needs: -- HTTP or HTTPS access to Core, and to the release download host unless Core carries an offline copy of the installers; -- Bash for environment setup and MiniMax Code tools; on Windows, Git Bash, which Claude Code also requires; +- HTTPS access to Core, and to the release download host unless Core carries an offline copy of the installers. Core's [public URL](../configuration.md#changing-the-public-url) must be https, or Core refuses the machine's enrollment; +- Bash for environment setup and MiniMax Code tools; - Python and pip when the Session's packages need them; - any system packages your setup needs. The daemon never runs apt, sudo or another elevation command, so install them through the host's normal administration. -No administrator privileges or Docker are needed. The Unix download command also uses `curl`, `tar`, `gzip`, a SHA-256 tool and the system file-lock command (`flock` on Linux, `lockf` on macOS). Windows needs its system `tar.exe`; the command checks for it before downloading. The Runtime home must allow executable files to run. On a Unix `noexec` mount, choose another absolute directory with `OAC_RUNTIME_HOME` before running the command. +No administrator privileges or Docker are needed. The download command also uses `curl`, `tar`, `gzip`, a SHA-256 tool and `flock`. The Runtime home must allow executable files to run. On a `noexec` mount, choose another absolute directory with `OAC_RUNTIME_HOME` before running the command. ## Connect a machine @@ -51,7 +47,7 @@ No administrator privileges or Docker are needed. The Unix download command also }, ) installation = session.model_dump()["x_agents_core"]["installation"] - print(installation["commands"]["posix"]) # use "powershell" on Windows + print(installation["commands"]["posix"]) ``` 2. Run the command on the target machine with the account that should run the tools. It downloads the installer matched to this Core, verifies its checksum, asks which Harnesses to install and where, installs them, creates the workspace if needed, starts the daemon and checks its connection. @@ -66,7 +62,7 @@ The installer reports three results: | Result | Meaning | | --- | --- | | **Installation** | The selected Harnesses passed their readiness checks | -| **Daemon connection** | Core confirmed the daemon's authenticated connection | +| **Host connection** | Core confirmed that the machine serves this Environment over the [sandbox Link](../sandbox-link-protocol.md) | | **Model configuration** | Not checked; the first Turn uses the Session's model provider | Downloads retry temporary network failures up to three attempts and show progress in a terminal. Disk space is checked before downloading, extracting and copying components. If a download or installation is interrupted, rerun the command: it clears unfinished temporary copies while preserving completed components, credentials and the workspace. Download staging lives in `native-download` under the Runtime home; its small `download.lock` file remains for concurrency control. An active download or installation is never cleared by another run. If the command expires, copy a fresh one from the Session. @@ -99,7 +95,7 @@ environment = { } ``` -Paths are absolute in the machine's own syntax (Unix, Windows drive or UNC); the daemon checks them, not Core. Fill these directories before the daemon connects. They are ordinary paths visible to the daemon; naming one does not mount it or create a sandbox. +Paths are absolute; the daemon checks them, not Core. Fill these directories before the daemon connects. They are ordinary paths visible to the daemon; naming one does not mount it or create a sandbox. Use `x_agents_core.environment` for the same Project-owned Skills, Plugin archives, files, packages, setup commands or Template used by a managed Session: @@ -114,7 +110,7 @@ session = client.beta.agents.sessions.create( ) ``` -The same extension works with `environment={"type": "openai_hosted"}`. Do not repeat a field in both `environment` and the extension. Setup runs with the daemon's account permissions. Deployment model keys are never sent to your machine. +The same extension works with `environment={"type": "openai_hosted"}`. Do not repeat a field in both `environment` and the extension. Setup runs with the daemon's account permissions. Before the first Turn the daemon copies these sources into a snapshot. Reconnecting reuses the snapshot even after you edit the sources; a new Session takes a new snapshot. The [preparation contract](../../contracts/agents-api/environments.md#runtime-capability-preparation) lists fields, merge rules, snapshot behavior and failures. @@ -124,14 +120,14 @@ The installation's `bin/oac-daemon` finds its own installation. Use it for: | Command | Effect | | --- | --- | -| `oac-daemon start` | Validate the installed Harnesses and start the daemon in the background | +| `oac-daemon start` | Enroll the machine and start the daemon in the background. The daemon runs the [Sandbox I/O service](../sandbox-bootstrap.md) and, when it exits, enrolls again and restarts it | | `oac-daemon status` | Show the local profile and process; not the connection | | `oac-daemon logs -n 100`, `oac-daemon logs -f` | Print or follow the daemon log | | `oac-daemon stop` | Stop the daemon | If you set `OAC_RUNTIME_HOME`, use the same value for every command. Check the connection under **Host connection** on the Session's page in Web, or with the [connection status](../../contracts/agents-api/environment-executor-credentials.md#connection-status). -To add a Harness, run the install command again (a fresh copy from Web if it has expired) with the same installation directory and the Harness to add. The installer checks the existing contents, adds only missing components and keeps the Harnesses already installed. Restart a running daemon afterwards so it discovers the new Harness. +To add a Harness, run the install command again (a fresh copy from Web if it has expired) with the same installation directory and the Harness to add. The installer checks the existing contents, adds only missing components and keeps the Harnesses already installed. Stopping the daemon, cancelling a Turn or deleting the Session never removes the machine's workspace, native history or capability snapshot. An installation from another daemon version, or one whose files were changed, is refused. The installer never upgrades, repairs or migrates it; install into a separate directory. @@ -162,4 +158,4 @@ The same installer accepts an already extracted distribution and a credential fi "$HOME/.oac/my-runtime/bin/oac-daemon" start ``` -Use the Session's `remote_url` and Environment ID. In PowerShell, run `.\oac-daemon.exe` with native absolute paths. This mode needs an existing workspace and does not start the daemon until you run `start`. +Use the Session's `remote_url` and Environment ID. This mode needs an existing workspace and does not start the daemon until you run `start`. diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index 7397e07fa..33fb5cc6e 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -29,7 +29,7 @@ The provider creates the account, mounts and workspace, delivers this file, sets The Runtime validates the input and owns authentication and connection. A successful launch proves only the handoff: an authenticated connection, prepared capabilities and execution readiness are separate observations under the [Core–Runtime protocol](./runtime-protocol.md), and the [Sandbox Provider guide](./sandbox-provider.md#four-distinct-readiness-facts) lists what each one proves. -Self-hosted executors and operator-provisioned devices get their daemon identity in other ways; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. All of them enter the same Runtime execution loop. +Operator-provisioned devices get their daemon identity in another way; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. A self-hosted machine runs no Runtime: it enrolls and serves its Environment through the [Sandbox I/O service](./sandbox-bootstrap.md). ## Verification diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index 8eb0956d2..05350b58c 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -106,7 +106,7 @@ Usage frames and the final usage snapshot each carry the cumulative measurement ## Session assignments -An assignment binds one Session to the Runtime that runs it. `Envelope.assignment` names it as `session_id`, `assignment_id` and `epoch`, and is the only place a frame carries it. Core advances the epoch whenever it changes the assignment's desired state, so a lower epoch is stale. +An assignment binds one Session to the Runtime that runs it. `Envelope.assignment` names it as `session_id`, `assignment_id` and `epoch`, and is the only place a frame carries it. Core advances the epoch whenever it changes the assignment's desired state, so a lower epoch is stale. Core binds each Session, whatever its Environment type, to the first connected agent host that admits its Harness, and a Session with an Environment only while that Environment's [Link](./sandbox-link-protocol.md) resource is Serving; the binding never moves to another Runtime. Rotating a `self_hosted` Environment's [executor credential](../contracts/agents-api/environment-executor-credentials.md) advances its enrollment's generation and, in the same write, the epoch of the Session's bound assignment. From then on the Link refuses the earlier epoch's attachments, so a running Turn loses the Environment. The next bind carries the higher epoch and the new generation, and the agent host settles the earlier epoch's work before it binds, as it does for a release. Every Session frame carries the assignment: `execution_prepare`, `execution_start` and `execution_release`; `prompt_cancel`, `prompt_steer` and `function_result`; every frame of `runtime_prepare`, `workspace_read`, `workspace_write` and `workspace_export`; and `environment_quiesce` and `environment_resume`. A reply echoes its request's assignment, and a Run's frames carry the assignment that started it; Core rejects a reply or Run frame that names another. Heartbeats carry none. diff --git a/docs/sandbox-bootstrap.md b/docs/sandbox-bootstrap.md index 27a11c0a3..1d0d99d1f 100644 --- a/docs/sandbox-bootstrap.md +++ b/docs/sandbox-bootstrap.md @@ -36,6 +36,8 @@ The service runs as the account the Provider starts it with. The input names no The Provider creates the account and the sandbox, delivers this file and starts `oac-sandbox-io` as that account. `make build-sandbox-io` builds the static Linux binary. The Provider keeps the file for process restarts and removes it only during explicit cleanup of the resources it owns. +On a [self-hosted machine](./getting-started/self-hosted.md), `oac-daemon start` acts as the Provider for the machine's enrollment. It enrolls with the executor credential and writes this file to `daemon/sandbox-io-bootstrap.json` under the Runtime home, with the `link_url` and `resource` Core returns and the executor token as `credential`. It runs `oac-sandbox-io` as the account that started it. When the service exits, it enrolls again, because a rotation advances the generation, then rewrites the file and restarts the service. + The service validates the input and owns the link: it connects as the serve peer, serves bound streams and reconnects while the credential stays valid. `resource`, including its generation, must be the resource the credential serves, or the relay refuses the link. The File service serves the single [world export](./file-access-protocol.md#attach), and the Provider's sandbox setup owns its isolation. The [Process service](./process-protocol.md#implement-a-service) runs processes as the service's account, and the service, a child subreaper, reaps their orphaned descendants. The service also serves the [Network protocol](./sandbox-network-protocol.md): it resolves names and dials TCP from the sandbox's network namespace, within the egress each stream's `Bind` carries. diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index b095c897e..d0abbe831 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -93,8 +93,8 @@ Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity` | Fact | Evidence | Does not establish | | --- | --- | --- | -| Compute available | Provider observation for the owned allocation | An authenticated Runtime connection or prepared capabilities | -| Runtime connected | Gateway authentication and the exact Environment and device binding | Completed preparation or a usable Harness | +| Compute available | Provider observation for the owned allocation | A Serving Link resource or prepared capabilities | +| Sandbox connected | The Environment's live [Link](./sandbox-link-protocol.md) resource is Serving at its current generation: the relay holds the serve peer of the allocation's Sandbox I/O service, or, for `self_hosted`, of the service on the machine that enrolled. This is what `connected` means for every Environment type | Completed preparation or a usable Harness | | Capabilities prepared | Successful common Runtime preparation with the fixed configuration | Acceptance or completion of a Turn | | Execution admitted | Qualified Harness capabilities and the executor and Turn acceptance path | A completed input, cancellation or reclaimed compute | diff --git a/docs/web/index.md b/docs/web/index.md index 476a44f9a..48e1fa80e 100644 --- a/docs/web/index.md +++ b/docs/web/index.md @@ -42,7 +42,7 @@ Missing data is shown as missing (—), never as zero. [Console API usage](./con Installation creates no Project or key. Opening the console neither allocates compute nor calls a model, and an installation may have zero nodes. Web never starts a Session or sends input. Archiving a hosted Session requests cancellation and reclamation; [administrator authority](../concepts.md#what-administrators-can-and-cannot-do) state what administrators can and cannot do. -The deployment's sandbox backend serves hosted Sessions. An application's `self_hosted` Runtime, including one in its own E2B account, is a separate path that the sandbox configuration does not change. +The deployment's sandbox backend serves hosted Sessions. An application's `self_hosted` machine is a separate path that the sandbox configuration does not change. A loopback public address (`local_only`) keeps nodes and remote applications from reaching Core. The console stays reachable at its own address and [warns about it](./console-api-usage.md#provenance-and-monitoring). diff --git a/docs/zh/api/public-agent-api.md b/docs/zh/api/public-agent-api.md index f7703954d..754be3740 100644 --- a/docs/zh/api/public-agent-api.md +++ b/docs/zh/api/public-agent-api.md @@ -1,7 +1,7 @@ --- title: "Agents API 指南" source: docs/api/public-agent-api.md -source_hash: 78f630fdbc9a1ff51ebcbe4de53708f79a7ad8a75546ad19d14baa9baafa0ddb +source_hash: d5cb941564c90dc90cf38c9bf9c839485aed5239b24d93e8bcabcd104c0a0df7 --- Core 在 `/v1` 提供 [OpenAI Agents API](https://platform.openai.com/docs/api-reference)。可以使用官方 OpenAI SDK 或普通 HTTP。本指南针对每项常见操作同时展示这两种方式，并说明 Core 与 OpenAI 存在差异的地方。 @@ -250,7 +250,7 @@ oac "/agents/sessions" -H "Idempotency-Key: $(uuidgen)" -d '{ | `environment.type` | 运行位置 | 备注 | | --- | --- | --- | | `openai_hosted` | Core 在某个节点或 E2B 上创建的沙箱；容量由管理员提供 | 可选 `network`、`packages`、`files`、`skills`、`plugins`、`env`、`capability_directories`、`setup_commands`，也可指定模板 | -| `self_hosted` | 你自己的 Linux、macOS 或 Windows 计算机 | 要求提供绝对路径 `workspace_directory`。Skills、软件包、文件或模板应放在 `x_agents_core.environment` 中。响应会在 `x_agents_core.installation` 中携带安装命令；请参阅 [self-hosted execution](../getting-started/self-hosted.md) | +| `self_hosted` | 你自己的 Linux 计算机 | 要求提供绝对路径 `workspace_directory`。Skills、软件包、文件或模板应放在 `x_agents_core.environment` 中。响应会在 `x_agents_core.installation` 中携带安装命令；请参阅 [self-hosted execution](../getting-started/self-hosted.md) | | `none` | 由操作员注册的设备连接，无工作区 | `input` 为必填 | 新建的 `openai_hosted` Session 在 Core 准备沙箱期间会读取到 `idle`；Environment 准备就绪后，其首个 Turn 才会启动。[Environment contract](../../../contracts/agents-api/zh/environments.md) 负责部署位置、过期和准备过程。 diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 16c9317b9..a37c6947f 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 45dfb918d391bad141bed29775184c9eaaca35d73167605689772511a5b01de5 +source_hash: 8b5982d90cdc89d85af1caf0aa0e9457e6fd8cb913c9bf5f3dcdd8b1dcba8a55 --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -33,7 +33,7 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 `OAC_PUBLIC_URL` 是应用、节点、沙箱和自托管执行器使用的唯一源地址。Core 从中派生守护进程 WebSocket URL、沙箱 Link URL、自托管 `remote_url` 和每个沙箱的连接地址。它是 http 或 https 源地址，也就是浏览器和节点使用的地址。安装通过 `OAC_WEB_PORT` 以 HTTP 提供 Web；前面有反向代理或托管平台时，由它们终止 HTTPS。 -源地址为 https 时，沙箱和 agent host 通过 `wss:///api/v1/sandbox-link` 连接[沙箱 Link](./sandbox-link-protocol.md)。`localhost` 或回环地址上的 http 源地址得到 `ws:///api/v1/sandbox-link`，只有 Core 自身网络命名空间内的 peer（例如同机的 agent host）能访问。其他主机上的 http 源地址没有 Link URL。托管沙箱运行在 Core 的网络命名空间之外，因此只有源地址是非回环主机上的 https 地址时，Core 才会选择和准入托管沙箱。 +源地址为 https 时，沙箱和 agent host 通过 `wss:///api/v1/sandbox-link` 连接[沙箱 Link](./sandbox-link-protocol.md)。`localhost` 或回环地址上的 http 源地址得到 `ws:///api/v1/sandbox-link`，只有 Core 自身网络命名空间内的 peer（例如同机的 agent host）能访问。其他主机上的 http 源地址没有 Link URL。托管沙箱运行在 Core 的网络命名空间之外，因此只有源地址是非回环主机上的 https 地址时，Core 才会选择和准入托管沙箱。自托管机器从自己的主机连接 Link，因此只有源地址是 https 时，Core 才会[登记](../../contracts/agents-api/zh/machine-api.md#enroll-a-self-hosted-daemon)这类机器。 要更改它，先把反向代理指向新地址，然后编辑 `OAC_PUBLIC_URL` 并运行 `oac apply`。之后： diff --git a/docs/zh/getting-started/self-hosted.md b/docs/zh/getting-started/self-hosted.md index 055fcb883..5076e9836 100644 --- a/docs/zh/getting-started/self-hosted.md +++ b/docs/zh/getting-started/self-hosted.md @@ -1,33 +1,29 @@ --- title: "自托管执行器" source: docs/getting-started/self-hosted.md -source_hash: 19506e4bab34b802e5c3c8a1818bcb5c8397aa91fdf9ace512b5aaeb99341b3a +source_hash: 90d238aec831101e7b9b8081920c8985b1f024a9d3f8f0235fc83b255f84125c --- `self_hosted` Session 在应用拥有的机器上运行：工作站、虚拟机或你管理的沙箱。应用通过 `/v1` 创建 Session，并获得安装 `oac-daemon`、启动它并连接 Core 的命令。Web 在 Session 页面展示同一命令；Web 是可选的。Core 不创建、停止或回收这台机器。 **守护进程不是沙箱。** 工具以启动守护进程的账号权限运行，能访问该账号可访问的所有资源。需要隔离时，请使用容器或虚拟机；参阅 [Runtime 与外层隔离](../concepts.md#runtime-and-outer-isolation)。守护进程不限制网络访问，因此要求网络策略的 Template 会被自托管 Session 拒绝。 -Session 的模型提供商与其他 Session 一样解析，因此当 Session 及其 Agent 都未提供时会使用安装默认模型，其密钥随后会到达这台机器（[模型执行](../../../contracts/agents-api/zh/model-execution.md#saved-defaults-and-precedence)）。机器获得的执行器凭据仅适用于这一个 Environment。 +Session 的 Harness 在部署的 agent host 上运行，通过这台机器的 [Sandbox I/O 服务](../sandbox-bootstrap.md)读取文件和运行工具，因此模型密钥不会到达这台机器。机器获得的执行器凭据仅适用于这一个 Environment。 ## 平台 {#platforms} -| 平台 | Codex | Claude Code | MiniMax Code | -| --- | --- | --- | --- | -| Linux amd64 | 支持 | 支持 | 支持 | -| macOS arm64 | 支持 | 支持 | 支持 | -| Windows amd64 | 支持 | 支持 | 不支持 | +自托管机器运行 Linux amd64。在 macOS 和 Windows 上，`oac-daemon start` 拒绝启动。 安装程序自带固定版本的 Node.js 和 Harness（列于 [`scripts/build-native-installer.mjs`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/scripts/build-native-installer.mjs)），不修改这些工具的其他安装。在没有匹配安装程序的平台上，命令会失败。 机器需要： -- 通过 HTTP 或 HTTPS 访问 Core，以及访问发布下载主机；如果 Core 已有安装程序的离线副本，则无需后者； -- 用于环境设置和 MiniMax Code 工具的 Bash；Windows 上需要 Git Bash，Claude Code 也要求它； +- 通过 HTTPS 访问 Core，以及访问发布下载主机；如果 Core 已有安装程序的离线副本，则无需后者。Core 的[公共 URL](../configuration.md#changing-the-public-url) 必须是 https 地址，否则 Core 拒绝该机器的注册； +- 用于环境设置和 MiniMax Code 工具的 Bash； - Session 的软件包需要时，安装 Python 和 pip； - 环境设置所需的系统软件包。守护进程不运行 apt、sudo 或其他提权命令，请通过主机的常规管理方式安装。 -不需要管理员权限或 Docker。Unix 下载命令还使用 `curl`、`tar`、`gzip`、SHA-256 工具和系统文件锁命令（Linux 为 `flock`，macOS 为 `lockf`）。Windows 需要系统自带的 `tar.exe`，命令会在下载前检查。Runtime 主目录必须允许执行文件。如果 Unix 挂载点设为 `noexec`，请先用 `OAC_RUNTIME_HOME` 指定另一个允许执行的绝对目录，再运行命令。 +不需要管理员权限或 Docker。下载命令还使用 `curl`、`tar`、`gzip`、SHA-256 工具和 `flock`。Runtime 主目录必须允许执行文件。如果挂载点设为 `noexec`，请先用 `OAC_RUNTIME_HOME` 指定另一个允许执行的绝对目录，再运行命令。 ## 连接机器 {#connect-a-machine} @@ -53,7 +49,7 @@ Session 的模型提供商与其他 Session 一样解析，因此当 Session 及 }, ) installation = session.model_dump()["x_agents_core"]["installation"] - print(installation["commands"]["posix"]) # use "powershell" on Windows + print(installation["commands"]["posix"]) ``` 2. 在目标机器上，以应运行工具的账号执行命令。命令下载与 Core 匹配的安装程序、验证校验和、询问要安装哪些 Harness 以及安装位置、完成安装、按需创建工作区、启动守护进程并检查连接。 @@ -68,7 +64,7 @@ Session 的模型提供商与其他 Session 一样解析，因此当 Session 及 | 结果 | 含义 | | --- | --- | | **Installation** | 所选 Harness 已通过就绪检查 | -| **Daemon connection** | Core 已确认守护进程的认证连接 | +| **Host connection** | Core 已确认这台机器通过[沙箱 Link](../sandbox-link-protocol.md) 为此 Environment 提供服务 | | **Model configuration** | 未检查；第一个 Turn 使用 Session 的模型提供商 | 临时网络故障会重试下载，最多尝试三次，并在终端展示进度。下载、解压和复制组件前会检查磁盘空间。如果下载或安装被中断，重新执行命令：它清理未完成的临时副本，同时保留已完成的组件、凭据和工作区。下载暂存位于 Runtime 主目录的 `native-download` 中；小型 `download.lock` 文件保留用于并发控制。其他运行不会清理仍在进行的下载或安装。命令过期时，从 Session 复制新命令。 @@ -101,7 +97,7 @@ environment = { } ``` -路径使用机器自身语法的绝对路径（Unix、Windows 驱动器或 UNC）；由守护进程而非 Core 检查。守护进程连接前，先准备这些目录。它们是守护进程可见的普通路径；指定路径不会挂载它或创建沙箱。 +路径为绝对路径；由守护进程而非 Core 检查。守护进程连接前，先准备这些目录。它们是守护进程可见的普通路径；指定路径不会挂载它或创建沙箱。 使用 `x_agents_core.environment` 提供与托管 Session 相同的 Project 所属 Skills、Plugin 归档、文件、软件包、设置命令或 Template： @@ -116,7 +112,7 @@ session = client.beta.agents.sessions.create( ) ``` -同一扩展也支持 `environment={"type": "openai_hosted"}`。不要在 `environment` 和扩展中重复指定同一个字段。设置过程使用守护进程账号权限。部署的模型密钥不会发送到你的机器。 +同一扩展也支持 `environment={"type": "openai_hosted"}`。不要在 `environment` 和扩展中重复指定同一个字段。设置过程使用守护进程账号权限。 第一个 Turn 之前，守护进程将这些来源复制成快照。即使来源之后被修改，重连仍复用快照；新的 Session 获取新快照。[准备协议](../../../contracts/agents-api/zh/environments.md#runtime-capability-preparation)列出字段、合并规则、快照行为和失败情况。 @@ -126,14 +122,14 @@ session = client.beta.agents.sessions.create( | 命令 | 效果 | | --- | --- | -| `oac-daemon start` | 验证已安装的 Harness，并在后台启动守护进程 | +| `oac-daemon start` | 注册机器并在后台启动守护进程。守护进程运行 [Sandbox I/O 服务](../sandbox-bootstrap.md)，服务退出时重新注册并重启它 | | `oac-daemon status` | 展示本地配置与进程，不表示连接状态 | | `oac-daemon logs -n 100`、`oac-daemon logs -f` | 输出或持续跟踪守护进程日志 | | `oac-daemon stop` | 停止守护进程 | 设置了 `OAC_RUNTIME_HOME` 时，每个命令都使用同一值。在 Web 的 Session 页面 **Host connection** 下检查连接，或使用[连接状态](../../../contracts/agents-api/zh/environment-executor-credentials.md#connection-status)。 -添加 Harness 时，以同一安装目录重新执行安装命令，并指定要添加的 Harness（命令过期时从 Web 复制新的）。安装程序检查已有内容、只添加缺失组件，并保留已安装的 Harness。之后重启正在运行的守护进程，让它发现新的 Harness。 +添加 Harness 时，以同一安装目录重新执行安装命令，并指定要添加的 Harness（命令过期时从 Web 复制新的）。安装程序检查已有内容、只添加缺失组件，并保留已安装的 Harness。 停止守护进程、取消 Turn 或删除 Session，都不会删除机器的工作区、原生历史或能力快照。安装程序拒绝其他守护进程版本的安装，以及文件已被修改的安装。程序不升级、修复或迁移它们；请安装到另一个目录。 @@ -164,4 +160,4 @@ session = client.beta.agents.sessions.create( "$HOME/.oac/my-runtime/bin/oac-daemon" start ``` -使用 Session 的 `remote_url` 和 Environment ID。在 PowerShell 中，以原生绝对路径运行 `.\oac-daemon.exe`。此模式要求工作区已存在，且在运行 `start` 前不会启动守护进程。 +使用 Session 的 `remote_url` 和 Environment ID。此模式要求工作区已存在，且在运行 `start` 前不会启动守护进程。 diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index d3338fa61..9246d935d 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: f7e9275feee79ac1fb1299227e148b85c26efc8b8af5ca0e8f498126c563d182 +source_hash: 0aa851ad8d3b6ebcd3d94ad85baad15725f4a7e970b75ae0e6570f58c9dcc271 --- Sandbox Provider 通过交付一个引导文件来启动托管 Runtime。本文负责 Provider 到 Runtime 的启动输入。类型与验证器位于 [`internal/runtimebootstrap`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/runtimebootstrap/bootstrap.go)；Go provider 使用 [`runtime_bootstrap.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/runtime_bootstrap.go) 中的 `sandbox.Bootstrap.RuntimeConnection()` 构造输入，SDK helper 原样转发序列化对象。provider 不读取或写入 Runtime 的私有认证存储。 @@ -31,7 +31,7 @@ provider 创建账户、挂载和工作区，交付该文件，设置 Runtime Runtime 验证输入，并负责认证与连接。启动成功仅证明交付完成：经过认证的连接、已准备的能力和执行就绪是 [Core–Runtime 协议](runtime-protocol.md) 下的独立观测；[Sandbox Provider 指南](sandbox-provider.md#four-distinct-readiness-facts) 列出各自证明的事实。 -自托管 executor 和运维人员供应的设备通过其他方式获取 daemon 身份；[机器连接 API](../../contracts/agents-api/zh/machine-api.md#credentials) 列出所有凭据来源。它们都进入同一 Runtime 执行循环。 +运维人员供应的设备通过其他方式获取 daemon 身份；[机器连接 API](../../contracts/agents-api/zh/machine-api.md#credentials) 列出所有凭据来源。自托管机器不运行 Runtime：它完成注册，并通过 [Sandbox I/O 服务](./sandbox-bootstrap.md)为其 Environment 提供服务。 ## 验证 {#verification} diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index 66104ed95..1af0d4cdd 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: 7adbec57ec29c33da6a85b1204490e54a3bc580b1a37dd871a9f2a8b587ec278 +source_hash: f3da984e5c04ed245950af6bbd51f110f66e8e0ec01f74e9458c21aefa84dc01 --- 此协议在 Runtime daemon 获取机器凭据后连接 Core 与 daemon，定义 daemon 连接上消息的含义和顺序。wire 类型、限制和验证器仅在 [`internal/agentdaemon/proto`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/internal/agentdaemon/proto) 中定义一次；Core 的 [gateway](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/runtimegateway) 与参考 Runtime 的 [dispatcher](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/apps/daemon/internal/dispatch) 都使用它们，因此无需同步第二套 payload schema。签发凭据和打开连接的 HTTP 路由见[机器连接 API](../../contracts/agents-api/zh/machine-api.md)。 @@ -108,7 +108,7 @@ Usage frame 和最终 usage snapshot 都携带当前执行的累计测量，替 ## Session 分配 {#session-assignments} -分配（assignment）把一个 Session 绑定到运行它的 Runtime。`Envelope.assignment` 以 `session_id`、`assignment_id` 和 `epoch` 命名它，这是 frame 携带分配的唯一位置。Core 每次改变分配的期望状态时推进 epoch，因此较低的 epoch 是陈旧的。 +分配（assignment）把一个 Session 绑定到运行它的 Runtime。`Envelope.assignment` 以 `session_id`、`assignment_id` 和 `epoch` 命名它，这是 frame 携带分配的唯一位置。Core 每次改变分配的期望状态时推进 epoch，因此较低的 epoch 是陈旧的。无论 Environment 类型如何，Core 都把每个 Session 绑定到第一个已连接且接纳其 Harness 的 agent host；带 Environment 的 Session 只在该 Environment 的 [Link](./sandbox-link-protocol.md) resource 正在 Serve 时绑定，绑定之后不会转移到其他 Runtime。轮换 `self_hosted` Environment 的[执行器凭证](../../contracts/agents-api/zh/environment-executor-credentials.md)会推进其 enrollment 的 generation，并在同一次写入中推进该 Session 已绑定分配的 epoch。此后 Link 拒绝较早 epoch 的 attachment，因此正在运行的 Turn 失去该 Environment。下一次 bind 携带更高的 epoch 和新的 generation，agent host 在绑定前先结算较早 epoch 的工作，与释放时相同。 每个 Session frame 都携带分配：`execution_prepare`、`execution_start` 和 `execution_release`；`prompt_cancel`、`prompt_steer` 和 `function_result`；`runtime_prepare`、`workspace_read`、`workspace_write` 和 `workspace_export` 的每个 frame；以及 `environment_quiesce` 和 `environment_resume`。回复回显请求的分配，Run 的 frame 携带启动它的分配；Core 拒绝指明其他分配的回复或 Run frame。heartbeat 不携带分配。 diff --git a/docs/zh/sandbox-bootstrap.md b/docs/zh/sandbox-bootstrap.md index 097064d12..fe7654466 100644 --- a/docs/zh/sandbox-bootstrap.md +++ b/docs/zh/sandbox-bootstrap.md @@ -1,7 +1,7 @@ --- title: "沙箱引导" source: docs/sandbox-bootstrap.md -source_hash: 1e2f61c5a7bafae47a72cfd66c9e604a5aa3b7120ca29b11e700241d892c5ada +source_hash: 8b705515313d1d190dac002902748ad9b60d6b24a3ff01cc549fdb70105ea50a --- Sandbox Provider 通过交付一个引导文件来启动 Sandbox I/O 服务。本文负责 Provider 到该服务的启动输入。类型与验证器位于 [`internal/sandboxbootstrap`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/sandboxbootstrap/bootstrap.go)。服务凭此输入以 [沙箱 Link 协议](./sandbox-link-protocol.md)的 serve peer 身份连接 relay。 @@ -38,6 +38,8 @@ oac-sandbox-io --bootstrap-file /home/runtime/sandbox-io-bootstrap.json Provider 创建账户和沙箱，交付该文件，并以该账户启动 `oac-sandbox-io`。`make build-sandbox-io` 构建静态 Linux 二进制。Provider 为进程重启保留该文件，仅在明确清理自己拥有的资源时删除。 +在[自托管机器](./getting-started/self-hosted.md)上，`oac-daemon start` 充当该机器 enrollment 的 Provider。它用执行器凭据注册，并把该文件写入 Runtime 主目录下的 `daemon/sandbox-io-bootstrap.json`，其中包含 Core 返回的 `link_url` 和 `resource`，并以执行器令牌作为 `credential`。它以启动它的账户运行 `oac-sandbox-io`。服务退出时，它重新注册（轮换会推进 generation），然后重写该文件并重启服务。 + 服务验证输入并负责 link：它以 serve peer 身份连接，服务已绑定的 stream，并在凭据有效期间重连。`resource`（包括其 generation）必须是该凭据服务的资源，否则 relay 拒绝该 link。 File 服务只提供一个 [world export](./file-access-protocol.md#attach)，其隔离由 Provider 的沙箱设置负责。[Process 服务](./process-protocol.md#implement-a-service)以服务账户运行进程，服务作为 child subreaper 回收它们的孤儿后代进程。服务还提供 [Network 协议](./sandbox-network-protocol.md)：它在沙箱的网络命名空间中解析名称并建立 TCP 连接，范围限于每个 stream 的 `Bind` 携带的 egress。 diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 231b42e58..3b6a3a09e 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: b6c2a9956d4b4060be3745c1340ece507d03a2b401daa52926555aec84ac9e65 +source_hash: 0f0306c893188c85e6a0c17eb1216a276af074449a80a8eb140fdb70ec0b437a --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -95,8 +95,8 @@ Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity` | 事实 | 证据 | 不证明 | | --- | --- | --- | -| 计算资源可用 | Provider 对所属 allocation 的观测 | 已认证 Runtime 连接或已准备能力 | -| Runtime 已连接 | Gateway 认证和精确 Environment、device 绑定 | preparation 已完成或 Harness 可用 | +| 计算资源可用 | Provider 对所属 allocation 的观测 | 正在 Serve 的 Link resource 或已准备能力 | +| Sandbox 已连接 | Environment 的 live [Link](./sandbox-link-protocol.md) resource 正以当前 generation Serve：relay 持有该 allocation 的 Sandbox I/O 服务的 serve peer；对 `self_hosted`，则是完成 enrollment 的机器上那个服务的 serve peer。这就是每种 Environment 类型的 `connected` 含义 | preparation 已完成或 Harness 可用 | | 能力已准备 | 使用固定配置完成公共 Runtime preparation | Turn 已接受或完成 | | 执行已准入 | 已验证 Harness 能力及 executor、Turn 接受路径 | 输入已完成、取消已完成或计算资源已回收 | diff --git a/docs/zh/web/index.md b/docs/zh/web/index.md index cfe4df338..9f2a76cff 100644 --- a/docs/zh/web/index.md +++ b/docs/zh/web/index.md @@ -1,7 +1,7 @@ --- title: "OpenAgentCore Web" source: docs/web/index.md -source_hash: 375d6245441e9f484af64798fb23c665d0ec712eb9a5088e7a41e3293374680a +source_hash: 6a2501e2a6439aa9e49452d4c7854cc09a34ed0f46b10bad34434fd94cdcf664 --- Web 是单个 OpenAgentCore 部署的管理员控制台。管理员用它查看健康状态、容量、用量和失败情况，检查各 Project 的资源与执行历史，并管理 Project、密钥、节点和部署设置。应用不使用 Web；它们通过自己的 Project API 密钥调用 Core 的 Agents API（`/v1`）。 @@ -44,7 +44,7 @@ Web 是单个 OpenAgentCore 部署的管理员控制台。管理员用它查看 安装不创建 Project 或密钥。打开控制台不分配计算资源，也不调用模型，安装可以没有节点。Web 不启动 Session，也不发送输入。归档托管 Session 会请求取消和回收；[管理员权限](../concepts.md#what-administrators-can-and-cannot-do)说明管理员能执行与不能执行的操作。 -部署的沙箱后端服务托管 Session。应用的 `self_hosted` Runtime（包括它自己 E2B 账号中的 Runtime）是独立路径，修改沙箱配置不会影响它。 +部署的沙箱后端服务托管 Session。应用的 `self_hosted` 机器是独立路径，修改沙箱配置不会影响它。 回环公开地址（`local_only`）使节点和远程应用无法访问 Core。控制台仍可通过自己的地址访问，并[展示警告](console-api-usage.md#provenance-and-monitoring)。 diff --git a/packages/agents-client/src/admin-client.test.ts b/packages/agents-client/src/admin-client.test.ts index a5b93df83..7617e1aa4 100644 --- a/packages/agents-client/src/admin-client.test.ts +++ b/packages/agents-client/src/admin-client.test.ts @@ -138,15 +138,15 @@ describe("AdminClient transport boundary", () => { it("lists executor credential metadata only and revokes with 204", async () => { const credential = { key_id: keyId, created_at: "2026-09-25T00:00:00Z", revoked_at: null }; - const connection = { status: "never_enrolled", bound_key_id: null, enrolled_at: null, last_seen_at: null }; + const connection = { status: "never_enrolled", bound_key_id: null, enrolled_at: null }; expect(await clientWith({ data: [credential], connection }).client.listExecutorCredentials(projectId, resourceId)).toEqual({ data: [credential], connection }); for (const entry of [{ ...credential, executor_token: "leak" }, { ...credential, revoked_at: "later" }, { key_id: keyId, created_at: "2026-09-25T00:00:00Z" }]) { await expect(clientWith({ data: [entry], connection }).client.listExecutorCredentials(projectId, resourceId)).rejects.toMatchObject({ code: "invalid_admin_response" }); } - const enrolled = { status: "connected", bound_key_id: keyId, enrolled_at: credential.created_at, last_seen_at: null }; + const enrolled = { status: "connected", bound_key_id: keyId, enrolled_at: credential.created_at }; expect((await clientWith({ data: [credential], connection: enrolled }).client.listExecutorCredentials(projectId, resourceId)).connection).toEqual(enrolled); for (const bad of [undefined, { ...enrolled, status: "ready" }, { ...enrolled, enrolled_at: null }, { ...enrolled, bound_key_id: null }, - { ...enrolled, last_seen_at: "invalid" }, { ...enrolled, credential_hash: "secret" }, { ...connection, last_seen_at: credential.created_at }]) { + { ...enrolled, enrolled_at: "invalid" }, { ...enrolled, credential_hash: "secret" }, { ...enrolled, last_seen_at: null }, { ...connection, enrolled_at: credential.created_at }]) { await expect(clientWith({ data: [credential], connection: bad }).client.listExecutorCredentials(projectId, resourceId)).rejects.toMatchObject({ code: "invalid_admin_response" }); } const fetch = vi.fn().mockImplementation(async () => new Response(null, { status: 204 })); diff --git a/packages/agents-client/src/admin-projection.ts b/packages/agents-client/src/admin-projection.ts index 28051cae1..497cdfa76 100644 --- a/packages/agents-client/src/admin-projection.ts +++ b/packages/agents-client/src/admin-projection.ts @@ -186,11 +186,11 @@ export function projectAdminAudit(value: unknown): AdminAuditPage { export function projectExecutorCredentials(value: unknown): ExecutorCredentialList { const page = record(value, ["data", "connection"]); if (!Array.isArray(page.data)) return invalidAdminResponse(); - const connection = record(page.connection, ["status", "bound_key_id", "enrolled_at", "last_seen_at"]); + const connection = record(page.connection, ["status", "bound_key_id", "enrolled_at"]); if (!["never_enrolled", "connected", "disconnected"].includes(connection.status as string) || !(connection.bound_key_id === null || (typeof connection.bound_key_id === "string" && connection.bound_key_id.length > 0)) || - !date(connection.enrolled_at) || !date(connection.last_seen_at)) return invalidAdminResponse(); - if (connection.status === "never_enrolled" && [connection.bound_key_id, connection.enrolled_at, connection.last_seen_at].some(value => value !== null)) return invalidAdminResponse(); + !date(connection.enrolled_at)) return invalidAdminResponse(); + if (connection.status === "never_enrolled" && [connection.bound_key_id, connection.enrolled_at].some(value => value !== null)) return invalidAdminResponse(); if (connection.status !== "never_enrolled" && connection.enrolled_at === null) return invalidAdminResponse(); if (connection.status === "connected" && connection.bound_key_id === null) return invalidAdminResponse(); return { connection: { ...connection } as unknown as ExecutorConnection, data: page.data.map((entry) => { diff --git a/packages/agents-client/src/admin-types.ts b/packages/agents-client/src/admin-types.ts index 815592d83..6ecdaeda2 100644 --- a/packages/agents-client/src/admin-types.ts +++ b/packages/agents-client/src/admin-types.ts @@ -147,12 +147,11 @@ export interface AdminAuditPage extends AdminPage { next_cursor /** Executor credential metadata for one self_hosted environment; the credential itself is never listed. */ export interface ExecutorCredential { key_id: string; created_at: string; revoked_at: string | null } -/** Core authority plus a matching live gateway peer; timestamps alone are not readiness. */ +/** Connected while the bound credential has authority and the sandbox it enrolled serves the Environment; `enrolled_at` is not readiness. */ export interface ExecutorConnection { status: "never_enrolled" | "connected" | "disconnected"; bound_key_id: string | null; enrolled_at: string | null; - last_seen_at: string | null; } export interface ExecutorCredentialList { data: ExecutorCredential[]; connection: ExecutorConnection } diff --git a/scripts/build-native-installer-ci.mjs b/scripts/build-native-installer-ci.mjs index ec874a0f6..0180a8b2a 100644 --- a/scripts/build-native-installer-ci.mjs +++ b/scripts/build-native-installer-ci.mjs @@ -29,7 +29,7 @@ async function findCodex(directory, depth = 0) { const candidates = await findCodex(join(root, 'native-tools', 'node_modules', '@openai')); assert.equal(candidates.length, 1, 'Expected exactly one native Codex artifact'); const bundle = join(root, 'native-installer'); -const receipt = await buildBundle({ daemon, node: await realpath(node), codex: candidates[0], claude: join(root, 'claude-runtime'), ...(!windows ? { minimax: join(root, 'minimax-runtime') } : {}), output: bundle }); +const receipt = await buildBundle({ daemon, node: await realpath(node), codex: candidates[0], claude: join(root, 'claude-runtime'), ...(!windows ? { minimax: join(root, 'minimax-runtime') } : {}), ...(process.platform === 'linux' ? { sandboxIo: join(root, 'oac-sandbox-io') } : {}), output: bundle }); console.log(JSON.stringify({ stage: 'bundle', os: receipt.os, arch: receipt.arch, components: Object.keys(receipt.components), model_requests: 0 })); const workspace = join(root, 'native-install-workspace'); await mkdir(workspace); const credential = join(root, 'native-install-credential.json'); diff --git a/scripts/build-native-installer.mjs b/scripts/build-native-installer.mjs index e8cc4a250..6ddf2d9e2 100644 --- a/scripts/build-native-installer.mjs +++ b/scripts/build-native-installer.mjs @@ -150,6 +150,8 @@ export async function buildBundle(options) { const output = await outputRealPath(options.output); // Refuse staging inside a source: recursive copying must not ingest its own output. for (const name of ['node', ...names]) if (inside(await realpath(options[name]), output)) throw new Error('Output must be outside component sources'); + // Only Linux distributions carry oac-sandbox-io, the service a self-hosted machine runs. + if ((process.platform === 'linux') !== Boolean(options.sandboxIo)) throw new Error(options.sandboxIo ? '--sandbox-io is Linux-only' : 'Missing --sandbox-io'); const daemonVersion = probe(options.daemon, ['version'], dirname(options.daemon), process.env); if (!/^[0-9A-Za-z][0-9A-Za-z.+_-]{0,127}$/.test(daemonVersion)) throw new Error('Invalid daemon version'); await mkdir(dirname(output), { recursive: true }); @@ -157,6 +159,7 @@ export async function buildBundle(options) { try { const daemon = join(staging, process.platform === 'win32' ? 'oac-daemon.exe' : 'oac-daemon'); await copyFile(options.daemon, daemon); await chmod(daemon, 0o755); + if (options.sandboxIo) { await copyFile(options.sandboxIo, join(staging, 'oac-sandbox-io')); await chmod(join(staging, 'oac-sandbox-io'), 0o755); } await mkdir(join(staging, 'components')); const components = {}; for (const name of ['node', ...names]) components[name] = { version: pins[name], files: await copyComponent(options[name], join(staging, 'components', name)) }; diff --git a/scripts/build-native-installer.test.mjs b/scripts/build-native-installer.test.mjs index 8ffa55b1f..1f9481413 100644 --- a/scripts/build-native-installer.test.mjs +++ b/scripts/build-native-installer.test.mjs @@ -54,6 +54,13 @@ test('bundle requires a selected harness and rejects output inside sources', asy await assert.rejects(buildBundle({ daemon: join(source, 'daemon'), node: source, codex: source, output: join(source, 'out') }), /outside/); }); +test('bundle carries oac-sandbox-io exactly on Linux', async t => { + const { source, output } = await fixture(t); + const options = { daemon: join(source, 'daemon'), node: source, codex: source, output }; + if (process.platform === 'linux') await assert.rejects(buildBundle(options), /Missing --sandbox-io/); + else await assert.rejects(buildBundle({ ...options, sandboxIo: join(source, 'oac-sandbox-io') }), /Linux-only/); +}); + test('component rejects nonportable paths', { skip: process.platform === 'win32' }, async t => { const { source, output } = await fixture(t); await writeFile(join(source, 'bad:name'), 'bad'); diff --git a/scripts/native-onboarding-smoke.mjs b/scripts/native-onboarding-smoke.mjs index 3837b8fed..2ea34c32a 100644 --- a/scripts/native-onboarding-smoke.mjs +++ b/scripts/native-onboarding-smoke.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Native end-to-end bootstrap against a local transport fixture. No model calls. +// Native end-to-end bootstrap on Linux against a local transport fixture. No model calls. import assert from 'node:assert/strict'; import { createHash, randomUUID } from 'node:crypto'; import { createServer } from 'node:http'; @@ -10,20 +10,19 @@ import { join, resolve } from 'node:path'; const root = process.env.RUNNER_TEMP; if (!root) throw new Error('RUNNER_TEMP is required'); -const windows = process.platform === 'win32'; const bundle = join(root, 'native-installer'); const manifest = JSON.parse(await readFile(join(bundle, 'bundle.json'), 'utf8')); const protocol = (await readFile('internal/agentdaemon/proto/version.go', 'utf8')).match(/const Version = "([^"]+)"/)[1]; -const environment = randomUUID(), session = randomUUID(), device = randomUUID(); +const environment = randomUUID(); +const resource = { tenant_id: randomUUID(), environment_id: environment, kind: 'enrollment', id: randomUUID(), generation: 1 }; const workspace = join(root, 'onboarding-workspace'); const installation = join(root, 'onboarding-installation'); const archive = join(root, 'onboarding.tar.gz'); -const tar = windows ? join(process.env.SystemRoot, 'System32', 'tar.exe') : 'tar'; -execFileSync(tar, ['-czf', archive, '-C', bundle, '.']); +execFileSync('tar', ['-czf', archive, '-C', bundle, '.']); const digest = createHash('sha256'); for await (const chunk of createReadStream(archive)) digest.update(chunk); const checksum = digest.digest('hex'); -let secret, connected = false, connections = 0, failClaim = true; +let secret, links = 0, unexpected = 0, failClaim = true; const authorization = 'fixture-install-authorization'; const server = createServer(async (request, response) => { const url = new URL(request.url, origin); @@ -42,20 +41,22 @@ const server = createServer(async (request, response) => { response.writeHead(204); return response.end(); } if (!secret || request.headers.authorization !== `Bearer ${secret}`) return json(401, {}); - if (url.pathname.endsWith('/enroll')) return json(200, { device_id: device, session_id: session, environment_id: environment, workspace_directory: workspace }); - if (url.pathname.endsWith('/bootstrap')) return json(200, { device_id: device, workspace_id: session, ws_url: remote, heartbeat_seconds: 15, protocol_version: protocol }); - if (url.pathname.endsWith('/connection')) return json(200, { environment_id: environment, status: connected ? 'connected' : 'disconnected' }); + if (url.pathname.endsWith('/enroll')) return json(200, { link_url: origin.replace('http:', 'ws:')+'/api/v1/sandbox-link', resource }); + if (url.pathname.endsWith('/connection')) return json(200, { environment_id: environment, status: sockets.size ? 'connected' : 'disconnected' }); return json(404, {}); }); const sockets = new Set(); +// oac-sandbox-io dials the Sandbox Link. The fixture never answers its Hello, +// so it redials after each handshake deadline; one live link is one service. server.on('upgrade', (request, socket) => { - if (request.headers.authorization !== `Bearer ${secret}`) { socket.destroy(); return; } + if (new URL(request.url, origin).pathname !== '/api/v1/sandbox-link') { unexpected++; socket.destroy(); return; } const accept = createHash('sha1').update(request.headers['sec-websocket-key']+'258EAFA5-E914-47DA-95CA-C5AB0DC85B11').digest('base64'); socket.write(`HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`); - sockets.add(socket); connected = true; connections++; + sockets.add(socket); links = Math.max(links, sockets.size); socket.on('data', () => {}); socket.on('error', () => {}); - socket.on('close', () => { sockets.delete(socket); connected = sockets.size > 0; }); + socket.on('end', () => socket.destroy()); + socket.on('close', () => sockets.delete(socket)); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const origin = `http://127.0.0.1:${server.address().port}`; @@ -72,7 +73,7 @@ function run(executable, args, input = '') { child.on('close', code => { clearTimeout(timer); if (secret) assert.ok(!output.includes(secret), 'Credential leaked into terminal'); resolve({ code, output }); }); }); } -const executable = join(bundle, windows ? 'oac-daemon.exe' : 'oac-daemon'); +const executable = join(bundle, 'oac-daemon'); const args = ['install', '--onboard-url', `${origin}/api/v1/agent-daemon/installation`, '--authorization', authorization, '--install-dir', installation]; try { assert.notEqual((await run(executable, [...args, '--authorization', 'expired', '--non-interactive', '--harness', 'codex'])).code, 0); @@ -80,21 +81,21 @@ try { assert.notEqual((await run(executable, [...args, '--non-interactive', '--harness', 'codex'])).code, 0, 'Lost claim response should fail safely'); const saved = JSON.parse(await readFile(join(installation, 'daemon', 'executor-credential.json'), 'utf8')); assert.equal(saved.executor_token, secret, 'Secret must survive a lost response'); - const script = resolve(`services/core/internal/nativeinstaller/assets/bootstrap.${windows ? 'ps1' : 'sh'}`); - const bootstrapArgs = windows ? ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', script, '-Base', base, '-Authorization', authorization] : [script, base, authorization]; - const result = await run(windows ? 'powershell.exe' : 'bash', [...bootstrapArgs, '--install-dir', installation], '\n\n'); + const script = resolve('services/core/internal/nativeinstaller/assets/bootstrap.sh'); + const result = await run('bash', [script, base, authorization, '--install-dir', installation], '\n\n'); assert.equal(result.code, 0, `Interactive bootstrap failed: ${result.output}`); - assert.match(result.output, /Daemon connection: connected/); + assert.match(result.output, /Host connection: connected/); assert.match(result.output, /Model configuration: not checked/); - assert.equal(connections, 1); + assert.equal(links, 1); const repeat = await run(executable, [...args, '--non-interactive', '--harness', 'codex']); assert.equal(repeat.code, 0, `Repeat failed: ${repeat.output}`); - assert.equal(connections, 1, 'Repeated installation created a duplicate daemon'); + assert.equal(links, 1, 'Repeated installation created a duplicate daemon'); + assert.equal(unexpected, 0, 'The daemon opened a connection other than the Sandbox Link'); console.log(JSON.stringify({ installation: 'passed', connection: 'passed', interactive: 'passed', retry: 'passed', repeat: 'passed', authentication: 'passed', model_requests: 0 })); } finally { // The installed executable resolves its own home unless explicitly overridden. delete process.env.OAC_RUNTIME_HOME; - const installed = join(installation, 'bin', windows ? 'oac-daemon.exe' : 'oac-daemon'); + const installed = join(installation, 'bin', 'oac-daemon'); try { execFileSync(installed, ['stop'], { env: { ...process.env, OAC_RUNTIME_HOME: installation }, stdio: 'ignore', timeout: 15000 }); } catch { /* A failed installation may never have started. */ } for (const socket of sockets) socket.destroy(); await new Promise(resolve => server.close(resolve)); diff --git a/services/core/IMPLEMENTATION.md b/services/core/IMPLEMENTATION.md index 5cf6c8771..eefc3d054 100644 --- a/services/core/IMPLEMENTATION.md +++ b/services/core/IMPLEMENTATION.md @@ -82,19 +82,19 @@ A committed input sends a coalesced hint to the Worker scheduler, which keeps it Core readiness, Start acknowledgement and input-to-first-text logs use one process monotonic clock; a Start acknowledgement confirms adapter ownership, not model input consumption. Runtime logs report Executor creation, reuse, idle and close separately from Turn completion. Never call these durations model latency or subtract clocks of different machines. -The Dispatcher prepares pending Environment input only on its exact enrolled or managed device, and keeps the same physical peer and preparation handle through readiness, atomic promotion and claim, and the first non-replay Start. Workspace and Environment identity come from store ownership, never caller-selected paths. The initial prompt and cursor come from the reserved batch; later messages use ordinary steering. Never hold a database lock during native preparation. While waiting for readiness, observe the original deadline, cancellation, deletion and peer loss. A preparation failure leaves pending input and its deadline intact unless storage settled it, and creates no failed Turn or input history. During Start, consume preparation controls alongside the Run stream so a control-only rejection or a pending-start cancellation settles promptly; once cancellation is sent, preparation errors cannot replace its receipt or timeout path, and a missing or unconfirmed outcome fails conservatively without a fabricated cancellation outcome. The connection owner spans preparation and the transferred Run, and every exit releases it. +The Dispatcher prepares pending Environment input only on the Session's bound agent host, and keeps the same physical peer and preparation handle through readiness, atomic promotion and claim, and the first non-replay Start. Workspace and Environment identity come from store ownership, never caller-selected paths. The initial prompt and cursor come from the reserved batch; later messages use ordinary steering. Never hold a database lock during native preparation. While waiting for readiness, observe the original deadline, cancellation, deletion and peer loss. A preparation failure leaves pending input and its deadline intact unless storage settled it, and creates no failed Turn or input history. During Start, consume preparation controls alongside the Run stream so a control-only rejection or a pending-start cancellation settles promptly; once cancellation is sent, preparation errors cannot replace its receipt or timeout path, and a missing or unconfirmed outcome fails conservatively without a fabricated cancellation outcome. The connection owner spans preparation and the transferred Run, and every exit releases it. -The Worker scans pending inputs with the same scheduling slots, Session locks, durable deadlines and engine capability checks: once a second, at most 100 candidates per scan. At the end of the queue after a nonempty cursor it refills the first page once in the same scan, and an empty queue never spins. The cursor advances before readiness checks so an unavailable Runtime cannot starve later candidates. Execution concurrency (`core.execution_concurrency`) bounds simultaneous work, not attempt frequency, and the Worker alternates between ordinary Turns and Environment inputs. A self-hosted Session waits for its dedicated enrolled device; it never selects another device of the tenant or migrates a binding. Managed lifecycle polling keeps its separate five-second interval. Input HTTP response budgets follow the persisted Environment type, independently of operator switches. +The Worker scans pending inputs with the same scheduling slots, Session locks, durable deadlines and engine capability checks: once a second, at most 100 candidates per scan. At the end of the queue after a nonempty cursor it refills the first page once in the same scan, and an empty queue never spins. The cursor advances before readiness checks so an unavailable Runtime cannot starve later candidates. Execution concurrency (`core.execution_concurrency`) bounds simultaneous work, not attempt frequency, and the Worker alternates between ordinary Turns and Environment inputs. Placement (`execution/worker_device.go`) binds an unbound Session of any Environment type to the first ready agent host, a Session with an Environment only while that Environment's Link resource is Serving, and never migrates a binding. Managed lifecycle polling keeps its separate five-second interval. Input HTTP response budgets follow the persisted Environment type, independently of operator switches. `OAC_HARNESSES` adds deployment-supported engines to the default engine and the managed profiles; a daemon's heartbeat alone never enables an engine. ## Runtime connections -`services/core/internal/runtimegateway` is Core's daemon connection implementation; its persistence interfaces use `services/core/internal/runtimedevice`, and the frames and validators live in the shared `internal/agentdaemon/proto`. It is a single-process registry: connectivity comes from the live registry, never a persisted online flag, and `last_seen_at` is diagnostic only. Session-to-device bindings are tenant-scoped and immutable. Revocation denies new connections and binding reads at once, and an open connection closes at its next heartbeat. +`services/core/internal/runtimegateway` is Core's daemon connection implementation; its persistence interfaces use `services/core/internal/runtimedevice`, and the frames and validators live in the shared `internal/agentdaemon/proto`. It is a single-process registry: connectivity comes from the live registry, never a persisted online flag, and `last_seen_at` is diagnostic only. A Session binds only to an agent host of no tenant or of its own tenant, and the binding is immutable; for a Session with an Environment, `BindSessionDevice` also requires a live `sandbox_resources` row under the Session lock. Revocation denies new connections and binding reads at once, and an open connection closes at its next heartbeat. -`runtimegateway.LinkAuthority` is Core's [Link](../../docs/sandbox-link-protocol.md) `Authority`. `cmd/server` builds it over `sessionpg.Store` and gives it to one `relay.New`, which the API serves at `/api/v1/sandbox-link`; the Worker revokes through that relay as `execution.Dispatcher.Links`. Every Hello, Open and renewal rereads the database. A Serve credential authenticates only its own resource while the `sandbox_resources` view marks it live, at that resource's current generation: an allocation in `creating` or `running` by its `serve_credential_hash`, or a `sandbox_enrollments` row by its executor key while the key would still authenticate for the Environment. Rotating the key advances the generation of each of its enrollments in the same transaction, so Opens and renewals for the old generation are refused and its attachments end within one lease. A key without an Environment restriction may hold several enrollments, and its holder is trusted for every Environment the key authenticates for: it may Serve any of them, replacing that enrollment's serve peer. Only a device marked `agent_host` Attaches, and its `credential_revision` is the peer's `Revision`. The deployment's agent host has no tenant: `cmd/server` registers it at startup from `OAC_AGENT_HOST_IDENTITY_FILE`, which advances the revision only for a new credential and never lifts a revocation. An attach grant is the assignment ID, epoch and resource generation followed by their keyed digest under the credential key, so Core stores none. It opens a service only while its assignment is the Session's bound assignment at that epoch, held by the peer's Runtime, and its generation is current. File gets the `world` export, Network gets every destination while the Session's network access is enabled and none otherwise, and each lease lasts one minute. Cleanup of an allocation and a release first commit, which withdraws their authority, then revoke the resource at the relay, and only then destroy the compute or send the release. +`runtimegateway.LinkAuthority` is Core's [Link](../../docs/sandbox-link-protocol.md) `Authority`. `cmd/server` builds it over `sessionpg.Store` and gives it to one `relay.New`, which the API serves at `/api/v1/sandbox-link`; the Worker revokes through that relay as `execution.Dispatcher.Links`. Every Hello, Open and renewal rereads the database. A Serve credential authenticates only its own resource while the `sandbox_resources` view marks it live, at that resource's current generation: an allocation in `creating` or `running` by its `serve_credential_hash`, or a `sandbox_enrollments` row by its executor key while the key would still authenticate for the Environment. Rotating the key advances the generation of each of its enrollments and the epoch of their Sessions' bound assignments in the same statement, so Opens and renewals for the old generation are refused, its attachments end within one lease and the next bind supersedes the old epoch. A key without an Environment restriction may hold several enrollments, and its holder is trusted for every Environment the key authenticates for: it may Serve any of them, replacing that enrollment's serve peer. Only a device marked `agent_host` Attaches, and its `credential_revision` is the peer's `Revision`. The deployment's agent host has no tenant: `cmd/server` registers it at startup from `OAC_AGENT_HOST_IDENTITY_FILE`, which advances the revision only for a new credential and never lifts a revocation. An attach grant is the assignment ID, epoch and resource generation followed by their keyed digest under the credential key, so Core stores none. It opens a service only while its assignment is the Session's bound assignment at that epoch, held by the peer's Runtime, and its generation is current. File gets the `world` export, Network gets every destination while the Session's network access is enabled and none otherwise, and each lease lasts one minute. Cleanup of an allocation and a release first commit, which withdraws their authority, then revoke the resource at the relay, and only then destroy the compute or send the release. Every other end of Serve authority, such as a credential revocation or rotation, an Environment expiry or a Session deletion, reaches the relay through the Worker's connection pass: it keeps each live resource's last generation in memory and revokes only on a transition, the previous generation when the generation advances and the last one when the resource leaves the view, because each revocation advances the relay's epoch. The relay and that memory are process-local, so a restarted Core starts both empty. -A dedicated self-hosted device is bound to exactly one Environment's Session and is excluded from general device selection, even within the tenant. Enrollment creates or recovers the device and binding atomically under the Session lock; the frozen workspace and capability directories come from the Session configuration and must match the local binding. Core rechecks the persisted Environment and device binding for preparation and active reads; capability discovery never selects or authorizes a device for this placement. +A self-hosted machine enrolls as its Environment's Link resource: enrollment authorizes the executor key and, under the Session lock, inserts the `sandbox_enrollments` row, where the first key wins; it creates no device and binds nothing. Its connection status is that resource Serving while the enrolled key keeps its authority (`runtimeenrollment.RuntimeConnected`), the same rule as a hosted Environment's. Connection observations use the execution lease and the Session lock. A separate `environment_connections` row holds the current generation and revision, and `environments.status` commits together with its Session Environment event. The producer serializes replacements and numbers socket observations within each generation; duplicate or older revisions and superseded generations are inert, and a replacement retires the previous connected observation before registering the new one. Registration alone creates no `connected` event. Event payloads carry only public Environment identity, type, status and nullable error, never configuration, credentials, registration IDs or revisions, and have no Turn association. `connected` and `disconnected` are distinct from native readiness: never cast resource `expired` into this vocabulary or emit `ready` for a self-hosted connection. On restart the Worker reconciles old observations before admitting new ones, and a failed or stale observation never establishes a connection. diff --git a/services/core/cmd/server/executor_connections.go b/services/core/cmd/server/executor_connections.go index 45e480e7d..3a0b12b7c 100644 --- a/services/core/cmd/server/executor_connections.go +++ b/services/core/cmd/server/executor_connections.go @@ -3,17 +3,17 @@ package main import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/relay" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeenrollment" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// executorConnections observes enrolled executors through the Runtime gateway. +// executorConnections observes enrolled sandboxes through the Link relay. type executorConnections struct { sessions sessions.Reader - registry *runtimegateway.Registry + links *relay.Relay } func (c executorConnections) ExecutorConnected(ctx context.Context, environment, digest string) (bool, error) { - return runtimeenrollment.RuntimeConnected(ctx, c.sessions, c.registry, environment, digest) + return runtimeenrollment.RuntimeConnected(ctx, c.sessions, c.links, environment, digest) } diff --git a/services/core/cmd/server/main.go b/services/core/cmd/server/main.go index db2bfe025..aa4e5899f 100644 --- a/services/core/cmd/server/main.go +++ b/services/core/cmd/server/main.go @@ -324,7 +324,7 @@ func run(config processconfig.Config) error { Artifacts: sessionService, ArtifactsReader: sessionStore, SessionAdmin: sessionStore, - Environments: sessionService, EnvironmentsReader: sessionStore, ExecutorConnections: executorConnections{sessions: sessionStore, registry: registry}, + Environments: sessionService, EnvironmentsReader: sessionStore, ExecutorConnections: executorConnections{sessions: sessionStore, links: linkRelay}, Admin: sessionStore, AdminAudit: auditStore, WriteAudit: auditStore, Metrics: metrics, RuntimeObservations: observationService, RuntimeHistory: historyService, Execution: api.Execution{ @@ -349,8 +349,8 @@ func run(config processconfig.Config) error { return err } handler := serverHandler(apiHandler, &daemonRoutes{gateway: daemonHandler, - enrollment: runtimeenrollment.EnrollmentHandler(sessionService), - connection: runtimeenrollment.ConnectionHandler(sessionStore, registry), + enrollment: runtimeenrollment.EnrollmentHandler(sessionService, config.PublicOrigin), + connection: runtimeenrollment.ConnectionHandler(sessionStore, linkRelay), nodeConnect: managedNodes.hub}) server := &http.Server{Addr: config.Addr, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 60 * time.Second} done := make(chan error, 1) diff --git a/services/core/internal/api/environment_executor_management.go b/services/core/internal/api/environment_executor_management.go index 5bbb1ee57..ed3c7329b 100644 --- a/services/core/internal/api/environment_executor_management.go +++ b/services/core/internal/api/environment_executor_management.go @@ -13,10 +13,10 @@ import ( "github.com/google/uuid" ) -// ExecutorConnections observes current executor authority and its actual -// gateway peer. The observer runs after the Environments snapshot closes and -// must recheck authority after inspecting the peer. Without a gateway peer, no -// binding is connected. +// ExecutorConnections observes current executor authority and the enrolled +// sandbox's serve peer at the Link relay. The observer runs after the +// Environments snapshot closes and must recheck authority after inspecting +// the relay. Without a serve peer, no enrollment is connected. type ExecutorConnections interface { ExecutorConnected(ctx context.Context, environmentID, credentialDigest string) (bool, error) } @@ -32,13 +32,12 @@ type ExecutorCredentialList struct { Connection ExecutorConnection `json:"connection" binding:"required"` } -// ExecutorConnection reports binding history and current Core-observed connectivity. -// Heartbeat times are observations, not execution or native readiness. +// ExecutorConnection reports the Environment's enrollment and whether its +// sandbox serves the Environment now. type ExecutorConnection struct { Status string `json:"status" binding:"required" enums:"never_enrolled,connected,disconnected"` BoundKeyID *string `json:"bound_key_id" binding:"required" extensions:"x-nullable" format:"uuid"` EnrolledAt *time.Time `json:"enrolled_at" binding:"required" format:"date-time" extensions:"x-nullable"` - LastSeenAt *time.Time `json:"last_seen_at" binding:"required" format:"date-time" extensions:"x-nullable"` } // registerExecutorCredentialRoutes adds executor credential issuance to the @@ -52,7 +51,7 @@ func (h *Handler) registerExecutorCredentialRoutes(r chi.Router) { } // @Summary List a self_hosted Environment's executor credentials -// @Description Core key only. Returns metadata of the credentials restricted to this Environment, oldest first; secrets are never listed. Connection combines current credential authority and an open matching gateway peer; timestamps are historical observations, not readiness. Without a gateway it is never connected. The Environment must be a self_hosted Environment of the Project whose Session exists; otherwise 404. +// @Description Core key only. Returns metadata of the credentials restricted to this Environment, oldest first; secrets are never listed. Connection is connected while the enrolled credential has authority and the sandbox it enrolled serves the Environment's Link resource; enrolled_at is when the Environment was first enrolled, not readiness. The Environment must be a self_hosted Environment of the Project whose Session exists; otherwise 404. // @Tags Executor Credentials // @Produce json // @Security DeploymentAdminAuth @@ -73,9 +72,9 @@ func (h *Handler) listExecutorCredentials(w http.ResponseWriter, r *http.Request } connection := ExecutorConnection{Status: "never_enrolled"} observed := state.Connection - if observed.DeviceID != "" { - connection = ExecutorConnection{Status: "disconnected", BoundKeyID: observed.BoundKeyID, EnrolledAt: observed.EnrolledAt, LastSeenAt: observed.LastSeenAt} - if observed.EnvironmentStatus == "connected" && observed.CredentialHash != "" { + if observed.Enrolled { + connection = ExecutorConnection{Status: "disconnected", BoundKeyID: observed.BoundKeyID, EnrolledAt: observed.EnrolledAt} + if observed.CredentialHash != "" { connected, err := h.ExecutorConnections.ExecutorConnected(r.Context(), state.EnvironmentID, observed.CredentialHash) if err != nil && !errors.Is(err, sessions.ErrNotFound) && !errors.Is(err, sessions.ErrDeviceBindingConflict) { writeSessionsError(w, r, err) diff --git a/services/core/internal/api/environment_executor_management_test.go b/services/core/internal/api/environment_executor_management_test.go index 252a5bc83..dc8b65de7 100644 --- a/services/core/internal/api/environment_executor_management_test.go +++ b/services/core/internal/api/environment_executor_management_test.go @@ -93,7 +93,7 @@ func TestProjectExecutorCredentialsHTTP(t *testing.T) { } w := projectKeyHTTP(h, "GET", path, "admin", "") - if w.Code != 200 || w.Body.String() != `{"data":[{"key_id":"listed","created_at":"1970-01-01T00:00:01Z","revoked_at":null}],"connection":{"status":"never_enrolled","bound_key_id":null,"enrolled_at":null,"last_seen_at":null}}`+"\n" { + if w.Code != 200 || w.Body.String() != `{"data":[{"key_id":"listed","created_at":"1970-01-01T00:00:01Z","revoked_at":null}],"connection":{"status":"never_enrolled","bound_key_id":null,"enrolled_at":null}}`+"\n" { t.Fatal("list", w.Code, w.Body) } w = projectKeyHTTP(h, "POST", path, "admin", body) @@ -156,7 +156,7 @@ func TestExecutorConnectionListObservation(t *testing.T) { key := callerBinding() at := time.Unix(1, 0).UTC() bound := "bound-key" - f := &executorManagementFixture{connection: sessions.ExecutorConnectionState{DeviceID: "device", BoundKeyID: &bound, EnrolledAt: &at, CredentialHash: "private-digest", EnvironmentStatus: "connected"}} + f := &executorManagementFixture{connection: sessions.ExecutorConnectionState{Enrolled: true, BoundKeyID: &bound, EnrolledAt: &at, CredentialHash: "private-digest"}} h := executorManagementHandler(t, key, f, func(_ context.Context, environment, digest string) (bool, error) { if environment != "environment" || digest != "private-digest" { t.Fatal("wrong binding") @@ -185,7 +185,7 @@ func TestExecutorConnectionListUsesResolvedEnvironment(t *testing.T) { key := callerBinding() f := &executorManagementFixture{ resolvedEnvironment: canonical, - connection: sessions.ExecutorConnectionState{DeviceID: "device", CredentialHash: "private-digest", EnvironmentStatus: "connected"}, + connection: sessions.ExecutorConnectionState{Enrolled: true, CredentialHash: "private-digest"}, } observations := 0 h := executorManagementHandler(t, key, f, func(_ context.Context, environment, digest string) (bool, error) { diff --git a/services/core/internal/api/session_model_defaults.go b/services/core/internal/api/session_model_defaults.go index da589f4a8..07ebf0398 100644 --- a/services/core/internal/api/session_model_defaults.go +++ b/services/core/internal/api/session_model_defaults.go @@ -66,12 +66,6 @@ func (h *Handler) resolveSessionExecution(ctx context.Context, input sessionRequ provider, source = extension.ModelProvider, v1.ModelProviderSourceSession } } - // The guest daemon of a self_hosted Environment runs on the caller's - // machine, so a deployment key must not reach it until the Harness runs on - // an agent host. - if provider == nil && input.Environment.Type == "self_hosted" { - return "", nil, "", uuid.Nil, &modelProviderRequiredError{"self_hosted Sessions need a model provider for harness " + engine + ": pass x_agents_core.model_provider or use an Agent that has one saved."} - } if provider == nil && input.deploymentDefaults != nil { provider, source, revision = input.deploymentDefaults.Provider, v1.ModelProviderSourceDeployment, input.deploymentDefaults.Revision } diff --git a/services/core/internal/db/queries/devices.sql b/services/core/internal/db/queries/devices.sql index 78d99e1d0..203e2ac37 100644 --- a/services/core/internal/db/queries/devices.sql +++ b/services/core/internal/db/queries/devices.sql @@ -2,8 +2,11 @@ INSERT INTO devices (id, tenant_id, name, credential_hash) VALUES ($1, $2, $3, $4) RETURNING id; --- name: GetDevice :one -SELECT id, name FROM devices WHERE tenant_id = $1 AND id = $2 AND revoked_at IS NULL; +-- name: GetAgentHost :one +-- An agent host that may run the tenant's Sessions: the deployment's own or +-- one of the tenant's. +SELECT id, name FROM devices +WHERE id = $2 AND agent_host AND revoked_at IS NULL AND (tenant_id IS NULL OR tenant_id = $1); -- name: GetDeviceCredential :one SELECT d.id, d.name, d.credential_hash, COALESCE(a.node_id::text, '')::text AS runtime_node_id, COALESCE(a.id::text, '')::text AS runtime_allocation_id @@ -29,11 +32,15 @@ UPDATE devices SET last_seen_at = clock_timestamp() WHERE devices.id = $1 AND EXISTS (SELECT 1 FROM runtime_device_authority a WHERE a.id = devices.id); -- name: BindSessionDevice :one +-- Binds the Session to an agent host. A Session with an Environment binds +-- only while its Environment has a live Link resource, through which the +-- agent host reaches the sandbox. INSERT INTO session_runtime_assignments (session_id, runtime_id) -SELECT s.id, d.id FROM sessions s JOIN devices d ON d.tenant_id = s.tenant_id +SELECT s.id, d.id FROM sessions s JOIN devices d ON d.agent_host AND (d.tenant_id IS NULL OR d.tenant_id = s.tenant_id) WHERE s.tenant_id = $1 AND s.id = $2 AND d.id = $3 AND d.revoked_at IS NULL -AND (d.environment_id IS NULL OR EXISTS ( - SELECT 1 FROM environments e WHERE e.id = d.environment_id AND e.session_id = s.id +AND (NOT EXISTS (SELECT 1 FROM environments e WHERE e.session_id = s.id) OR EXISTS ( + SELECT 1 FROM environments e JOIN sandbox_resources r ON r.environment_id = e.id + WHERE e.session_id = s.id AND r.live )) ON CONFLICT (session_id) DO UPDATE SET runtime_id = session_runtime_assignments.runtime_id WHERE session_runtime_assignments.runtime_id = EXCLUDED.runtime_id AND session_runtime_assignments.desired_state = 'bound' @@ -41,27 +48,24 @@ RETURNING runtime_id; -- name: GetSessionDevice :one -- The Session's bound Runtime: a device of its tenant or the deployment's --- agent host. environment_id is the device's own Environment and --- session_environment_id the Session's, which the assignment binds. -SELECT d.id, d.name, d.environment_id, e.id AS session_environment_id, b.assignment_id, b.epoch FROM session_runtime_assignments b +-- agent host, with the Session's Environment, which the assignment binds. +SELECT d.id, d.name, e.id AS session_environment_id, b.assignment_id, b.epoch FROM session_runtime_assignments b JOIN sessions s ON s.id = b.session_id JOIN devices d ON d.id = b.runtime_id AND (d.tenant_id = s.tenant_id OR (d.agent_host AND d.tenant_id IS NULL)) LEFT JOIN environments e ON e.session_id = s.id WHERE s.tenant_id = $1 AND s.id = $2 AND d.revoked_at IS NULL AND b.desired_state = 'bound' -AND EXISTS (SELECT 1 FROM runtime_device_authority a WHERE a.id = d.id) -AND (d.environment_id IS NULL OR d.environment_id = e.id); +AND EXISTS (SELECT 1 FROM runtime_device_authority a WHERE a.id = d.id); -- name: GetSessionExecutionBinding :one -- The bound Runtime as GetSessionDevice reads it, with the native session. -SELECT d.id, d.name, b.native_session_id, d.environment_id, e.id AS session_environment_id, b.assignment_id, b.epoch, +SELECT d.id, d.name, b.native_session_id, e.id AS session_environment_id, b.assignment_id, b.epoch, EXISTS (SELECT 1 FROM turns t WHERE t.session_id = s.id AND t.started_at IS NOT NULL) AS has_started_turn FROM session_runtime_assignments b JOIN sessions s ON s.id = b.session_id JOIN devices d ON d.id = b.runtime_id AND (d.tenant_id = s.tenant_id OR (d.agent_host AND d.tenant_id IS NULL)) LEFT JOIN environments e ON e.session_id = s.id WHERE s.tenant_id = $1 AND s.id = $2 AND d.revoked_at IS NULL AND b.desired_state = 'bound' -AND EXISTS (SELECT 1 FROM runtime_device_authority a WHERE a.id = d.id) -AND (d.environment_id IS NULL OR d.environment_id = e.id); +AND EXISTS (SELECT 1 FROM runtime_device_authority a WHERE a.id = d.id); -- name: RememberNativeSession :execrows UPDATE session_runtime_assignments SET native_session_id = $2 WHERE session_id = $1; diff --git a/services/core/internal/db/queries/environment_executor_credentials.sql b/services/core/internal/db/queries/environment_executor_credentials.sql index a8b4e484f..abcde8349 100644 --- a/services/core/internal/db/queries/environment_executor_credentials.sql +++ b/services/core/internal/db/queries/environment_executor_credentials.sql @@ -31,7 +31,9 @@ WHERE c.key_id = sqlc.arg(key_id) AND c.tenant_id = sqlc.arg(tenant_id) -- name: RotateExecutorCredential :one -- Rotation advances the generation of the key's enrollments, so the Link --- authority refuses what the old secret served. +-- authority refuses what the old secret served, and the epoch of the bound +-- assignments of their Sessions, so the next Bind carries the new +-- generation. WITH rotated AS ( UPDATE environment_executor_credentials SET token_sha256 = sqlc.arg(token_sha256), issued_at = clock_timestamp(), revoked_at = NULL @@ -41,6 +43,11 @@ WITH rotated AS ( ), advanced AS ( UPDATE sandbox_enrollments n SET generation = n.generation + 1 FROM rotated r WHERE n.executor_key_id = r.key_id + RETURNING n.environment_id +), rebound AS ( + UPDATE session_runtime_assignments b SET epoch = b.epoch + 1 + FROM advanced a JOIN environments e ON e.id = a.environment_id + WHERE b.session_id = e.session_id AND b.desired_state = 'bound' ) SELECT key_id, environment_id FROM rotated; @@ -67,11 +74,12 @@ WHERE tenant_id = sqlc.arg(tenant_id) AND environment_id = sqlc.arg(environment_ ORDER BY created_at, key_id; -- name: GetEnvironmentExecutorConnection :one -SELECT d.id AS device_id, d.executor_key_id, d.created_at AS enrolled_at, - d.last_seen_at, a.credential_hash, e.status AS environment_status +-- The Environment's enrollment, with the credential that may Serve it while +-- its Link resource is live. +SELECT n.id AS enrollment_id, n.executor_key_id, n.created_at AS enrolled_at, r.credential_hash FROM environments e JOIN sessions s ON s.id = e.session_id -LEFT JOIN devices d ON d.environment_id = e.id AND d.tenant_id = s.tenant_id -LEFT JOIN runtime_device_authority a ON a.id = d.id +LEFT JOIN sandbox_enrollments n ON n.environment_id = e.id +LEFT JOIN sandbox_resources r ON r.kind = 'enrollment' AND r.id = n.id AND r.live WHERE e.id = sqlc.arg(environment_id) AND s.tenant_id = sqlc.arg(tenant_id) AND s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'self_hosted'; diff --git a/services/core/internal/db/queries/runtime_enrollment.sql b/services/core/internal/db/queries/runtime_enrollment.sql index 9fadd340c..0f79c31e9 100644 --- a/services/core/internal/db/queries/runtime_enrollment.sql +++ b/services/core/internal/db/queries/runtime_enrollment.sql @@ -1,5 +1,5 @@ -- name: AuthorizeRuntimeEnrollment :one -SELECT c.key_id, e.session_id, COALESCE(s.configuration->'environment'->>'workspace_directory', '')::text AS workspace_directory +SELECT c.key_id FROM environment_executor_credentials c JOIN environments e ON e.id = sqlc.arg(environment_id) JOIN sessions s ON s.id = e.session_id @@ -12,12 +12,14 @@ WHERE c.token_sha256 = sqlc.arg(token_sha256) AND c.revoked_at IS NULL AND s.configuration->'environment'->>'type' = 'self_hosted' FOR SHARE OF c; --- name: EnrollRuntimeDevice :one -INSERT INTO devices (id, tenant_id, name, environment_id, executor_key_id) -VALUES (sqlc.arg(id), sqlc.arg(tenant_id), 'User-managed Runtime', sqlc.arg(environment_id), sqlc.arg(executor_key_id)) -ON CONFLICT (environment_id) DO UPDATE SET name = devices.name -WHERE devices.executor_key_id = EXCLUDED.executor_key_id AND devices.revoked_at IS NULL -RETURNING id, name, environment_id; +-- name: EnrollSandbox :one +-- The first key to enroll the Environment keeps its enrollment; the same key +-- enrolling again returns it, and another key conflicts. +INSERT INTO sandbox_enrollments (id, environment_id, executor_key_id) +VALUES (sqlc.arg(id), sqlc.arg(environment_id), sqlc.arg(executor_key_id)) +ON CONFLICT (environment_id) DO UPDATE SET executor_key_id = EXCLUDED.executor_key_id +WHERE sandbox_enrollments.executor_key_id = EXCLUDED.executor_key_id +RETURNING id, generation; -- name: TouchAuthenticatedDevice :execrows UPDATE devices SET last_seen_at = clock_timestamp() @@ -25,12 +27,3 @@ WHERE devices.id = sqlc.arg(id) AND EXISTS ( SELECT 1 FROM runtime_device_authority a WHERE a.id = devices.id AND a.credential_hash = sqlc.arg(credential_hash) ); - --- name: ListEnrolledRuntimeBindings :many -SELECT d.id AS device_id, d.tenant_id, e.id AS environment_id, e.session_id -FROM devices d -JOIN environments e ON e.id = d.environment_id -JOIN sessions s ON s.id = e.session_id AND s.tenant_id = d.tenant_id -WHERE d.executor_key_id IS NOT NULL AND s.deleted_at IS NULL - AND e.status NOT IN ('failed', 'expired') -ORDER BY d.id; diff --git a/services/core/internal/db/queries/sandbox_link.sql b/services/core/internal/db/queries/sandbox_link.sql index 39ae37b94..14f52a9df 100644 --- a/services/core/internal/db/queries/sandbox_link.sql +++ b/services/core/internal/db/queries/sandbox_link.sql @@ -11,6 +11,12 @@ FROM sandbox_resources r LEFT JOIN runtime_allocations a ON r.kind = 'allocation' AND a.id = r.id WHERE r.live; +-- name: GetEnvironmentResource :one +-- The Environment's live Link resource. An Environment has at most one Link +-- resource: its allocation's or its enrollment's. +SELECT kind, id, generation, credential_hash FROM sandbox_resources +WHERE tenant_id = $1 AND environment_id = $2 AND live; + -- name: GetAgentHostCredential :one SELECT COALESCE(credential_hash, '')::text AS credential_hash, credential_revision FROM devices WHERE id = $1 AND agent_host AND revoked_at IS NULL; diff --git a/services/core/internal/db/queries/scheduling.sql b/services/core/internal/db/queries/scheduling.sql index 7dac84e27..2539433a0 100644 --- a/services/core/internal/db/queries/scheduling.sql +++ b/services/core/internal/db/queries/scheduling.sql @@ -7,8 +7,8 @@ FROM turns t JOIN sessions s ON s.id = t.session_id WHERE t.status = ANY(sqlc.arg(statuses)::text[]) AND t.id > sqlc.arg(after_id)::uuid AND (s.deleted_at IS NULL OR t.status <> 'queued') AND (NOT sqlc.arg(connected_only)::boolean OR EXISTS ( - SELECT 1 FROM devices d WHERE d.tenant_id = s.tenant_id AND d.revoked_at IS NULL - AND d.id = ANY(sqlc.arg(connected_devices)::uuid[]) + SELECT 1 FROM devices d WHERE d.agent_host AND (d.tenant_id IS NULL OR d.tenant_id = s.tenant_id) + AND d.revoked_at IS NULL AND d.id = ANY(sqlc.arg(connected_devices)::uuid[]) )) ORDER BY t.id LIMIT 100; @@ -20,15 +20,16 @@ LEFT JOIN session_runtime_assignments b ON b.session_id = s.id WHERE r.state = 'pending' AND r.deadline > clock_timestamp() AND r.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND EXISTS ( - SELECT 1 FROM devices d WHERE d.tenant_id = s.tenant_id AND d.revoked_at IS NULL - AND d.id = ANY(sqlc.arg(connected_devices)::uuid[]) + SELECT 1 FROM devices d WHERE d.agent_host AND (d.tenant_id IS NULL OR d.tenant_id = s.tenant_id) + AND d.revoked_at IS NULL AND d.id = ANY(sqlc.arg(connected_devices)::uuid[]) AND (b.runtime_id IS NULL OR d.id = b.runtime_id) ) ORDER BY r.id LIMIT 100; --- name: ListExecutionDevices :many +-- name: ListAgentHosts :many +-- The agent hosts that may run the tenant's Sessions; see GetAgentHost. SELECT id, name FROM devices -WHERE tenant_id = $1 AND revoked_at IS NULL AND environment_id IS NULL +WHERE agent_host AND revoked_at IS NULL AND (tenant_id IS NULL OR tenant_id = $1) ORDER BY id; -- name: GetLatestSessionTurn :one diff --git a/services/core/internal/db/sqlc/devices.sql.go b/services/core/internal/db/sqlc/devices.sql.go index dcdd2e9b0..df0bffa03 100644 --- a/services/core/internal/db/sqlc/devices.sql.go +++ b/services/core/internal/db/sqlc/devices.sql.go @@ -32,10 +32,11 @@ func (q *Queries) AcknowledgeAssignmentRelease(ctx context.Context, arg Acknowle const bindSessionDevice = `-- name: BindSessionDevice :one INSERT INTO session_runtime_assignments (session_id, runtime_id) -SELECT s.id, d.id FROM sessions s JOIN devices d ON d.tenant_id = s.tenant_id +SELECT s.id, d.id FROM sessions s JOIN devices d ON d.agent_host AND (d.tenant_id IS NULL OR d.tenant_id = s.tenant_id) WHERE s.tenant_id = $1 AND s.id = $2 AND d.id = $3 AND d.revoked_at IS NULL -AND (d.environment_id IS NULL OR EXISTS ( - SELECT 1 FROM environments e WHERE e.id = d.environment_id AND e.session_id = s.id +AND (NOT EXISTS (SELECT 1 FROM environments e WHERE e.session_id = s.id) OR EXISTS ( + SELECT 1 FROM environments e JOIN sandbox_resources r ON r.environment_id = e.id + WHERE e.session_id = s.id AND r.live )) ON CONFLICT (session_id) DO UPDATE SET runtime_id = session_runtime_assignments.runtime_id WHERE session_runtime_assignments.runtime_id = EXCLUDED.runtime_id AND session_runtime_assignments.desired_state = 'bound' @@ -48,6 +49,9 @@ type BindSessionDeviceParams struct { ID_2 pgtype.UUID `json:"id_2"` } +// Binds the Session to an agent host. A Session with an Environment binds +// only while its Environment has a live Link resource, through which the +// agent host reaches the sandbox. func (q *Queries) BindSessionDevice(ctx context.Context, arg BindSessionDeviceParams) (pgtype.UUID, error) { row := q.db.QueryRow(ctx, bindSessionDevice, arg.TenantID, arg.ID, arg.ID_2) var runtime_id pgtype.UUID @@ -79,23 +83,26 @@ func (q *Queries) CreateDevice(ctx context.Context, arg CreateDeviceParams) (pgt return id, err } -const getDevice = `-- name: GetDevice :one -SELECT id, name FROM devices WHERE tenant_id = $1 AND id = $2 AND revoked_at IS NULL +const getAgentHost = `-- name: GetAgentHost :one +SELECT id, name FROM devices +WHERE id = $2 AND agent_host AND revoked_at IS NULL AND (tenant_id IS NULL OR tenant_id = $1) ` -type GetDeviceParams struct { +type GetAgentHostParams struct { TenantID pgtype.UUID `json:"tenant_id"` ID pgtype.UUID `json:"id"` } -type GetDeviceRow struct { +type GetAgentHostRow struct { ID pgtype.UUID `json:"id"` Name string `json:"name"` } -func (q *Queries) GetDevice(ctx context.Context, arg GetDeviceParams) (GetDeviceRow, error) { - row := q.db.QueryRow(ctx, getDevice, arg.TenantID, arg.ID) - var i GetDeviceRow +// An agent host that may run the tenant's Sessions: the deployment's own or +// one of the tenant's. +func (q *Queries) GetAgentHost(ctx context.Context, arg GetAgentHostParams) (GetAgentHostRow, error) { + row := q.db.QueryRow(ctx, getAgentHost, arg.TenantID, arg.ID) + var i GetAgentHostRow err := row.Scan(&i.ID, &i.Name) return i, err } @@ -129,13 +136,12 @@ func (q *Queries) GetDeviceCredential(ctx context.Context, id pgtype.UUID) (GetD } const getSessionDevice = `-- name: GetSessionDevice :one -SELECT d.id, d.name, d.environment_id, e.id AS session_environment_id, b.assignment_id, b.epoch FROM session_runtime_assignments b +SELECT d.id, d.name, e.id AS session_environment_id, b.assignment_id, b.epoch FROM session_runtime_assignments b JOIN sessions s ON s.id = b.session_id JOIN devices d ON d.id = b.runtime_id AND (d.tenant_id = s.tenant_id OR (d.agent_host AND d.tenant_id IS NULL)) LEFT JOIN environments e ON e.session_id = s.id WHERE s.tenant_id = $1 AND s.id = $2 AND d.revoked_at IS NULL AND b.desired_state = 'bound' AND EXISTS (SELECT 1 FROM runtime_device_authority a WHERE a.id = d.id) -AND (d.environment_id IS NULL OR d.environment_id = e.id) ` type GetSessionDeviceParams struct { @@ -146,22 +152,19 @@ type GetSessionDeviceParams struct { type GetSessionDeviceRow struct { ID pgtype.UUID `json:"id"` Name string `json:"name"` - EnvironmentID pgtype.UUID `json:"environment_id"` SessionEnvironmentID pgtype.UUID `json:"session_environment_id"` AssignmentID pgtype.UUID `json:"assignment_id"` Epoch int64 `json:"epoch"` } // The Session's bound Runtime: a device of its tenant or the deployment's -// agent host. environment_id is the device's own Environment and -// session_environment_id the Session's, which the assignment binds. +// agent host, with the Session's Environment, which the assignment binds. func (q *Queries) GetSessionDevice(ctx context.Context, arg GetSessionDeviceParams) (GetSessionDeviceRow, error) { row := q.db.QueryRow(ctx, getSessionDevice, arg.TenantID, arg.ID) var i GetSessionDeviceRow err := row.Scan( &i.ID, &i.Name, - &i.EnvironmentID, &i.SessionEnvironmentID, &i.AssignmentID, &i.Epoch, @@ -170,7 +173,7 @@ func (q *Queries) GetSessionDevice(ctx context.Context, arg GetSessionDevicePara } const getSessionExecutionBinding = `-- name: GetSessionExecutionBinding :one -SELECT d.id, d.name, b.native_session_id, d.environment_id, e.id AS session_environment_id, b.assignment_id, b.epoch, +SELECT d.id, d.name, b.native_session_id, e.id AS session_environment_id, b.assignment_id, b.epoch, EXISTS (SELECT 1 FROM turns t WHERE t.session_id = s.id AND t.started_at IS NOT NULL) AS has_started_turn FROM session_runtime_assignments b JOIN sessions s ON s.id = b.session_id @@ -178,7 +181,6 @@ JOIN devices d ON d.id = b.runtime_id AND (d.tenant_id = s.tenant_id OR (d.agent LEFT JOIN environments e ON e.session_id = s.id WHERE s.tenant_id = $1 AND s.id = $2 AND d.revoked_at IS NULL AND b.desired_state = 'bound' AND EXISTS (SELECT 1 FROM runtime_device_authority a WHERE a.id = d.id) -AND (d.environment_id IS NULL OR d.environment_id = e.id) ` type GetSessionExecutionBindingParams struct { @@ -190,7 +192,6 @@ type GetSessionExecutionBindingRow struct { ID pgtype.UUID `json:"id"` Name string `json:"name"` NativeSessionID string `json:"native_session_id"` - EnvironmentID pgtype.UUID `json:"environment_id"` SessionEnvironmentID pgtype.UUID `json:"session_environment_id"` AssignmentID pgtype.UUID `json:"assignment_id"` Epoch int64 `json:"epoch"` @@ -205,7 +206,6 @@ func (q *Queries) GetSessionExecutionBinding(ctx context.Context, arg GetSession &i.ID, &i.Name, &i.NativeSessionID, - &i.EnvironmentID, &i.SessionEnvironmentID, &i.AssignmentID, &i.Epoch, diff --git a/services/core/internal/db/sqlc/environment_executor_credentials.sql.go b/services/core/internal/db/sqlc/environment_executor_credentials.sql.go index d759bf02c..1caf27f3b 100644 --- a/services/core/internal/db/sqlc/environment_executor_credentials.sql.go +++ b/services/core/internal/db/sqlc/environment_executor_credentials.sql.go @@ -55,12 +55,11 @@ func (q *Queries) ExecutorProjectScopeExists(ctx context.Context, arg ExecutorPr } const getEnvironmentExecutorConnection = `-- name: GetEnvironmentExecutorConnection :one -SELECT d.id AS device_id, d.executor_key_id, d.created_at AS enrolled_at, - d.last_seen_at, a.credential_hash, e.status AS environment_status +SELECT n.id AS enrollment_id, n.executor_key_id, n.created_at AS enrolled_at, r.credential_hash FROM environments e JOIN sessions s ON s.id = e.session_id -LEFT JOIN devices d ON d.environment_id = e.id AND d.tenant_id = s.tenant_id -LEFT JOIN runtime_device_authority a ON a.id = d.id +LEFT JOIN sandbox_enrollments n ON n.environment_id = e.id +LEFT JOIN sandbox_resources r ON r.kind = 'enrollment' AND r.id = n.id AND r.live WHERE e.id = $1 AND s.tenant_id = $2 AND s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'self_hosted' ` @@ -71,24 +70,22 @@ type GetEnvironmentExecutorConnectionParams struct { } type GetEnvironmentExecutorConnectionRow struct { - DeviceID pgtype.UUID `json:"device_id"` - ExecutorKeyID pgtype.UUID `json:"executor_key_id"` - EnrolledAt pgtype.Timestamptz `json:"enrolled_at"` - LastSeenAt pgtype.Timestamptz `json:"last_seen_at"` - CredentialHash pgtype.Text `json:"credential_hash"` - EnvironmentStatus string `json:"environment_status"` + EnrollmentID pgtype.UUID `json:"enrollment_id"` + ExecutorKeyID pgtype.UUID `json:"executor_key_id"` + EnrolledAt pgtype.Timestamptz `json:"enrolled_at"` + CredentialHash pgtype.Text `json:"credential_hash"` } +// The Environment's enrollment, with the credential that may Serve it while +// its Link resource is live. func (q *Queries) GetEnvironmentExecutorConnection(ctx context.Context, arg GetEnvironmentExecutorConnectionParams) (GetEnvironmentExecutorConnectionRow, error) { row := q.db.QueryRow(ctx, getEnvironmentExecutorConnection, arg.EnvironmentID, arg.TenantID) var i GetEnvironmentExecutorConnectionRow err := row.Scan( - &i.DeviceID, + &i.EnrollmentID, &i.ExecutorKeyID, &i.EnrolledAt, - &i.LastSeenAt, &i.CredentialHash, - &i.EnvironmentStatus, ) return i, err } @@ -257,6 +254,11 @@ WITH rotated AS ( ), advanced AS ( UPDATE sandbox_enrollments n SET generation = n.generation + 1 FROM rotated r WHERE n.executor_key_id = r.key_id + RETURNING n.environment_id +), rebound AS ( + UPDATE session_runtime_assignments b SET epoch = b.epoch + 1 + FROM advanced a JOIN environments e ON e.id = a.environment_id + WHERE b.session_id = e.session_id AND b.desired_state = 'bound' ) SELECT key_id, environment_id FROM rotated ` @@ -275,7 +277,9 @@ type RotateExecutorCredentialRow struct { } // Rotation advances the generation of the key's enrollments, so the Link -// authority refuses what the old secret served. +// authority refuses what the old secret served, and the epoch of the bound +// assignments of their Sessions, so the next Bind carries the new +// generation. func (q *Queries) RotateExecutorCredential(ctx context.Context, arg RotateExecutorCredentialParams) (RotateExecutorCredentialRow, error) { row := q.db.QueryRow(ctx, rotateExecutorCredential, arg.TokenSha256, diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index be47f8926..91dd5249d 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -349,10 +349,11 @@ type RuntimePlacement struct { } type SandboxEnrollment struct { - ID pgtype.UUID `json:"id"` - EnvironmentID pgtype.UUID `json:"environment_id"` - ExecutorKeyID pgtype.UUID `json:"executor_key_id"` - Generation int64 `json:"generation"` + ID pgtype.UUID `json:"id"` + EnvironmentID pgtype.UUID `json:"environment_id"` + ExecutorKeyID pgtype.UUID `json:"executor_key_id"` + Generation int64 `json:"generation"` + CreatedAt pgtype.Timestamptz `json:"created_at"` } type SandboxResource struct { diff --git a/services/core/internal/db/sqlc/runtime_enrollment.sql.go b/services/core/internal/db/sqlc/runtime_enrollment.sql.go index 5bc1b1d0c..7f0ef4495 100644 --- a/services/core/internal/db/sqlc/runtime_enrollment.sql.go +++ b/services/core/internal/db/sqlc/runtime_enrollment.sql.go @@ -12,7 +12,7 @@ import ( ) const authorizeRuntimeEnrollment = `-- name: AuthorizeRuntimeEnrollment :one -SELECT c.key_id, e.session_id, COALESCE(s.configuration->'environment'->>'workspace_directory', '')::text AS workspace_directory +SELECT c.key_id FROM environment_executor_credentials c JOIN environments e ON e.id = $1 JOIN sessions s ON s.id = e.session_id @@ -32,94 +32,41 @@ type AuthorizeRuntimeEnrollmentParams struct { TenantID pgtype.UUID `json:"tenant_id"` } -type AuthorizeRuntimeEnrollmentRow struct { - KeyID pgtype.UUID `json:"key_id"` - SessionID pgtype.UUID `json:"session_id"` - WorkspaceDirectory string `json:"workspace_directory"` -} - -func (q *Queries) AuthorizeRuntimeEnrollment(ctx context.Context, arg AuthorizeRuntimeEnrollmentParams) (AuthorizeRuntimeEnrollmentRow, error) { +func (q *Queries) AuthorizeRuntimeEnrollment(ctx context.Context, arg AuthorizeRuntimeEnrollmentParams) (pgtype.UUID, error) { row := q.db.QueryRow(ctx, authorizeRuntimeEnrollment, arg.EnvironmentID, arg.TokenSha256, arg.TenantID) - var i AuthorizeRuntimeEnrollmentRow - err := row.Scan(&i.KeyID, &i.SessionID, &i.WorkspaceDirectory) - return i, err + var key_id pgtype.UUID + err := row.Scan(&key_id) + return key_id, err } -const enrollRuntimeDevice = `-- name: EnrollRuntimeDevice :one -INSERT INTO devices (id, tenant_id, name, environment_id, executor_key_id) -VALUES ($1, $2, 'User-managed Runtime', $3, $4) -ON CONFLICT (environment_id) DO UPDATE SET name = devices.name -WHERE devices.executor_key_id = EXCLUDED.executor_key_id AND devices.revoked_at IS NULL -RETURNING id, name, environment_id +const enrollSandbox = `-- name: EnrollSandbox :one +INSERT INTO sandbox_enrollments (id, environment_id, executor_key_id) +VALUES ($1, $2, $3) +ON CONFLICT (environment_id) DO UPDATE SET executor_key_id = EXCLUDED.executor_key_id +WHERE sandbox_enrollments.executor_key_id = EXCLUDED.executor_key_id +RETURNING id, generation ` -type EnrollRuntimeDeviceParams struct { +type EnrollSandboxParams struct { ID pgtype.UUID `json:"id"` - TenantID pgtype.UUID `json:"tenant_id"` EnvironmentID pgtype.UUID `json:"environment_id"` ExecutorKeyID pgtype.UUID `json:"executor_key_id"` } -type EnrollRuntimeDeviceRow struct { - ID pgtype.UUID `json:"id"` - Name string `json:"name"` - EnvironmentID pgtype.UUID `json:"environment_id"` +type EnrollSandboxRow struct { + ID pgtype.UUID `json:"id"` + Generation int64 `json:"generation"` } -func (q *Queries) EnrollRuntimeDevice(ctx context.Context, arg EnrollRuntimeDeviceParams) (EnrollRuntimeDeviceRow, error) { - row := q.db.QueryRow(ctx, enrollRuntimeDevice, - arg.ID, - arg.TenantID, - arg.EnvironmentID, - arg.ExecutorKeyID, - ) - var i EnrollRuntimeDeviceRow - err := row.Scan(&i.ID, &i.Name, &i.EnvironmentID) +// The first key to enroll the Environment keeps its enrollment; the same key +// enrolling again returns it, and another key conflicts. +func (q *Queries) EnrollSandbox(ctx context.Context, arg EnrollSandboxParams) (EnrollSandboxRow, error) { + row := q.db.QueryRow(ctx, enrollSandbox, arg.ID, arg.EnvironmentID, arg.ExecutorKeyID) + var i EnrollSandboxRow + err := row.Scan(&i.ID, &i.Generation) return i, err } -const listEnrolledRuntimeBindings = `-- name: ListEnrolledRuntimeBindings :many -SELECT d.id AS device_id, d.tenant_id, e.id AS environment_id, e.session_id -FROM devices d -JOIN environments e ON e.id = d.environment_id -JOIN sessions s ON s.id = e.session_id AND s.tenant_id = d.tenant_id -WHERE d.executor_key_id IS NOT NULL AND s.deleted_at IS NULL - AND e.status NOT IN ('failed', 'expired') -ORDER BY d.id -` - -type ListEnrolledRuntimeBindingsRow struct { - DeviceID pgtype.UUID `json:"device_id"` - TenantID pgtype.UUID `json:"tenant_id"` - EnvironmentID pgtype.UUID `json:"environment_id"` - SessionID pgtype.UUID `json:"session_id"` -} - -func (q *Queries) ListEnrolledRuntimeBindings(ctx context.Context) ([]ListEnrolledRuntimeBindingsRow, error) { - rows, err := q.db.Query(ctx, listEnrolledRuntimeBindings) - if err != nil { - return nil, err - } - defer rows.Close() - items := []ListEnrolledRuntimeBindingsRow{} - for rows.Next() { - var i ListEnrolledRuntimeBindingsRow - if err := rows.Scan( - &i.DeviceID, - &i.TenantID, - &i.EnvironmentID, - &i.SessionID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const touchAuthenticatedDevice = `-- name: TouchAuthenticatedDevice :execrows UPDATE devices SET last_seen_at = clock_timestamp() WHERE devices.id = $1 AND EXISTS ( diff --git a/services/core/internal/db/sqlc/sandbox_link.sql.go b/services/core/internal/db/sqlc/sandbox_link.sql.go index f9c23dfb3..a997df574 100644 --- a/services/core/internal/db/sqlc/sandbox_link.sql.go +++ b/services/core/internal/db/sqlc/sandbox_link.sql.go @@ -28,6 +28,37 @@ func (q *Queries) GetAgentHostCredential(ctx context.Context, id pgtype.UUID) (G return i, err } +const getEnvironmentResource = `-- name: GetEnvironmentResource :one +SELECT kind, id, generation, credential_hash FROM sandbox_resources +WHERE tenant_id = $1 AND environment_id = $2 AND live +` + +type GetEnvironmentResourceParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + EnvironmentID pgtype.UUID `json:"environment_id"` +} + +type GetEnvironmentResourceRow struct { + Kind string `json:"kind"` + ID pgtype.UUID `json:"id"` + Generation int64 `json:"generation"` + CredentialHash pgtype.Text `json:"credential_hash"` +} + +// The Environment's live Link resource. An Environment has at most one Link +// resource: its allocation's or its enrollment's. +func (q *Queries) GetEnvironmentResource(ctx context.Context, arg GetEnvironmentResourceParams) (GetEnvironmentResourceRow, error) { + row := q.db.QueryRow(ctx, getEnvironmentResource, arg.TenantID, arg.EnvironmentID) + var i GetEnvironmentResourceRow + err := row.Scan( + &i.Kind, + &i.ID, + &i.Generation, + &i.CredentialHash, + ) + return i, err +} + const getLinkAssignment = `-- name: GetLinkAssignment :one SELECT b.session_id, b.runtime_id, b.epoch, b.desired_state = 'bound' AS bound, (d.agent_host AND d.revoked_at IS NULL)::boolean AS agent_host, d.credential_revision, diff --git a/services/core/internal/db/sqlc/scheduling.sql.go b/services/core/internal/db/sqlc/scheduling.sql.go index 9cdc8213b..ad31ebbdf 100644 --- a/services/core/internal/db/sqlc/scheduling.sql.go +++ b/services/core/internal/db/sqlc/scheduling.sql.go @@ -35,6 +35,38 @@ func (q *Queries) GetLatestSessionTurn(ctx context.Context, sessionID pgtype.UUI return i, err } +const listAgentHosts = `-- name: ListAgentHosts :many +SELECT id, name FROM devices +WHERE agent_host AND revoked_at IS NULL AND (tenant_id IS NULL OR tenant_id = $1) +ORDER BY id +` + +type ListAgentHostsRow struct { + ID pgtype.UUID `json:"id"` + Name string `json:"name"` +} + +// The agent hosts that may run the tenant's Sessions; see GetAgentHost. +func (q *Queries) ListAgentHosts(ctx context.Context, tenantID pgtype.UUID) ([]ListAgentHostsRow, error) { + rows, err := q.db.Query(ctx, listAgentHosts, tenantID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListAgentHostsRow{} + for rows.Next() { + var i ListAgentHostsRow + if err := rows.Scan(&i.ID, &i.Name); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listEnvironmentInputWork = `-- name: ListEnvironmentInputWork :many SELECT r.id, r.session_id, s.tenant_id FROM environment_input_reservations r @@ -43,8 +75,8 @@ LEFT JOIN session_runtime_assignments b ON b.session_id = s.id WHERE r.state = 'pending' AND r.deadline > clock_timestamp() AND r.id > $1::uuid AND s.deleted_at IS NULL AND EXISTS ( - SELECT 1 FROM devices d WHERE d.tenant_id = s.tenant_id AND d.revoked_at IS NULL - AND d.id = ANY($2::uuid[]) + SELECT 1 FROM devices d WHERE d.agent_host AND (d.tenant_id IS NULL OR d.tenant_id = s.tenant_id) + AND d.revoked_at IS NULL AND d.id = ANY($2::uuid[]) AND (b.runtime_id IS NULL OR d.id = b.runtime_id) ) ORDER BY r.id LIMIT 100 @@ -81,45 +113,14 @@ func (q *Queries) ListEnvironmentInputWork(ctx context.Context, arg ListEnvironm return items, nil } -const listExecutionDevices = `-- name: ListExecutionDevices :many -SELECT id, name FROM devices -WHERE tenant_id = $1 AND revoked_at IS NULL AND environment_id IS NULL -ORDER BY id -` - -type ListExecutionDevicesRow struct { - ID pgtype.UUID `json:"id"` - Name string `json:"name"` -} - -func (q *Queries) ListExecutionDevices(ctx context.Context, tenantID pgtype.UUID) ([]ListExecutionDevicesRow, error) { - rows, err := q.db.Query(ctx, listExecutionDevices, tenantID) - if err != nil { - return nil, err - } - defer rows.Close() - items := []ListExecutionDevicesRow{} - for rows.Next() { - var i ListExecutionDevicesRow - if err := rows.Scan(&i.ID, &i.Name); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const listExecutionWork = `-- name: ListExecutionWork :many SELECT t.id, t.session_id, s.tenant_id, t.status FROM turns t JOIN sessions s ON s.id = t.session_id WHERE t.status = ANY($1::text[]) AND t.id > $2::uuid AND (s.deleted_at IS NULL OR t.status <> 'queued') AND (NOT $3::boolean OR EXISTS ( - SELECT 1 FROM devices d WHERE d.tenant_id = s.tenant_id AND d.revoked_at IS NULL - AND d.id = ANY($4::uuid[]) + SELECT 1 FROM devices d WHERE d.agent_host AND (d.tenant_id IS NULL OR d.tenant_id = s.tenant_id) + AND d.revoked_at IS NULL AND d.id = ANY($4::uuid[]) )) ORDER BY t.id LIMIT 100 ` diff --git a/services/core/internal/deployment/allocation.go b/services/core/internal/deployment/allocation.go index 95f90b5d8..11cea1ef0 100644 --- a/services/core/internal/deployment/allocation.go +++ b/services/core/internal/deployment/allocation.go @@ -75,9 +75,6 @@ type NewAllocation struct { ServeCredentialHash string } -// SessionDevice is the Runtime device a Session is bound to. -type SessionDevice struct{ ID, EnvironmentID string } - // Activity separates real work from the connection keepalive. ObservedAt is // the database clock when it was read. type Activity struct { diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index 36f2dce8a..f6329b7b2 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -81,8 +81,8 @@ func (e *ExecutionOperations) ReserveAllocation(ctx context.Context, key Allocat } allocation.NodeID, allocation.Generation = reserved.NodeID, reserved.Generation } - device := sessions.ExecutionDevice{ID: allocation.DeviceID, Name: registration.Name, EnvironmentID: environment.ID} - if err := sessions.CreateEnvironmentDevice(ctx, tx, device, registration.CredentialHash); err != nil { + device := sessions.ExecutionDevice{ID: allocation.DeviceID, Name: registration.Name} + if err := sessions.CreateEnvironmentDevice(ctx, tx, environment.ID, device, registration.CredentialHash); err != nil { return err } result, err = tx.InsertAllocation(allocation) @@ -297,7 +297,7 @@ func (e *ExecutionOperations) change(ctx context.Context, owner Allocation, live // checkAllocation returns the stored allocation when it is still the owner's: // the same allocation, device, installation and node. A live change also -// requires the Session undeleted and bound to the allocation's device. +// requires the Session undeleted. func checkAllocation(tx AllocationTx, owner Allocation, live bool) (Allocation, error) { current, err := tx.LoadAllocation() if err != nil { @@ -312,13 +312,6 @@ func checkAllocation(tx AllocationTx, owner Allocation, live bool) (Allocation, if current.SessionDeleted { return Allocation{}, sessions.ErrNotFound } - device, bound, err := tx.LoadSessionDevice() - if err != nil { - return Allocation{}, err - } - if !bound || device.ID != current.DeviceID || device.EnvironmentID != current.EnvironmentID { - return Allocation{}, ErrAllocationConflict - } return current, nil } diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 2cc744247..ada74b16a 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -19,7 +19,7 @@ import ( type fakeReservationTx struct { t testing.TB loadBoundDevice func() (bool, error) - insertEnvironmentDevice func(sessions.ExecutionDevice, string) error + insertEnvironmentDevice func(string, sessions.ExecutionDevice, string) error loadEnvironment func() (sessions.Environment, error) findAllocation func() (Allocation, bool, error) lockDeployment func() (placement.Deployment, error) @@ -34,11 +34,11 @@ func (f *fakeReservationTx) LoadBoundDevice(context.Context) (bool, error) { return f.loadBoundDevice() } -func (f *fakeReservationTx) InsertEnvironmentDevice(_ context.Context, device sessions.ExecutionDevice, credentialHash string) error { +func (f *fakeReservationTx) InsertEnvironmentDevice(_ context.Context, environment string, device sessions.ExecutionDevice, credentialHash string) error { if f.insertEnvironmentDevice == nil { unexpected(f.t, "InsertEnvironmentDevice") } - return f.insertEnvironmentDevice(device, credentialHash) + return f.insertEnvironmentDevice(environment, device, credentialHash) } func (f *fakeReservationTx) LoadEnvironment(context.Context) (sessions.Environment, error) { @@ -77,12 +77,11 @@ func (f *fakeReservationTx) InsertAllocation(allocation NewAllocation) (Allocati } type fakeAllocationTx struct { - t testing.TB - loadAllocation func() (Allocation, error) - loadSessionDevice func() (SessionDevice, bool, error) - settleCreation func(Allocation) (Allocation, error) - keep func(Allocation) (Allocation, error) - release func(Allocation) (Allocation, error) + t testing.TB + loadAllocation func() (Allocation, error) + settleCreation func(Allocation) (Allocation, error) + keep func(Allocation) (Allocation, error) + release func(Allocation) (Allocation, error) } func (f *fakeAllocationTx) LoadAllocation() (Allocation, error) { @@ -92,13 +91,6 @@ func (f *fakeAllocationTx) LoadAllocation() (Allocation, error) { return f.loadAllocation() } -func (f *fakeAllocationTx) LoadSessionDevice() (SessionDevice, bool, error) { - if f.loadSessionDevice == nil { - unexpected(f.t, "LoadSessionDevice") - } - return f.loadSessionDevice() -} - func (f *fakeAllocationTx) LoadActivity(Allocation) (Activity, error) { unexpected(f.t, "LoadActivity") return Activity{}, nil @@ -339,17 +331,18 @@ func TestReserveAllocationAdmitsAndTakesTheReservedNode(t *testing.T) { t.Fatal("a released placement reserved an allocation", err) } var device sessions.ExecutionDevice + var deviceEnvironment string var inserted NewAllocation tx := fresh() tx.loadReserved = func() (placement.Reserved, error) { return placement.Reserved{NodeID: node, Generation: 5, Available: true}, nil } tx.loadBoundDevice = func() (bool, error) { return false, nil } - tx.insertEnvironmentDevice = func(d sessions.ExecutionDevice, hash string) error { + tx.insertEnvironmentDevice = func(environment string, d sessions.ExecutionDevice, hash string) error { if hash != testCredentialHash() { t.Fatal("device credential", hash) } - device = d + device, deviceEnvironment = d, environment return nil } tx.insertAllocation = func(a NewAllocation) (Allocation, error) { @@ -358,14 +351,13 @@ func TestReserveAllocationAdmitsAndTakesTheReservedNode(t *testing.T) { } serveHash := testCredentialHash() result, err := allocationOperations(t, tx, sessions.LockedSession{}, nil).ReserveAllocation(t.Context(), key, installation, testCredentialHash(), serveHash) - if err != nil || result.Replayed || inserted.NodeID != node || inserted.Generation != 5 || inserted.ProviderKey != installation || inserted.DeviceID != device.ID || device.EnvironmentID != key.EnvironmentID || inserted.ServeCredentialHash != serveHash { + if err != nil || result.Replayed || inserted.NodeID != node || inserted.Generation != 5 || inserted.ProviderKey != installation || inserted.DeviceID != device.ID || deviceEnvironment != key.EnvironmentID || inserted.ServeCredentialHash != serveHash { t.Fatal("reservation", result, inserted, device, err) } } -// A live change needs the Session undeleted and bound to the allocation's -// device; settlement continues for a deleted Session. Any other owner is a -// conflict. +// A live change needs the Session undeleted; settlement continues for a +// deleted Session. Any other owner is a conflict. func TestAllocationChangesCheckTheOwner(t *testing.T) { owner := Allocation{ID: uuid.NewString(), DeviceID: uuid.NewString(), EnvironmentID: uuid.NewString(), TenantID: uuid.NewString(), ProviderKey: uuid.NewString()} stored := func(current Allocation) func() (Allocation, error) { @@ -388,12 +380,6 @@ func TestAllocationChangesCheckTheOwner(t *testing.T) { if result, err := allocationOperations(t, nil, sessions.LockedSession{}, settled).SettleCreation(t.Context(), owner); err != nil || !result.CreateSettled { t.Fatal("a deleted Session's creation did not settle", result, err) } - unbound := &fakeAllocationTx{t: t, loadAllocation: stored(owner), loadSessionDevice: func() (SessionDevice, bool, error) { - return SessionDevice{ID: uuid.NewString(), EnvironmentID: owner.EnvironmentID}, true, nil - }} - if _, err := allocationOperations(t, nil, sessions.LockedSession{}, unbound).KeepAllocation(t.Context(), owner); !errors.Is(err, ErrAllocationConflict) { - t.Fatal("a Session bound to another device kept the allocation", err) - } } // SetCompute rejects an invalid phase change before it reaches storage. diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index ab6dac7f9..57653b5eb 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -104,9 +104,6 @@ type ReservationTx interface { type AllocationTx interface { // LoadAllocation returns the Environment's allocation. LoadAllocation() (Allocation, error) - // LoadSessionDevice returns the device the Session is bound to and - // whether it is bound to one. - LoadSessionDevice() (SessionDevice, bool, error) // LoadActivity returns the allocation's activity. LoadActivity(current Allocation) (Activity, error) // LoadRestore locks the deployment and returns what restoring the diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index 440f80ca2..97e90b8bc 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -100,8 +100,7 @@ func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { t.Fatal(err) } input := inputs[0] - // This fixture isolates lifecycle ordering. Protocol-driven waiting is - // independently exercised in TestArchiveWaitingCancellationReceipts. + // This fixture isolates lifecycle ordering. for _, transition := range []sessions.TurnTransition{{ExpectedStatus: sessions.TurnQueued, Status: sessions.TurnInProgress}, {ExpectedStatus: sessions.TurnInProgress, Status: sessions.TurnWaiting}} { if _, err := leased.Sessions.TransitionTurn(t.Context(), project.TenantID, session.ID, input.TurnID, transition); err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/directory_preparation_test.go b/services/core/internal/execution/directory_preparation_test.go index e6388203b..ba2f63044 100644 --- a/services/core/internal/execution/directory_preparation_test.go +++ b/services/core/internal/execution/directory_preparation_test.go @@ -12,7 +12,7 @@ func TestDirectoryPreparationRejectsForeignBindingBeforeTransport(t *testing.T) result := (&Dispatcher{}).readPreparedDirectory(t.Context(), nil, sessions.Session{ID: "session", TenantID: "tenant"}, sessions.Environment{ID: "environment", SessionID: "session", TenantID: "tenant", Configuration: []byte(`{"type":"self_hosted","workspace_directory":"/workspace"}`)}, - sessions.ExecutionDevice{EnvironmentID: "other"}, proto.WorkspaceReadPayload{}) + sessions.ExecutionDevice{SessionEnvironmentID: "other"}, proto.WorkspaceReadPayload{}) if !errors.Is(result.err, ErrExecutionUnavailable) { t.Fatal("foreign binding reached transport", result.err) } diff --git a/services/core/internal/execution/environment_capabilities_test.go b/services/core/internal/execution/environment_capabilities_test.go index 0e947b4a8..12350f637 100644 --- a/services/core/internal/execution/environment_capabilities_test.go +++ b/services/core/internal/execution/environment_capabilities_test.go @@ -14,7 +14,7 @@ func TestSelfHostedCapabilitySourcesAreFrozenAndStrict(t *testing.T) { environment := sessions.Environment{ID: "environment", SessionID: session.ID, TenantID: session.TenantID, Configuration: []byte(`{"type":"self_hosted","workspace_directory":"/home/user/project","capability_directories":["/home/user/skills","/opt/plugins"]}`)} var request proto.PromptRequestPayload - if err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{EnvironmentID: environment.ID}, &request); err != nil { + if err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{SessionEnvironmentID: environment.ID}, &request); err != nil { t.Fatal(err) } local := request.LocalEnvironment @@ -39,7 +39,7 @@ func TestSelfHostedPreparedPathsArePlatformNeutral(t *testing.T) { session := sessions.Session{ID: "session", TenantID: "tenant"} environment := sessions.Environment{ID: "environment", SessionID: session.ID, TenantID: session.TenantID, Configuration: raw} var request proto.PromptRequestPayload - err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{EnvironmentID: environment.ID}, &request) + err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{SessionEnvironmentID: environment.ID}, &request) if err != nil || request.LocalEnvironment.WorkspaceDirectory != directory || request.LocalEnvironment.CapabilitySources.Directories[0] != directory { t.Fatal("Core interpreted a Runtime source path", directory, err) } diff --git a/services/core/internal/execution/environment_directory.go b/services/core/internal/execution/environment_directory.go index 0e431266c..4a322c528 100644 --- a/services/core/internal/execution/environment_directory.go +++ b/services/core/internal/execution/environment_directory.go @@ -105,7 +105,10 @@ func (w *Worker) runDirectoryRead(owner context.Context, request directoryReadRe // Capture retains the public Turn after its native Run has been released. prepare := reserved || session.LastTurn != nil && session.LastTurn.ArtifactCaptureStarted bound, err := w.dispatcher.SessionsReader.GetSessionDevice(check, session.TenantID, session.ID) - if err != nil || !environmentDeviceMatches(session, environment, bound) || !w.directoryDeviceReady(check, bound.ID, session.Engine) { + if err != nil || bound.SessionEnvironmentID != environment.ID || !w.directoryDeviceReady(check, bound.ID, session.Engine) { + return + } + if serving, err := w.dispatcher.environmentServing(check, environment.TenantID, environment.ID); err != nil || !serving { return } peer, err := w.dispatcher.assignedPeer(check, bound) diff --git a/services/core/internal/execution/environment_file_write.go b/services/core/internal/execution/environment_file_write.go index 5f3e2b91f..cac1d95d7 100644 --- a/services/core/internal/execution/environment_file_write.go +++ b/services/core/internal/execution/environment_file_write.go @@ -90,7 +90,10 @@ func (w *Worker) runFileWrite(owner context.Context, request fileWriteRequest) f return fileWriteResult{err: err} } bound, err := w.dispatcher.SessionsReader.GetSessionDevice(ctx, session.TenantID, session.ID) - if err != nil || !environmentDeviceMatches(session, environment, bound) || w.dispatcher.Registry == nil { + if err != nil || bound.SessionEnvironmentID != environment.ID || w.dispatcher.Registry == nil { + return unavailable + } + if serving, err := w.dispatcher.environmentServing(ctx, environment.TenantID, environment.ID); err != nil || !serving { return unavailable } peer, err := w.dispatcher.assignedPeer(ctx, bound) diff --git a/services/core/internal/execution/environment_placement.go b/services/core/internal/execution/environment_placement.go index 7046d726f..4940dc885 100644 --- a/services/core/internal/execution/environment_placement.go +++ b/services/core/internal/execution/environment_placement.go @@ -76,16 +76,9 @@ func parseEnvironmentPlacement(configuration json.RawMessage) (environmentPlacem return placement, nil } -func environmentDeviceMatches(session sessions.Session, environment sessions.Environment, bound sessions.ExecutionDevice) bool { - if environment.SessionID != session.ID || environment.TenantID != session.TenantID { - return false - } - return bound.EnvironmentID == environment.ID -} - func (d *Dispatcher) configurePreparedEnvironment(session sessions.Session, environment sessions.Environment, bound sessions.ExecutionDevice, req *proto.PromptRequestPayload) error { placement, err := parseEnvironmentPlacement(environment.Configuration) - if err != nil || !environmentDeviceMatches(session, environment, bound) { + if err != nil || environment.SessionID != session.ID || environment.TenantID != session.TenantID || bound.SessionEnvironmentID != environment.ID { return sessions.ErrInvalidInput } sources := &agentcapabilities.Input{Plugins: append([]agentplugin.Metadata(nil), placement.Plugins...), Directories: append([]string(nil), placement.CapabilityDirectories...)} diff --git a/services/core/internal/execution/environment_placement_test.go b/services/core/internal/execution/environment_placement_test.go index 01fa05091..16e69d7a5 100644 --- a/services/core/internal/execution/environment_placement_test.go +++ b/services/core/internal/execution/environment_placement_test.go @@ -14,7 +14,7 @@ func TestSkillReferenceIdentityStopsAtCoreBoundary(t *testing.T) { environment := sessions.Environment{ID: "environment", SessionID: session.ID, TenantID: session.TenantID, Configuration: []byte(`{"type":"openai_hosted","initialization":true,"skills":[{"type":"skill_reference","skill_id":"skill-private","version":"1","name":"proof","description":"A proof."}]}`)} var request proto.PromptRequestPayload - err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{EnvironmentID: environment.ID}, &request) + err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{SessionEnvironmentID: environment.ID}, &request) if err != nil || request.LocalEnvironment == nil { t.Fatal("resolved Skill did not use the common installation descriptor", err) } @@ -26,7 +26,7 @@ func TestSkillReferenceIdentityStopsAtCoreBoundary(t *testing.T) { if !LocalWorkspaceConfiguration(environment.Configuration) { t.Fatal("admission demanded installed metadata before the creation transaction") } - if err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{EnvironmentID: environment.ID}, &proto.PromptRequestPayload{}); err == nil { + if err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{SessionEnvironmentID: environment.ID}, &proto.PromptRequestPayload{}); err == nil { t.Fatal("execution received an unresolved Skill selector") } } @@ -48,7 +48,7 @@ func TestLocalEnvironmentRequiresQualifiedProfileAndExactAuthority(t *testing.T) d := &Dispatcher{} for _, scope := range []string{"", "other", environment.ID} { var req proto.PromptRequestPayload - err := d.configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{EnvironmentID: scope}, &req) + err := d.configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{SessionEnvironmentID: scope}, &req) if scope == environment.ID { if err != nil || req.LocalEnvironment == nil || req.LocalEnvironment.ID != environment.ID { t.Fatal("local identity was not preserved", err) @@ -63,7 +63,7 @@ func TestToolEnvironmentRemainsInExecutionBinding(t *testing.T) { session := sessions.Session{ID: "session", TenantID: "tenant"} environment := sessions.Environment{ID: "environment", SessionID: session.ID, TenantID: session.TenantID, Configuration: []byte(`{"type":"openai_hosted","initialization":true,"packages":{"npm":["is-number"]}}`)} var req proto.PromptRequestPayload - err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{EnvironmentID: environment.ID}, &req) + err := (&Dispatcher{}).configurePreparedEnvironment(session, environment, sessions.ExecutionDevice{SessionEnvironmentID: environment.ID}, &req) if err != nil || req.LocalEnvironment == nil || !req.LocalEnvironment.ToolEnvironment { t.Fatal("tool initialization requirement lost", err) } diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 6f64bc441..33b273823 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -106,14 +106,9 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. } // allocationAssignment reads the assignment of the Session whose Environment -// the allocation runs; it is ErrNotFound unless the allocation's Runtime holds -// it. +// the allocation runs; it is ErrNotFound until placement binds the Session. func (r *runtimeLifecycle) allocationAssignment(ctx context.Context, owner deployment.Allocation) (sessions.ExecutionDevice, error) { - bound, err := r.sessions.GetSessionRuntimeDevice(ctx, owner.TenantID, owner.SessionID) - if err == nil && bound.ID != owner.DeviceID { - err = sessions.ErrNotFound - } - return bound, err + return r.sessions.GetSessionRuntimeDevice(ctx, owner.TenantID, owner.SessionID) } func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { @@ -168,7 +163,7 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro } // Publish disconnected only after receiving the daemon's receipt barrier. r.connections.mu.Lock() - err = observeRuntimeConnection(ctx, r.sessionExecution, r.connections.current, owner.TenantID, owner.EnvironmentID, nil, false) + err = observeRuntimeConnection(ctx, r.sessionExecution, r.connections.current, owner.TenantID, owner.EnvironmentID, false) r.connections.mu.Unlock() if err != nil { return err diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index dac40bd71..1fa34af68 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -7,7 +7,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -18,37 +17,19 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro return err } } - peer, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) + // The resumed sandbox serves again before its Runtime resumes the + // Environment. A later pass retries a wake whose sandbox or agent host is + // not connected. + if err := r.waitServing(ctx, owner); err != nil { + return err + } + bound, err := r.allocationAssignment(ctx, owner) if err != nil { - if !errors.Is(err, sessions.ErrNotFound) && !errors.Is(err, runtimegateway.ErrDeviceNotRegistered) && !errors.Is(err, runtimegateway.ErrSessionClosed) { - return err - } - // This idempotent control signal is fenced by guest PID/start time and the - // suspension token. It cannot execute or replay an agent request. - result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", "/run/oac/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) - if err != nil { - return err - } - if result.ExitCode != 0 { - return sandbox.ErrComputeUnconfirmed - } - timer := time.NewTicker(100 * time.Millisecond) - defer timer.Stop() - for { - peer, err = authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) - if err == nil { - break - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - } - } + return err } - // Resume carries the reference that quiesced the Runtime; a suspended + // Resume carries the reference that quiesced the Runtime; a quiesced // Runtime admits nothing else, so it is not bound again. - bound, err := r.allocationAssignment(ctx, owner) + peer, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, bound.ID) if err != nil { return err } @@ -74,6 +55,23 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro return r.deployment.ClearWake(ctx, next, owner.ComputeActivityAt) } +// waitServing waits, for at most 30 seconds, until the relay holds the serve +// peer of the allocation's Link resource. +func (r *runtimeLifecycle) waitServing(ctx context.Context, owner deployment.Allocation) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + timer := time.NewTicker(100 * time.Millisecond) + defer timer.Stop() + for !r.links.Serving(serveResource(owner).Ref()) { + select { + case <-ctx.Done(): + return sandbox.ErrComputeUnconfirmed + case <-timer.C: + } + } + return nil +} + func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { if err := r.lease.CheckOwnership(ctx); err != nil { return err diff --git a/services/core/internal/execution/runtime_connections.go b/services/core/internal/execution/runtime_connections.go index 6338da9f0..0e5e3f370 100644 --- a/services/core/internal/execution/runtime_connections.go +++ b/services/core/internal/execution/runtime_connections.go @@ -7,30 +7,35 @@ import ( "github.com/google/uuid" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) // environmentConnections holds the connection generation of each Environment -// with a live Link resource. The Worker's pass and the hosted lifecycle's -// quiesce publish through it; mu orders a whole pass before or after a -// quiesce's publish, so no pass republishes an Environment that quiesced. +// with a live Link resource, and the generation of each live Link resource +// this process saw. The Worker's pass and the hosted lifecycle's quiesce +// publish through it; mu orders a whole pass before or after a quiesce's +// publish, so no pass republishes an Environment that quiesced. type environmentConnections struct { mu sync.Mutex current map[string]*runtimeConnection + // served maps each live Link resource, without its generation, to the + // generation last seen. The relay is process-local, so a restart starts + // both empty. + served map[sandboxbootstrap.Resource]uint64 } // Durable generations fence old observations; a runtimeConnection only // remembers this process's generation of an Environment's connection. type runtimeConnection struct { - peer *runtimegateway.Session tenant string generation string revision int64 connected bool } -// observeSandboxConnections publishes each Environment with a live Link +// observeSandboxConnections revokes at the relay each Link resource whose +// Serve authority ended, then publishes each Environment with a live Link // resource as connected while the relay holds its serve peer, and as // disconnected once its resource is gone. It leaves a quiesced Environment to // the hosted lifecycle, which publishes it disconnected once its Runtime has @@ -43,6 +48,7 @@ func (w *Worker) observeSandboxConnections(ctx context.Context) error { if err != nil { return err } + w.revokeEndedResources(resources) live := make(map[string]bool, len(resources)) for _, resource := range resources { live[resource.Resource.EnvironmentID] = true @@ -50,7 +56,7 @@ func (w *Worker) observeSandboxConnections(ctx context.Context) error { continue } serving := w.dispatcher.Links.Serving(resource.Resource.Ref()) - if err := observeRuntimeConnection(ctx, w.dispatcher.sessionExecution, c.current, resource.Resource.TenantID, resource.Resource.EnvironmentID, nil, serving); err != nil && !environmentGone(err) { + if err := observeRuntimeConnection(ctx, w.dispatcher.sessionExecution, c.current, resource.Resource.TenantID, resource.Resource.EnvironmentID, serving); err != nil && !environmentGone(err) { return err } } @@ -58,7 +64,7 @@ func (w *Worker) observeSandboxConnections(ctx context.Context) error { if live[environment] { continue } - if err := observeRuntimeConnection(ctx, w.dispatcher.sessionExecution, c.current, current.tenant, environment, nil, false); err != nil && !environmentGone(err) { + if err := observeRuntimeConnection(ctx, w.dispatcher.sessionExecution, c.current, current.tenant, environment, false); err != nil && !environmentGone(err) { return err } delete(c.current, environment) @@ -66,6 +72,37 @@ func (w *Worker) observeSandboxConnections(ctx context.Context) error { return nil } +// revokeEndedResources revokes at the relay the previous generation of each +// resource whose generation advanced, and the last seen generation of each +// resource that is no longer live, which covers credential revocation and +// rotation, Environment expiry and Session deletion. It revokes only on those +// transitions, because each revocation advances the relay's epoch. The caller +// holds w.connections.mu. +func (w *Worker) revokeEndedResources(resources []sessions.SandboxResource) { + served := w.connections.served + live := make(map[sandboxbootstrap.Resource]bool, len(resources)) + for _, resource := range resources { + key := resource.Resource + key.Generation = 0 + live[key] = true + if previous, seen := served[key]; seen && previous < resource.Resource.Generation { + w.dispatcher.Links.RevokeResource(withGeneration(key, previous).Ref()) + } + served[key] = resource.Resource.Generation + } + for key, generation := range served { + if !live[key] { + w.dispatcher.Links.RevokeResource(withGeneration(key, generation).Ref()) + delete(served, key) + } + } +} + +func withGeneration(resource sandboxbootstrap.Resource, generation uint64) sandboxbootstrap.Resource { + resource.Generation = generation + return resource +} + // environmentGone reports a connection observation of an Environment that is // gone, failed or expired. func environmentGone(err error) bool { @@ -73,16 +110,16 @@ func environmentGone(err error) bool { } // Each Environment has one observer: the Worker's pass over Link resources, -// with the hosted lifecycle's quiesce, or the Worker loop for enrolled user -// compute. Both publish the same durable generation/revision rules. -func observeRuntimeConnection(ctx context.Context, operations *sessions.ExecutionOperations, connections map[string]*runtimeConnection, tenant, environment string, peer *runtimegateway.Session, connected bool) error { +// with the hosted lifecycle's quiesce. The first connection this process +// observes starts a durable generation; each change advances its revision. +func observeRuntimeConnection(ctx context.Context, operations *sessions.ExecutionOperations, connections map[string]*runtimeConnection, tenant, environment string, connected bool) error { current := connections[environment] - if connected && (current == nil || current.peer != peer) { + if connected && current == nil { generation := uuid.NewString() if err := operations.ReplaceEnvironmentConnection(ctx, tenant, environment, generation); err != nil { return err } - current = &runtimeConnection{peer: peer, tenant: tenant, generation: generation} + current = &runtimeConnection{tenant: tenant, generation: generation} connections[environment] = current } if current == nil || current.connected == connected { @@ -95,31 +132,3 @@ func observeRuntimeConnection(ctx context.Context, operations *sessions.Executio current.connected = connected return nil } - -func (w *Worker) observeEnrolledRuntimes(ctx context.Context) error { - bindings, err := w.dispatcher.SessionsReader.ListEnrolledRuntimeBindings(ctx) - if err != nil { - return err - } - live := make(map[string]bool, len(bindings)) - for _, bound := range bindings { - live[bound.EnvironmentID] = true - peer, err := w.dispatcher.authorizedPeer(ctx, bound.DeviceID) - connected := err == nil - if err != nil && !errors.Is(err, sessions.ErrNotFound) && !errors.Is(err, runtimegateway.ErrSessionClosed) && !errors.Is(err, runtimegateway.ErrDeviceNotRegistered) { - return err - } - if err := observeRuntimeConnection(ctx, w.dispatcher.sessionExecution, w.enrolledConnections, bound.TenantID, bound.EnvironmentID, peer, connected); err != nil && !environmentGone(err) { - return err - } - } - for id, current := range w.enrolledConnections { - if !live[id] { - if current.peer != nil { - current.peer.Close("Environment is no longer available") - } - delete(w.enrolledConnections, id) - } - } - return nil -} diff --git a/services/core/internal/execution/runtime_initialization.go b/services/core/internal/execution/runtime_initialization.go index 33ef2657d..531592e90 100644 --- a/services/core/internal/execution/runtime_initialization.go +++ b/services/core/internal/execution/runtime_initialization.go @@ -57,7 +57,25 @@ func (w *Worker) runEnvironmentInitializations(ctx context.Context) error { if len(active) >= w.executionConcurrency() { continue } - if owner.Resource.Kind != "" && (!owner.ResourceLive || !w.dispatcher.Links.Serving(owner.Resource.Ref())) { + // The agent host prepares an Environment through its Link + // resource, so one without a live resource Serving waits, even + // while its Session stays bound. + if !owner.ResourceLive || !w.dispatcher.Links.Serving(owner.Resource.Ref()) { + continue + } + if owner.DeviceID == "" { + // Placement binds an agent host once the Environment serves; + // a later scan claims the preparation on it. + if _, err := w.place(ctx, owner.TenantID, owner.SessionID, owner.EnvironmentID, func(id string) bool { + peer, err := w.dispatcher.authorizedPeer(ctx, id) + if err != nil { + return false + } + _, _, known := peer.AgentKindStatus(owner.Engine) + return known + }); err != nil && !errors.Is(err, sessions.ErrNotFound) && !errors.Is(err, sessions.ErrDeviceBindingConflict) { + return err + } continue } peer, err := w.dispatcher.authorizedPeer(ctx, owner.DeviceID) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index bcc90e47e..9d7541f0c 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -16,18 +17,17 @@ const DefaultExecutionConcurrency = 4 // Worker owns queued work; the database lease excludes a second execution service. type Worker struct { - concurrency int - metrics workerMetricsState - dispatcher *Dispatcher - lease Ownership - directoryReads chan directoryReadRequest - fileWrites chan fileWriteRequest - scheduleWake chan struct{} - stopped chan struct{} - stopOnce sync.Once - runtimes *runtimeManager - connections *environmentConnections - enrolledConnections map[string]*runtimeConnection + concurrency int + metrics workerMetricsState + dispatcher *Dispatcher + lease Ownership + directoryReads chan directoryReadRequest + fileWrites chan fileWriteRequest + scheduleWake chan struct{} + stopped chan struct{} + stopOnce sync.Once + runtimes *runtimeManager + connections *environmentConnections } // StartWorker takes over owner.Lease from the moment it is called: a failed @@ -77,7 +77,7 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W return nil, errors.New("execution worker requires the Link relay") } owned.notifications = &executionNotifications{} - worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: owned, lease: owner.Lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), connections: &environmentConnections{current: make(map[string]*runtimeConnection)}, enrolledConnections: make(map[string]*runtimeConnection)} + worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: owned, lease: owner.Lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), connections: &environmentConnections{current: make(map[string]*runtimeConnection), served: make(map[sandboxbootstrap.Resource]uint64)}} worker.runtimes, err = newRuntimeManager(owner, owned.Deployment, owned.DeploymentReader, owned.SessionsReader, owned.Registry, owned.Links, worker.connections, owned.ManagedRuntimes) if err != nil { return nil, err @@ -300,10 +300,6 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { w.observeSchedulerPoll(0, err) return err } - if err := w.observeEnrolledRuntimes(ctx); err != nil { - w.observeSchedulerPoll(0, err) - return err - } } if len(active) == w.executionConcurrency() { w.observeSchedulerPoll(0, nil) diff --git a/services/core/internal/execution/worker_device.go b/services/core/internal/execution/worker_device.go index 4634c06f4..9b6ec58e1 100644 --- a/services/core/internal/execution/worker_device.go +++ b/services/core/internal/execution/worker_device.go @@ -67,54 +67,81 @@ func (w *Worker) bindDevice(ctx context.Context, tenantID, sessionID string, inp }) } +// bindSessionDevice places the Session for a Turn or Environment input. A +// hosted or self_hosted Session is placed once its Environment completed +// initialization and while its compute is not quiesced; the initialization +// scanner places it before that. func (w *Worker) bindSessionDevice(ctx context.Context, session sessions.Session, ready func(string) bool) (bool, error) { var snapshot Snapshot if json.Unmarshal(session.Configuration, &snapshot) != nil { return false, sessions.ErrInvalidInput } - if snapshot.Environment != nil && (snapshot.Environment.Type == "openai_hosted" || snapshot.Environment.Type == "self_hosted") { - environment, err := w.dispatcher.SessionsReader.GetSessionEnvironment(ctx, session.TenantID, session.ID) - if err != nil { - return false, err - } - if _, err := parseEnvironmentPlacement(environment.Configuration); err != nil { + if snapshot.Environment == nil || (snapshot.Environment.Type != "openai_hosted" && snapshot.Environment.Type != "self_hosted") { + return w.place(ctx, session.TenantID, session.ID, "", ready) + } + environment, err := w.dispatcher.SessionsReader.GetSessionEnvironment(ctx, session.TenantID, session.ID) + if err != nil { + return false, err + } + if _, err := parseEnvironmentPlacement(environment.Configuration); err != nil || environment.Initialization != "complete" { + return false, nil + } + allocation, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(ctx, deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: environment.ID}) + if errors.Is(err, deployment.ErrNotFound) { + if snapshot.Environment.Type == "openai_hosted" { return false, nil } - allocation, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(ctx, deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: environment.ID}) - if errors.Is(err, deployment.ErrNotFound) { - if snapshot.Environment.Type == "openai_hosted" { - return false, nil - } - } else if err != nil { + } else if err != nil { + return false, err + } else if allocation.ComputePhase != "disabled" && allocation.ComputePhase != "running" { + // A reconnect authenticates transport before the retained Environment + // resumes. Its first control frame must remain the lifecycle's Resume. + return false, nil + } + return w.place(ctx, session.TenantID, session.ID, environment.ID, ready) +} + +// place reports whether the Session's bound Runtime is ready and, for a +// Session with an Environment, the Environment's Link resource is Serving. +// A Session without an assignment is bound to the first ready agent host once +// its Environment, if any, is Serving. +func (w *Worker) place(ctx context.Context, tenant, session, environment string, ready func(string) bool) (bool, error) { + if environment != "" { + serving, err := w.dispatcher.environmentServing(ctx, tenant, environment) + if err != nil || !serving { return false, err - } else if allocation.ComputePhase != "disabled" && allocation.ComputePhase != "running" { - // A reconnect authenticates transport before the retained Environment - // resumes. Its first control frame must remain the lifecycle's Resume. - return false, nil - } - bound, err := w.dispatcher.SessionsReader.GetSessionDevice(ctx, session.TenantID, session.ID) - if errors.Is(err, sessions.ErrNotFound) { - return false, nil } - return err == nil && environmentDeviceMatches(session, environment, bound) && ready(bound.ID), err } - bound, err := w.dispatcher.SessionsReader.GetSessionDevice(ctx, session.TenantID, session.ID) + bound, err := w.dispatcher.SessionsReader.GetSessionRuntimeDevice(ctx, tenant, session) if err == nil { - return bound.EnvironmentID == "" && ready(bound.ID), nil + return ready(bound.ID), nil } if !errors.Is(err, sessions.ErrNotFound) { return false, err } - devices, err := w.dispatcher.SessionsReader.ListExecutionDevices(ctx, session.TenantID) + hosts, err := w.dispatcher.SessionsReader.ListAgentHosts(ctx, tenant) if err != nil { return false, err } - for _, device := range devices { - if !ready(device.ID) { + for _, host := range hosts { + if !ready(host.ID) { continue } - err := w.dispatcher.sessionExecution.BindSessionDevice(ctx, session.TenantID, session.ID, device.ID) + err := w.dispatcher.sessionExecution.BindSessionDevice(ctx, tenant, session, host.ID) return err == nil, err } return false, nil } + +// environmentServing reports whether the relay holds the serve peer of the +// tenant's Environment's live Link resource. +func (d *Dispatcher) environmentServing(ctx context.Context, tenant, environment string) (bool, error) { + resource, err := d.SessionsReader.GetEnvironmentResource(ctx, tenant, environment) + if errors.Is(err, sessions.ErrNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return d.Links.Serving(resource.Resource.Ref()), nil +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index e64ebd9e7..57eda73e0 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -241,17 +241,6 @@ func (t *allocationTx) LoadAllocation() (deployment.Allocation, error) { return current, err } -func (t *allocationTx) LoadSessionDevice() (deployment.SessionDevice, bool, error) { - row, err := t.q.GetSessionDevice(t.ctx, sqlc.GetSessionDeviceParams{TenantID: t.tenant, ID: t.session}) - if errors.Is(err, pgx.ErrNoRows) { - return deployment.SessionDevice{}, false, nil - } - if err != nil { - return deployment.SessionDevice{}, false, err - } - return deployment.SessionDevice{ID: uuidString(row.ID), EnvironmentID: uuidString(row.EnvironmentID)}, true, nil -} - func (t *allocationTx) LoadActivity(current deployment.Allocation) (deployment.Activity, error) { id, err := parseID(current.ID) if err != nil { diff --git a/services/core/internal/persistence/postgres/sessionpg/binding.go b/services/core/internal/persistence/postgres/sessionpg/binding.go index dad3703c5..d73d38122 100644 --- a/services/core/internal/persistence/postgres/sessionpg/binding.go +++ b/services/core/internal/persistence/postgres/sessionpg/binding.go @@ -166,26 +166,22 @@ func (t *SessionTx) LoadBoundDevice(ctx context.Context) (bool, error) { // Environment. The insert takes no row when the Environment already has a // device, including one a concurrent transaction inserted first, or cannot // take one: that is sessions.ErrDeviceBindingConflict. -func (t *SessionTx) InsertEnvironmentDevice(ctx context.Context, device sessions.ExecutionDevice, credentialHash string) error { +func (t *SessionTx) InsertEnvironmentDevice(ctx context.Context, environment string, device sessions.ExecutionDevice, credentialHash string) error { id, err := parseID(device.ID) if err != nil { return err } - environment, err := parseID(device.EnvironmentID) + environmentID, err := parseID(environment) if err != nil { return err } - inserted, err := t.q.CreateEnvironmentDevice(ctx, sqlc.CreateEnvironmentDeviceParams{ - ID: id, TenantID: t.tenant, SessionID: t.session, EnvironmentID: environment, + _, err = t.q.CreateEnvironmentDevice(ctx, sqlc.CreateEnvironmentDeviceParams{ + ID: id, TenantID: t.tenant, SessionID: t.session, EnvironmentID: environmentID, Name: device.Name, CredentialHash: pgtype.Text{String: credentialHash, Valid: true}, }) if errors.Is(err, pgx.ErrNoRows) { return sessions.ErrDeviceBindingConflict } - if err != nil { - return err - } - _, err = t.q.BindSessionDevice(ctx, sqlc.BindSessionDeviceParams{TenantID: t.tenant, ID: t.session, ID_2: inserted}) return err } diff --git a/services/core/internal/persistence/postgres/sessionpg/devices.go b/services/core/internal/persistence/postgres/sessionpg/devices.go index c5e23051a..5f3d4c04a 100644 --- a/services/core/internal/persistence/postgres/sessionpg/devices.go +++ b/services/core/internal/persistence/postgres/sessionpg/devices.go @@ -10,6 +10,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" @@ -44,7 +45,7 @@ func (s *Store) GetSessionExecutionBinding(ctx context.Context, tenant, session return sessions.ExecutionBinding{}, err } return sessions.ExecutionBinding{ - Device: sessions.ExecutionDevice{ID: uuid.UUID(row.ID.Bytes).String(), Name: row.Name, EnvironmentID: optionalID(row.EnvironmentID), + Device: sessions.ExecutionDevice{ID: uuid.UUID(row.ID.Bytes).String(), Name: row.Name, Assignment: assignmentRef(lookup.ID, row.AssignmentID, row.Epoch), SessionEnvironmentID: optionalID(row.SessionEnvironmentID)}, NativeSessionID: row.NativeSessionID, HasStartedTurn: row.HasStartedTurn, @@ -54,7 +55,7 @@ func (s *Store) GetSessionExecutionBinding(ctx context.Context, tenant, session // requireInitialized requires that the tenant's Session completed its // Environment preparation; before that, and for a missing Session, it is // sessions.ErrNotFound. -func requireInitialized(ctx context.Context, q *sqlc.Queries, lookup sqlc.GetDeviceParams) error { +func requireInitialized(ctx context.Context, q *sqlc.Queries, lookup Lookup) error { ready, err := q.GetSessionInitializationReady(ctx, sqlc.GetSessionInitializationReadyParams(lookup)) if errors.Is(err, pgx.ErrNoRows) || (err == nil && !ready) { return sessions.ErrNotFound @@ -62,12 +63,12 @@ func requireInitialized(ctx context.Context, q *sqlc.Queries, lookup sqlc.GetDev return err } -func (s *Store) ListExecutionDevices(ctx context.Context, tenant string) ([]sessions.ExecutionDevice, error) { +func (s *Store) ListAgentHosts(ctx context.Context, tenant string) ([]sessions.ExecutionDevice, error) { id, err := parseID(tenant) if err != nil { return nil, err } - rows, err := s.units.Queries().ListExecutionDevices(ctx, id) + rows, err := s.units.Queries().ListAgentHosts(ctx, id) if err != nil { return nil, err } @@ -100,7 +101,7 @@ func loadSessionDevice(ctx context.Context, q *sqlc.Queries, tenant, session pgt if err != nil { return sessions.ExecutionDevice{}, false, err } - return sessions.ExecutionDevice{ID: uuid.UUID(row.ID.Bytes).String(), Name: row.Name, EnvironmentID: optionalID(row.EnvironmentID), + return sessions.ExecutionDevice{ID: uuid.UUID(row.ID.Bytes).String(), Name: row.Name, Assignment: assignmentRef(session, row.AssignmentID, row.Epoch), SessionEnvironmentID: optionalID(row.SessionEnvironmentID)}, true, nil } @@ -157,21 +158,6 @@ func (s *Store) ArchivedCancellationReceipt(ctx context.Context, device, credent return runtimedevice.ArchivedCancellationReceipt{RunID: uuid.UUID(row.ID.Bytes).String(), Deadline: row.CancelRequestedAt.Time.Add(runtimedevice.ArchivedCancellationReceiptLimit)}, nil } -func (s *Store) ListEnrolledRuntimeBindings(ctx context.Context) ([]sessions.EnrolledRuntimeBinding, error) { - rows, err := s.units.Queries().ListEnrolledRuntimeBindings(ctx) - if err != nil { - return nil, err - } - result := make([]sessions.EnrolledRuntimeBinding, 0, len(rows)) - for _, row := range rows { - result = append(result, sessions.EnrolledRuntimeBinding{ - DeviceID: optionalID(row.DeviceID), TenantID: optionalID(row.TenantID), - EnvironmentID: optionalID(row.EnvironmentID), SessionID: optionalID(row.SessionID), - }) - } - return result, nil -} - func (s *Store) CreateDevice(ctx context.Context, tenant string, registration sessions.DeviceRegistration) (sessions.ExecutionDevice, error) { tenantID, err := parseID(tenant) if err != nil { @@ -255,32 +241,32 @@ type enrollmentTx struct { var _ sessions.EnrollmentTx = (*enrollmentTx)(nil) -func (t *enrollmentTx) AuthorizeEnrollment(ctx context.Context) (sessions.EnrollmentAuthority, error) { - row, err := t.q.AuthorizeRuntimeEnrollment(ctx, sqlc.AuthorizeRuntimeEnrollmentParams{EnvironmentID: t.environment, TenantID: t.tenant, TokenSha256: t.credentialHash}) +func (t *enrollmentTx) AuthorizeEnrollment(ctx context.Context) (string, error) { + key, err := t.q.AuthorizeRuntimeEnrollment(ctx, sqlc.AuthorizeRuntimeEnrollmentParams{EnvironmentID: t.environment, TenantID: t.tenant, TokenSha256: t.credentialHash}) if errors.Is(err, pgx.ErrNoRows) { - return sessions.EnrollmentAuthority{}, sessions.ErrNotFound + return "", sessions.ErrNotFound } if err != nil { - return sessions.EnrollmentAuthority{}, err + return "", err } - return sessions.EnrollmentAuthority{KeyID: optionalID(row.KeyID), WorkspaceDirectory: row.WorkspaceDirectory}, nil + return optionalID(key), nil } -func (t *enrollmentTx) EnrollDevice(ctx context.Context, key string) (string, error) { +func (t *enrollmentTx) EnrollSandbox(ctx context.Context, key string) (sandboxbootstrap.Resource, error) { keyID, err := parseID(key) if err != nil { - return "", err + return sandboxbootstrap.Resource{}, err } - row, err := t.q.EnrollRuntimeDevice(ctx, sqlc.EnrollRuntimeDeviceParams{ - ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, TenantID: t.tenant, EnvironmentID: t.environment, ExecutorKeyID: keyID, + row, err := t.q.EnrollSandbox(ctx, sqlc.EnrollSandboxParams{ + ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, EnvironmentID: t.environment, ExecutorKeyID: keyID, }) if errors.Is(err, pgx.ErrNoRows) { - return "", sessions.ErrDeviceBindingConflict + return sandboxbootstrap.Resource{}, sessions.ErrDeviceBindingConflict } if err != nil { - return "", err + return sandboxbootstrap.Resource{}, err } - return uuid.UUID(row.ID.Bytes).String(), nil + return linkResource(t.tenant, t.environment, pgtype.Text{String: "enrollment", Valid: true}, row.ID, pgtype.Int8{Int64: row.Generation, Valid: true}), nil } var _ sessions.DeviceBindingTx = (*SessionTx)(nil) @@ -290,7 +276,7 @@ func (t *SessionTx) LoadDevice(ctx context.Context, device string) (bool, error) if err != nil { return false, err } - _, err = t.q.GetDevice(ctx, sqlc.GetDeviceParams{TenantID: t.tenant, ID: id}) + _, err = t.q.GetAgentHost(ctx, sqlc.GetAgentHostParams{TenantID: t.tenant, ID: id}) if errors.Is(err, pgx.ErrNoRows) { return false, nil } @@ -308,8 +294,9 @@ func (t *SessionTx) ReleaseAssignment(ctx context.Context, removeHome bool) erro return t.q.ReleaseSessionAssignment(ctx, sqlc.ReleaseSessionAssignmentParams{SessionID: t.session, RemoveHome: removeHome}) } -// BindDevice binds the device to the Session; a Session bound to another -// device, or a released assignment, is sessions.ErrDeviceBindingConflict. +// BindDevice binds the agent host to the Session; a Session bound to another +// Runtime, a released assignment, or a Session whose Environment has no live +// Link resource is sessions.ErrDeviceBindingConflict. func (t *SessionTx) BindDevice(ctx context.Context, device string) error { id, err := parseID(device) if err != nil { diff --git a/services/core/internal/persistence/postgres/sessionpg/devices_test.go b/services/core/internal/persistence/postgres/sessionpg/devices_test.go index 55ff5bcc9..4f9da5bfb 100644 --- a/services/core/internal/persistence/postgres/sessionpg/devices_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/devices_test.go @@ -15,15 +15,28 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// newDevice stores a device of tenant, dedicated to environment when it is +// newAgentHost stores an agent host of tenant, or of none when tenant is not // valid, and returns its ID. -func newDevice(t *testing.T, pool *pgxpool.Pool, tenant, environment pgtype.UUID) string { +func newAgentHost(t *testing.T, pool *pgxpool.Pool, tenant pgtype.UUID) string { t.Helper() id := uuid.NewString() - exec(t, pool, `INSERT INTO devices(id, tenant_id, name, credential_hash, environment_id) VALUES ($1, $2, 'runtime', $3, $4)`, id, tenant, strings.Repeat("a", 64), environment) + exec(t, pool, `INSERT INTO devices(id, tenant_id, name, credential_hash, agent_host) VALUES ($1, $2, 'agent host', $3, true)`, id, tenant, strings.Repeat("a", 64)) return id } +// enroll gives the self_hosted Environment a live enrollment Link resource. +func enroll(t *testing.T, pool *pgxpool.Pool, tenant, session, environment pgtype.UUID) { + t.Helper() + key := uuid.New() + exec(t, pool, `UPDATE sessions SET creator_kind = 'user', creator_id = 'owner' WHERE id = $1`, session) + exec(t, pool, `INSERT INTO execution_project_scopes(tenant_id, organization_id, project_id) VALUES ($1, 'org', $2)`, tenant, uuid.UUID(tenant.Bytes).String()) + exec(t, pool, `INSERT INTO environment_executor_credentials(key_id, tenant_id, subject_kind, subject_id, environment_id, token_sha256, created_at, issued_at) + VALUES ($1, $2, 'user', 'owner', $3, $4, clock_timestamp(), clock_timestamp())`, key, tenant, environment, strings.ReplaceAll(uuid.NewString()+uuid.NewString(), "-", "")) + exec(t, pool, `INSERT INTO sandbox_enrollments(id, environment_id, executor_key_id) VALUES ($1, $2, $3)`, uuid.New(), environment, key) +} + +// Placement binds a Session to an agent host only, and a Session with an +// Environment only while that Environment has a live Link resource. func TestBindSessionDeviceTranslatesTheBindingOutcome(t *testing.T) { pool := pgtest.OpenIsolated(t, nil) lease, err := pgunit.AcquireLease(t.Context(), pool) @@ -35,34 +48,37 @@ func TestBindSessionDeviceTranslatesTheBindingOutcome(t *testing.T) { if err != nil { t.Fatal(err) } - tenantID, sessionID, _ := newEnvironment(t, pool, "self_hosted", "pending") + tenantID, sessionID, environmentID := newEnvironment(t, pool, "self_hosted", "pending") tenant, session := uuidText(tenantID), uuidText(sessionID) - _, _, otherEnvironment := newEnvironment(t, pool, "self_hosted", "pending") otherTenant, _, _ := newEnvironment(t, pool, "self_hosted", "pending") - device, replacement := newDevice(t, pool, tenantID, pgtype.UUID{}), newDevice(t, pool, tenantID, pgtype.UUID{}) - revoked := newDevice(t, pool, tenantID, pgtype.UUID{}) + device, replacement := newAgentHost(t, pool, pgtype.UUID{}), newAgentHost(t, pool, tenantID) + revoked := newAgentHost(t, pool, pgtype.UUID{}) exec(t, pool, `UPDATE devices SET revoked_at = clock_timestamp() WHERE id = $1`, revoked) + tenantDevice := uuid.NewString() + exec(t, pool, `INSERT INTO devices(id, tenant_id, name, credential_hash) VALUES ($1, $2, 'runtime', $3)`, tenantDevice, tenantID, strings.Repeat("a", 64)) for name, test := range map[string]struct { device string want error }{ - "another tenant's device": {newDevice(t, pool, otherTenant, pgtype.UUID{}), sessions.ErrNotFound}, - "revoked device": {revoked, sessions.ErrNotFound}, - "device of another Environment": {newDevice(t, pool, tenantID, otherEnvironment), sessions.ErrDeviceBindingConflict}, - "malformed device": {"device", sessions.ErrInvalidInput}, + "another tenant's agent host": {newAgentHost(t, pool, otherTenant), sessions.ErrNotFound}, + "revoked agent host": {revoked, sessions.ErrNotFound}, + "device that is no agent host": {tenantDevice, sessions.ErrNotFound}, + "malformed device": {"device", sessions.ErrInvalidInput}, + "Environment without a live resource": {device, sessions.ErrDeviceBindingConflict}, } { if err := operations.BindSessionDevice(t.Context(), tenant, session, test.device); !errors.Is(err, test.want) { t.Fatalf("%s: %v, want %v", name, err, test.want) } } + enroll(t, pool, tenantID, sessionID, environmentID) for range 2 { if err := operations.BindSessionDevice(t.Context(), tenant, session, device); err != nil { - t.Fatal("binding the bound device again", err) + t.Fatal("binding the bound agent host again", err) } } if err := operations.BindSessionDevice(t.Context(), tenant, session, replacement); !errors.Is(err, sessions.ErrDeviceBindingConflict) { - t.Fatal("rebinding the Session to another device", err) + t.Fatal("rebinding the Session to another agent host", err) } var bound string if err := pool.QueryRow(t.Context(), `SELECT runtime_id::text FROM session_runtime_assignments WHERE session_id = $1`, sessionID).Scan(&bound); err != nil || bound != device { diff --git a/services/core/internal/persistence/postgres/sessionpg/executor_credentials.go b/services/core/internal/persistence/postgres/sessionpg/executor_credentials.go index 6d77d7a8b..3179f3334 100644 --- a/services/core/internal/persistence/postgres/sessionpg/executor_credentials.go +++ b/services/core/internal/persistence/postgres/sessionpg/executor_credentials.go @@ -62,8 +62,7 @@ func (s *Store) ProjectExecutorCredentialState(ctx context.Context, project iden return err } result.EnvironmentID = uuid.UUID(environmentID.Bytes).String() - result.Connection.EnvironmentStatus = row.EnvironmentStatus - result.Connection.DeviceID = optionalID(row.DeviceID) + result.Connection.Enrolled = row.EnrollmentID.Valid if row.ExecutorKeyID.Valid { key := optionalID(row.ExecutorKeyID) result.Connection.BoundKeyID = &key @@ -72,10 +71,6 @@ func (s *Store) ProjectExecutorCredentialState(ctx context.Context, project iden at := row.EnrolledAt.Time result.Connection.EnrolledAt = &at } - if row.LastSeenAt.Valid { - at := row.LastSeenAt.Time - result.Connection.LastSeenAt = &at - } result.Connection.CredentialHash = row.CredentialHash.String result.Credentials, err = listExecutorCredentials(ctx, q, tenant, environmentID, project.Subject()) return err diff --git a/services/core/internal/persistence/postgres/sessionpg/link.go b/services/core/internal/persistence/postgres/sessionpg/link.go index 266af9b8c..fce516440 100644 --- a/services/core/internal/persistence/postgres/sessionpg/link.go +++ b/services/core/internal/persistence/postgres/sessionpg/link.go @@ -54,6 +54,26 @@ func (s *Store) ListLiveSandboxResources(ctx context.Context) ([]sessions.Sandbo return result, nil } +// GetEnvironmentResource reads the live Link resource of the tenant's +// Environment; without one, or for a malformed ID, it is ErrNotFound. +func (s *Store) GetEnvironmentResource(ctx context.Context, tenant, environment string) (runtimedevice.ServeAuthority, error) { + lookup, err := ResourceLookup(tenant, environment) + if err != nil { + return runtimedevice.ServeAuthority{}, sessions.ErrNotFound + } + row, err := s.units.Queries().GetEnvironmentResource(ctx, sqlc.GetEnvironmentResourceParams{TenantID: lookup.TenantID, EnvironmentID: lookup.ID}) + if errors.Is(err, pgx.ErrNoRows) { + return runtimedevice.ServeAuthority{}, sessions.ErrNotFound + } + if err != nil { + return runtimedevice.ServeAuthority{}, err + } + return runtimedevice.ServeAuthority{ + Resource: linkResource(lookup.TenantID, lookup.ID, pgtype.Text{String: row.Kind, Valid: true}, row.ID, pgtype.Int8{Int64: row.Generation, Valid: true}), + CredentialHash: row.CredentialHash.String, + }, nil +} + // GetAgentHostCredential reads a live agent host's credential. A malformed or // unknown device, or one that is not an agent host, has none. func (s *Store) GetAgentHostCredential(ctx context.Context, runtime string) (runtimedevice.AgentHost, bool, error) { diff --git a/services/core/internal/persistence/postgres/sessionpg/rows.go b/services/core/internal/persistence/postgres/sessionpg/rows.go index d635b72bf..a5060dd7b 100644 --- a/services/core/internal/persistence/postgres/sessionpg/rows.go +++ b/services/core/internal/persistence/postgres/sessionpg/rows.go @@ -42,11 +42,17 @@ func TurnLookup(tenantID, sessionID, turnID string) (sqlc.GetTurnParams, error) return p, err } +// Lookup is a tenant and the internal identifier of one of its resources. +type Lookup struct { + TenantID pgtype.UUID + ID pgtype.UUID +} + // ResourceLookup parses a tenant and the internal identifier of one of its // resources, such as a device, Session or Environment; a malformed one is // sessions.ErrInvalidInput. -func ResourceLookup(tenantID, id string) (sqlc.GetDeviceParams, error) { - var p sqlc.GetDeviceParams +func ResourceLookup(tenantID, id string) (Lookup, error) { + var p Lookup var err error if p.TenantID, err = parseID(tenantID); err != nil { return p, err diff --git a/services/core/internal/persistence/postgres/sessionpg/session_test.go b/services/core/internal/persistence/postgres/sessionpg/session_test.go index d192899d7..421a1f1f4 100644 --- a/services/core/internal/persistence/postgres/sessionpg/session_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/session_test.go @@ -251,20 +251,20 @@ func TestTerminateEnvironmentThroughTheBindingExpires(t *testing.T) { } } -func TestCreateEnvironmentDeviceBindsOneDevice(t *testing.T) { +func TestCreateEnvironmentDeviceCreatesOneDevice(t *testing.T) { pool := pgtest.Open(t) hash := strings.Repeat("a", 64) create := func(tenant, session, environment pgtype.UUID, device uuid.UUID, commit bool) error { - dedicated := sessions.ExecutionDevice{ID: device.String(), Name: "runtime", EnvironmentID: uuid.UUID(environment.Bytes).String()} + dedicated := sessions.ExecutionDevice{ID: device.String(), Name: "runtime"} return pgx.BeginFunc(t.Context(), pool, func(tx pgx.Tx) error { - if err := sessions.CreateEnvironmentDevice(t.Context(), BindSession(sqlc.New(tx), tenant, session), dedicated, hash); err != nil || commit { + if err := sessions.CreateEnvironmentDevice(t.Context(), BindSession(sqlc.New(tx), tenant, session), uuid.UUID(environment.Bytes).String(), dedicated, hash); err != nil || commit { return err } return errRollback }) } - bound := func(session pgtype.UUID) []uuid.UUID { - rows, err := pool.Query(t.Context(), `SELECT runtime_id FROM session_runtime_assignments WHERE session_id = $1`, session) + created := func(session pgtype.UUID) []uuid.UUID { + rows, err := pool.Query(t.Context(), `SELECT d.id FROM devices d JOIN environments e ON e.id = d.environment_id WHERE e.session_id = $1`, session) if err != nil { t.Fatal(err) } @@ -276,15 +276,15 @@ func TestCreateEnvironmentDeviceBindsOneDevice(t *testing.T) { } tenant, session, environment := newEnvironment(t, pool, "openai_hosted", "pending") - if err := create(tenant, session, environment, uuid.New(), false); !errors.Is(err, errRollback) || len(bound(session)) != 0 { - t.Fatalf("rolled back creation bound %v: %v", bound(session), err) + if err := create(tenant, session, environment, uuid.New(), false); !errors.Is(err, errRollback) || len(created(session)) != 0 { + t.Fatalf("rolled back creation created %v: %v", created(session), err) } - if err := create(pgID(uuid.New()), session, environment, uuid.New(), true); !errors.Is(err, sessions.ErrDeviceBindingConflict) || len(bound(session)) != 0 { - t.Fatalf("other tenant bound %v: %v", bound(session), err) + if err := create(pgID(uuid.New()), session, environment, uuid.New(), true); !errors.Is(err, sessions.ErrDeviceBindingConflict) || len(created(session)) != 0 { + t.Fatalf("other tenant created %v: %v", created(session), err) } device := uuid.New() - if err := create(tenant, session, environment, device, true); err != nil || !reflect.DeepEqual(bound(session), []uuid.UUID{device}) { - t.Fatalf("bound %v: %v", bound(session), err) + if err := create(tenant, session, environment, device, true); err != nil || !reflect.DeepEqual(created(session), []uuid.UUID{device}) { + t.Fatalf("created %v: %v", created(session), err) } if err := create(tenant, session, environment, uuid.New(), true); !errors.Is(err, sessions.ErrDeviceBindingConflict) { t.Fatalf("second device: %v", err) diff --git a/services/core/internal/runtimeenrollment/connection.go b/services/core/internal/runtimeenrollment/connection.go index 7bebb425f..1250acb0c 100644 --- a/services/core/internal/runtimeenrollment/connection.go +++ b/services/core/internal/runtimeenrollment/connection.go @@ -9,21 +9,21 @@ import ( "strings" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/relay" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) type ConnectionStore interface { AuthenticateEnvironmentExecutor(context.Context, string, string) (string, error) GetEnvironment(context.Context, string, string) (sessions.Environment, error) - GetSessionDevice(context.Context, string, string) (sessions.ExecutionDevice, error) - GetDeviceCredential(context.Context, string) (runtimedevice.Credential, bool, error) + GetEnvironmentResource(context.Context, string, string) (runtimedevice.ServeAuthority, error) } -// ConnectionHandler observes an existing binding without enrollment or execution. -// Executor authority never grants access to the public Session API. -func ConnectionHandler(s ConnectionStore, registry *runtimegateway.Registry) http.Handler { +// ConnectionHandler observes an existing enrollment without enrollment or +// execution. Executor authority never grants access to the public Session API. +func ConnectionHandler(s ConnectionStore, links *relay.Relay) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") fail := func(status int) { http.Error(w, http.StatusText(status), status) } @@ -46,7 +46,7 @@ func ConnectionHandler(s ConnectionStore, registry *runtimegateway.Registry) htt digest := runtimedevice.HashCredential(authorization[1]) ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() - connected, err := RuntimeConnected(ctx, s, registry, environment, digest) + connected, err := RuntimeConnected(ctx, s, links, environment, digest) switch { case errors.Is(err, sessions.ErrNotFound): fail(http.StatusUnauthorized) @@ -68,9 +68,13 @@ func ConnectionHandler(s ConnectionStore, registry *runtimegateway.Registry) htt }) } -// RuntimeConnected observes current executor authority and a matching open peer. -// It rechecks authority after the peer; callers must not supply a stale transaction. -func RuntimeConnected(ctx context.Context, s ConnectionStore, registry *runtimegateway.Registry, environment, digest string) (bool, error) { +// RuntimeConnected reports whether the sandbox the executor credential +// enrolled serves the Environment: the credential authenticates for the +// Environment, the Environment's live Link resource is that enrollment with +// the same credential, the relay holds its serve peer, and the credential +// still has that authority afterwards. A live resource of another credential +// is ErrDeviceBindingConflict. +func RuntimeConnected(ctx context.Context, s ConnectionStore, links *relay.Relay, environment, digest string) (bool, error) { tenant, err := s.AuthenticateEnvironmentExecutor(ctx, environment, digest) if err != nil { return false, err @@ -82,52 +86,32 @@ func RuntimeConnected(ctx context.Context, s ConnectionStore, registry *runtimeg if current.Status == "failed" || current.Status == "expired" { return false, sessions.ErrNotFound } - bound, err := s.GetSessionDevice(ctx, tenant, current.SessionID) - if errors.Is(err, sessions.ErrNotFound) { - return false, nil - } - if err != nil { - return false, err - } - if bound.EnvironmentID != environment { - return false, sessions.ErrDeviceBindingConflict - } - credential, found, err := s.GetDeviceCredential(ctx, bound.ID) - if err != nil { - return false, err - } - if !found { - return false, sessions.ErrNotFound - } - if credential.CredentialHash != digest { - return false, sessions.ErrDeviceBindingConflict - } - peer, err := registry.LookupDevice(bound.ID) - if errors.Is(err, runtimegateway.ErrDeviceNotRegistered) { - return false, nil - } - if err != nil { + resource, found, err := enrolledResource(ctx, s, tenant, environment, digest) + if err != nil || !found || !links.Serving(resource.Ref()) { return false, err } - if current.Status != "connected" || peer.IsClosed() || !peer.AuthenticatedWith(digest) { - return false, nil - } - // Recheck authority after reading the socket; rotation/revocation never inherits - // the connected observation of a socket authenticated with the former key. + // Recheck authority after the relay; rotation or revocation never + // inherits the serve peer of the former credential. if _, err = s.AuthenticateEnvironmentExecutor(ctx, environment, digest); err != nil { return false, err } - // The executor key can remain valid while the device itself is revoked. - // Recheck the shared authority view too, including Environment retirement. - credential, found, err = s.GetDeviceCredential(ctx, bound.ID) - if err != nil { - return false, err + again, found, err := enrolledResource(ctx, s, tenant, environment, digest) + return err == nil && found && again == resource && links.Serving(resource.Ref()), err +} + +// enrolledResource reads the Environment's live Link resource and reports +// whether it has one; the resource must be an enrollment served with the +// credential. +func enrolledResource(ctx context.Context, s ConnectionStore, tenant, environment, digest string) (sandboxbootstrap.Resource, bool, error) { + authority, err := s.GetEnvironmentResource(ctx, tenant, environment) + if errors.Is(err, sessions.ErrNotFound) { + return sandboxbootstrap.Resource{}, false, nil } - if !found { - return false, sessions.ErrNotFound + if err != nil { + return sandboxbootstrap.Resource{}, false, err } - if credential.CredentialHash != digest { - return false, sessions.ErrDeviceBindingConflict + if authority.Resource.Kind != "enrollment" || authority.CredentialHash != digest { + return sandboxbootstrap.Resource{}, false, sessions.ErrDeviceBindingConflict } - return !peer.IsClosed() && peer.AuthenticatedWith(digest), nil + return authority.Resource, true, nil } diff --git a/services/core/internal/runtimeenrollment/connection_test.go b/services/core/internal/runtimeenrollment/connection_test.go index 4db7b557e..eb36aa14f 100644 --- a/services/core/internal/runtimeenrollment/connection_test.go +++ b/services/core/internal/runtimeenrollment/connection_test.go @@ -3,16 +3,19 @@ package runtimeenrollment import ( "context" "errors" - "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/gorilla/websocket" - "net/http" "net/http/httptest" "strings" "testing" "time" + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/relay" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/sandboxlinktest" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxwire" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -31,11 +34,8 @@ func (s *connectionStub) AuthenticateEnvironmentExecutor(_ context.Context, envi func (*connectionStub) GetEnvironment(context.Context, string, string) (sessions.Environment, error) { return sessions.Environment{ID: "environment", SessionID: "session", Status: "pending"}, nil } -func (*connectionStub) GetSessionDevice(context.Context, string, string) (sessions.ExecutionDevice, error) { - return sessions.ExecutionDevice{}, sessions.ErrNotFound -} -func (*connectionStub) GetDeviceCredential(context.Context, string) (runtimedevice.Credential, bool, error) { - panic("unbound lookup") +func (*connectionStub) GetEnvironmentResource(context.Context, string, string) (runtimedevice.ServeAuthority, error) { + return runtimedevice.ServeAuthority{}, sessions.ErrNotFound } func TestConnectionReadContract(t *testing.T) { @@ -57,7 +57,7 @@ func TestConnectionReadContract(t *testing.T) { req := httptest.NewRequest(tc.method, "/api/v1/agent-daemon/connection?"+tc.query, nil) req.Header.Set("Authorization", tc.bearer) res := httptest.NewRecorder() - ConnectionHandler(s, runtimegateway.NewRegistry()).ServeHTTP(res, req) + ConnectionHandler(s, relay.New(sandboxlinktest.NewAuthority())).ServeHTTP(res, req) if res.Code != tc.code || s.calls != tc.calls || res.Header().Get("Cache-Control") != "no-store" { t.Fatalf("%s %s: %d, %d calls", tc.method, tc.query, res.Code, s.calls) } @@ -70,16 +70,15 @@ func TestConnectionReadContract(t *testing.T) { } } -// A real gateway peer captures its digest at HTTP upgrade. The test store makes -// authority changes at the deterministic post-peer recheck, without timing sleeps. +// The test store changes authority at the post-relay recheck, the second +// authentication, without timing sleeps. type liveConnectionStore struct { - digest string - authCalls int - credentialCalls int - revokeAtRecheck bool - deviceRevokedAtRecheck bool - recheckError error - credentialRecheckError error + resource sandboxbootstrap.Resource + digest string + authCalls int + revokeAtRecheck bool + rotateAtRecheck bool + recheckError error } func (s *liveConnectionStore) AuthenticateEnvironmentExecutor(context.Context, string, string) (string, error) { @@ -92,75 +91,74 @@ func (s *liveConnectionStore) AuthenticateEnvironmentExecutor(context.Context, s return "", sessions.ErrNotFound } } - return "tenant", nil + return s.resource.TenantID, nil } func (s *liveConnectionStore) GetEnvironment(context.Context, string, string) (sessions.Environment, error) { - return sessions.Environment{ID: "environment", SessionID: "session", Status: "connected"}, nil + return sessions.Environment{ID: s.resource.EnvironmentID, SessionID: "session", Status: "connected"}, nil } -func (s *liveConnectionStore) GetSessionDevice(context.Context, string, string) (sessions.ExecutionDevice, error) { - return sessions.ExecutionDevice{ID: "device", EnvironmentID: "environment"}, nil +func (s *liveConnectionStore) GetEnvironmentResource(context.Context, string, string) (runtimedevice.ServeAuthority, error) { + resource := s.resource + if s.rotateAtRecheck && s.authCalls >= 2 { + resource.Generation++ + } + return runtimedevice.ServeAuthority{Resource: resource, CredentialHash: s.digest}, nil } -func (s *liveConnectionStore) GetDeviceCredential(context.Context, string) (runtimedevice.Credential, bool, error) { - s.credentialCalls++ - if s.authCalls >= 2 && s.credentialRecheckError != nil { - return runtimedevice.Credential{}, false, s.credentialRecheckError + +// Connected follows the enrolled sandbox's serve peer at the relay and the +// credential's authority after reading it. +func TestRuntimeConnectedFollowsServeAndCurrentAuthority(t *testing.T) { + resource := sandboxbootstrap.Resource{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), Kind: "enrollment", ID: uuid.NewString(), Generation: 1} + auth := sandboxlinktest.NewAuthority() + auth.AddServe([]byte("fixture-key"), sandboxlink.ServePeer{PeerID: sandboxwire.NewID(), Resource: resource.Ref()}) + srv := sandboxlinktest.StartRelay(t, auth) + ctx, cancel := context.WithCancel(t.Context()) + connected := make(chan struct{}, 1) + done := make(chan struct{}) + go func() { + defer close(done) + hold := func(ctx context.Context, _ sandboxlink.Bind, _ uint64, _ sandboxlink.Stream) { <-ctx.Done() } + _ = sandboxlink.Serve(ctx, sandboxlink.ServeConfig{URL: srv.URL, TLS: srv.TLS, Credential: []byte("fixture-key"), Resource: resource.Ref(), ServerInstanceID: sandboxwire.NewID(), + Services: []sandboxlink.ServiceHandler{{Service: sandboxlink.ServiceFile, Version: 1, Serve: hold}}, + OnConnected: func() { connected <- struct{}{} }, MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond}) + }() + defer func() { cancel(); <-done }() + select { + case <-connected: + case <-time.After(5 * time.Second): + t.Fatal("the sandbox did not serve") } - if s.deviceRevokedAtRecheck && s.authCalls >= 2 { - return runtimedevice.Credential{}, false, nil + for !srv.Relay.Serving(resource.Ref()) { + time.Sleep(10 * time.Millisecond) } - return runtimedevice.Credential{ID: "device", WorkspaceID: "tenant", Type: runtimedevice.RuntimeTypeAgentDaemon, CredentialHash: s.digest}, true, nil -} -func TestRuntimeConnectedCurrentAuthorityAfterPeer(t *testing.T) { - for _, name := range []string{"connected", "rotated before read", "revoked after peer", "device revoked after peer", "retired after peer", "store error after peer", "device store error after peer", "closed"} { - t.Run(name, func(t *testing.T) { - digest := runtimedevice.HashCredential("fixture-key") - s := &liveConnectionStore{digest: digest} - registry := runtimegateway.NewRegistry() - handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Registry: registry, Authenticator: runtimegateway.NewAuthenticator(s)}) - server := httptest.NewServer(http.HandlerFunc(handler.WS)) - defer server.Close() - conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http")+"?device_id=device&version="+proto.Version, http.Header{"Authorization": {"Bearer fixture-key"}}) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - // Wait for the actual registration, not merely the transport upgrade. - ctx, cancel := context.WithTimeout(t.Context(), time.Second) - defer cancel() - peer, err := registry.WaitForDevice(ctx, "device", time.Second) - if err != nil { - t.Fatal(err) - } - defer peer.Close("test complete") - s.authCalls = 0 - var wantErr error - want := name == "connected" - switch name { - case "rotated before read": - s.digest = runtimedevice.HashCredential("new-key") - digest = s.digest - case "revoked after peer": - s.revokeAtRecheck = true - wantErr = sessions.ErrNotFound - case "device revoked after peer", "retired after peer": - s.deviceRevokedAtRecheck = true - wantErr = sessions.ErrNotFound - case "store error after peer": - s.recheckError = errors.New("database unavailable") - wantErr = s.recheckError - case "device store error after peer": - s.credentialRecheckError = errors.New("device authority unavailable") - wantErr = s.credentialRecheckError - case "closed": - peer.Close("closed before observation") - } - connected, err := RuntimeConnected(t.Context(), s, registry, "environment", digest) - if connected != want || !errors.Is(err, wantErr) { - t.Fatalf("connected=%v err=%v", connected, err) + digest := runtimedevice.HashCredential("fixture-key") + unavailable := errors.New("database unavailable") + for _, tc := range []struct { + name string + store liveConnectionStore + want bool + err error + }{ + {name: "connected", store: liveConnectionStore{resource: resource, digest: digest}, want: true}, + {name: "another credential", store: liveConnectionStore{resource: resource, digest: runtimedevice.HashCredential("other-key")}, err: sessions.ErrDeviceBindingConflict}, + {name: "not served", store: liveConnectionStore{resource: withGeneration(resource, 2), digest: digest}}, + {name: "revoked after relay", store: liveConnectionStore{resource: resource, digest: digest, revokeAtRecheck: true}, err: sessions.ErrNotFound}, + {name: "rotated after relay", store: liveConnectionStore{resource: resource, digest: digest, rotateAtRecheck: true}}, + {name: "store error after relay", store: liveConnectionStore{resource: resource, digest: digest, recheckError: unavailable}, err: unavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + s := tc.store + got, err := RuntimeConnected(t.Context(), &s, srv.Relay, resource.EnvironmentID, digest) + if got != tc.want || !errors.Is(err, tc.err) { + t.Fatalf("connected=%v err=%v", got, err) } - if name == "connected" && s.authCalls != 2 { - t.Fatal("post-peer authority was not checked") + if tc.name == "connected" && s.authCalls != 2 { + t.Fatal("authority was not rechecked after the relay") } }) } } + +func withGeneration(resource sandboxbootstrap.Resource, generation uint64) sandboxbootstrap.Resource { + resource.Generation = generation + return resource +} diff --git a/services/core/internal/runtimeenrollment/enrollment.go b/services/core/internal/runtimeenrollment/enrollment.go index 2a2ff3ce6..089007961 100644 --- a/services/core/internal/runtimeenrollment/enrollment.go +++ b/services/core/internal/runtimeenrollment/enrollment.go @@ -9,17 +9,27 @@ import ( "strings" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) type EnrollmentStore interface { - EnrollRuntime(context.Context, string, string) (sessions.RuntimeEnrollment, error) + EnrollRuntime(context.Context, string, string) (sandboxbootstrap.Resource, error) } // EnrollmentHandler is part of our daemon connection contract, not an upstream -// Agents resource. It grants no Session API access and never issues another key. -func EnrollmentHandler(s EnrollmentStore) http.Handler { +// Agents resource. It grants no Session API access and never issues another +// key: the enrolled sandbox serves its Link resource with the executor +// credential. A self_hosted sandbox dials the Link from its own host, so an +// origin without a wss Link enrolls none. +func EnrollmentHandler(s EnrollmentStore, origin deployment.PublicOrigin) http.Handler { + link, err := origin.SandboxLink() + if err == nil && !strings.HasPrefix(link, "wss:") { + err = deployment.ErrNoLink + } + noLink := err != nil return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fail := func(status int) { http.Error(w, http.StatusText(status), status) } if r.Method != http.MethodPost { @@ -41,9 +51,18 @@ func EnrollmentHandler(s EnrollmentStore) http.Handler { fail(http.StatusBadRequest) return } + if noLink { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _ = json.NewEncoder(w).Encode(struct { + Error string `json:"error"` + Detail string `json:"detail"` + }{"no_sandbox_link", "a self_hosted sandbox needs an https public URL"}) + return + } ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() - binding, err := s.EnrollRuntime(ctx, input.EnvironmentID, runtimedevice.HashCredential(authorization[1])) + resource, err := s.EnrollRuntime(ctx, input.EnvironmentID, runtimedevice.HashCredential(authorization[1])) switch { case errors.Is(err, sessions.ErrNotFound): fail(http.StatusUnauthorized) @@ -55,11 +74,9 @@ func EnrollmentHandler(s EnrollmentStore) http.Handler { w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "no-store") _ = json.NewEncoder(w).Encode(struct { - DeviceID string `json:"device_id"` - SessionID string `json:"session_id"` - EnvironmentID string `json:"environment_id"` - WorkspaceDirectory string `json:"workspace_directory"` - }{binding.DeviceID, binding.SessionID, binding.EnvironmentID, binding.WorkspaceDirectory}) + LinkURL string `json:"link_url"` + Resource sandboxbootstrap.Resource `json:"resource"` + }{link, resource}) } }) } diff --git a/services/core/internal/runtimeenrollment/enrollment_test.go b/services/core/internal/runtimeenrollment/enrollment_test.go index 6eb774833..8009f24e9 100644 --- a/services/core/internal/runtimeenrollment/enrollment_test.go +++ b/services/core/internal/runtimeenrollment/enrollment_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -16,42 +18,56 @@ type enrollmentStub struct { err error } -func (s *enrollmentStub) EnrollRuntime(_ context.Context, environment, digest string) (sessions.RuntimeEnrollment, error) { +func (s *enrollmentStub) EnrollRuntime(_ context.Context, environment, digest string) (sandboxbootstrap.Resource, error) { s.calls++ if environment != "environment" || digest != runtimedevice.HashCredential("private-test-token") { - return sessions.RuntimeEnrollment{}, errors.New("unexpected enrollment input") + return sandboxbootstrap.Resource{}, errors.New("unexpected enrollment input") } - return sessions.RuntimeEnrollment{DeviceID: "device", SessionID: "session", EnvironmentID: environment, WorkspaceDirectory: "/workspace"}, s.err + return sandboxbootstrap.Resource{TenantID: "tenant", EnvironmentID: environment, Kind: "enrollment", ID: "enrollment", Generation: 2}, s.err } func TestEnrollmentConnectionContract(t *testing.T) { + origin := func(value string) deployment.PublicOrigin { + o, err := deployment.NewPublicOrigin(value) + if err != nil { + t.Fatal(err) + } + return o + } + https := origin("https://core.example") for _, test := range []struct { name, body, authorization string + origin deployment.PublicOrigin err error status, calls int }{ - {"valid", `{"environment_id":"environment"}`, "Bearer private-test-token", nil, 200, 1}, - {"missing authority", `{"environment_id":"environment"}`, "", nil, 401, 0}, - {"caller binding", `{"environment_id":"environment","session_id":"other"}`, "Bearer private-test-token", nil, 400, 0}, - {"extra input", `{"environment_id":"environment"}{}`, "Bearer private-test-token", nil, 400, 0}, - {"foreign", `{"environment_id":"environment"}`, "Bearer private-test-token", sessions.ErrNotFound, 401, 1}, - {"conflict", `{"environment_id":"environment"}`, "Bearer private-test-token", sessions.ErrDeviceBindingConflict, 409, 1}, - {"internal failure", `{"environment_id":"environment"}`, "Bearer private-test-token", errors.New("private database detail"), 503, 1}, + {"valid", `{"environment_id":"environment"}`, "Bearer private-test-token", https, nil, 200, 1}, + // A self_hosted sandbox dials the Link from its own host. + {"no wss Link", `{"environment_id":"environment"}`, "Bearer private-test-token", origin("http://127.0.0.1:8080"), nil, 503, 0}, + {"missing authority", `{"environment_id":"environment"}`, "", https, nil, 401, 0}, + {"caller binding", `{"environment_id":"environment","session_id":"other"}`, "Bearer private-test-token", https, nil, 400, 0}, + {"extra input", `{"environment_id":"environment"}{}`, "Bearer private-test-token", https, nil, 400, 0}, + {"foreign", `{"environment_id":"environment"}`, "Bearer private-test-token", https, sessions.ErrNotFound, 401, 1}, + {"conflict", `{"environment_id":"environment"}`, "Bearer private-test-token", https, sessions.ErrDeviceBindingConflict, 409, 1}, + {"internal failure", `{"environment_id":"environment"}`, "Bearer private-test-token", https, errors.New("private database detail"), 503, 1}, } { t.Run(test.name, func(t *testing.T) { s := &enrollmentStub{err: test.err} req := httptest.NewRequest("POST", "/api/v1/agent-daemon/enroll", strings.NewReader(test.body)) req.Header.Set("Authorization", test.authorization) response := httptest.NewRecorder() - EnrollmentHandler(s).ServeHTTP(response, req) + EnrollmentHandler(s, test.origin).ServeHTTP(response, req) if response.Code != test.status || s.calls != test.calls { t.Fatalf("status/calls %d/%d", response.Code, s.calls) } if strings.Contains(response.Body.String(), "private") { t.Fatal("enrollment leaked confidential details") } - if response.Code == 200 && (response.Body.String() != `{"device_id":"device","session_id":"session","environment_id":"environment","workspace_directory":"/workspace"}`+"\n" || response.Header().Get("Cache-Control") != "no-store") { - t.Fatalf("binding response %s", response.Body.String()) + if response.Code == 200 && (response.Body.String() != `{"link_url":"wss://core.example/api/v1/sandbox-link","resource":{"tenant_id":"tenant","environment_id":"environment","kind":"enrollment","id":"enrollment","generation":2}}`+"\n" || response.Header().Get("Cache-Control") != "no-store") { + t.Fatalf("enrollment response %s", response.Body.String()) + } + if test.name == "no wss Link" && !strings.Contains(response.Body.String(), `"error":"no_sandbox_link"`) { + t.Fatalf("no Link response %s", response.Body.String()) } }) } diff --git a/services/core/internal/runtimegateway/runtime_prepare.go b/services/core/internal/runtimegateway/runtime_prepare.go index 6fe0e38da..b05e6e3e1 100644 --- a/services/core/internal/runtimegateway/runtime_prepare.go +++ b/services/core/internal/runtimegateway/runtime_prepare.go @@ -42,12 +42,13 @@ func (s *Session) PrepareRuntime(ctx context.Context, id string, ref proto.Assig s.capabilitiesMu.Unlock() return unknown, ErrSessionClosed } - if len(s.capabilities) != 0 { - s.capabilitiesMu.Unlock() - return unknown, errors.New("agentdaemon gateway: Runtime preparation capacity") - } + // An agent host prepares many Sessions' Environments on one connection; + // the Runtime bounds its transfers per Session and per connection. replies := make(chan proto.Envelope, 1) - s.capabilities = map[string]chan proto.Envelope{id: replies} + if s.capabilities == nil { + s.capabilities = map[string]chan proto.Envelope{} + } + s.capabilities[id] = replies s.capabilitiesMu.Unlock() defer func() { s.capabilitiesMu.Lock(); delete(s.capabilities, id); s.capabilitiesMu.Unlock() }() ctx, cancel := context.WithTimeout(ctx, 195*time.Second) diff --git a/services/core/internal/runtimegateway/runtime_prepare_test.go b/services/core/internal/runtimegateway/runtime_prepare_test.go index 2ef92bc92..d5708c072 100644 --- a/services/core/internal/runtimegateway/runtime_prepare_test.go +++ b/services/core/internal/runtimegateway/runtime_prepare_test.go @@ -201,31 +201,36 @@ func TestCapabilitiesRejectsWrongChunkReceipt(t *testing.T) { noCapabilityFrame(t, s) } -func TestCapabilitiesConnectionOwnershipAndUnknownInterruption(t *testing.T) { +func TestCapabilitiesConcurrentTransfersAndUnknownInterruption(t *testing.T) { for _, closeConnection := range []bool{false, true} { t.Run(map[bool]string{false: "deadline", true: "disconnect"}[closeConnection], func(t *testing.T) { s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) defer s.Close("test") ctx, cancel := context.WithTimeout(t.Context(), 150*time.Millisecond) defer cancel() - done := beginCapabilities(s, ctx, uuid.NewString(), skillPreparation(), []byte("data")) + first := beginCapabilities(s, ctx, uuid.NewString(), skillPreparation(), []byte("data")) nextCapabilityFrame(t, s) - if _, err := s.PrepareRuntime(t.Context(), uuid.NewString(), testAssignment, skillPreparation(), []byte("second")); err == nil { - t.Fatal("concurrent transfer admitted") + // An agent host prepares other Sessions' Environments on the same connection. + id := uuid.NewString() + second := beginCapabilities(s, ctx, id, skillPreparation(), []byte("second")) + if env := nextCapabilityFrame(t, s); env.ID != id { + t.Fatal("concurrent preparation refused") } if closeConnection { s.Close("lost connection") } - result := finishCapabilities(t, done) - if result.result.Outcome != "unknown" || result.result.ErrorCode != "runtime_preparation_unconfirmed" { - t.Fatal("interruption claimed rejection", result) - } expected := error(context.DeadlineExceeded) if closeConnection { expected = ErrSessionClosed } - if !errors.Is(result.err, expected) { - t.Fatal(result.err) + for _, done := range []<-chan capabilityOutcome{first, second} { + result := finishCapabilities(t, done) + if result.result.Outcome != "unknown" || result.result.ErrorCode != "runtime_preparation_unconfirmed" { + t.Fatal("interruption claimed rejection", result) + } + if !errors.Is(result.err, expected) { + t.Fatal(result.err) + } } noCapabilityFrame(t, s) s.capabilitiesMu.Lock() diff --git a/services/core/internal/sessions/devices.go b/services/core/internal/sessions/devices.go index dc8ba7d40..91214e807 100644 --- a/services/core/internal/sessions/devices.go +++ b/services/core/internal/sessions/devices.go @@ -29,22 +29,6 @@ func NewDeviceRegistration(name, credentialHash string) (DeviceRegistration, err return DeviceRegistration{Name: name, CredentialHash: hex.EncodeToString(digest)}, nil } -// RuntimeEnrollment is the resource binding an enrolled user-managed Runtime -// receives. It never carries another secret. -type RuntimeEnrollment struct { - DeviceID string - SessionID string - EnvironmentID string - WorkspaceDirectory string -} - -// EnrolledRuntimeBinding identifies user-managed compute, without an -// allocation or any promise of live authorization. The Worker rechecks the -// socket's key. -type EnrolledRuntimeBinding struct { - DeviceID, TenantID, EnvironmentID, SessionID string -} - // SandboxResource is a live Link resource. Quiesced compute is between a // quiesce and the wake that resumes it. type SandboxResource struct { @@ -52,13 +36,6 @@ type SandboxResource struct { Quiesced bool } -// EnrollmentAuthority is what an executor credential authorizes when it -// enrolls a Runtime: the key and the Environment's workspace directory. -type EnrollmentAuthority struct { - KeyID string - WorkspaceDirectory string -} - // DeviceReader reads Runtime devices and their Session bindings. type DeviceReader interface { // GetSessionDevice reads the Runtime device bound to the Session once its @@ -77,19 +54,19 @@ type DeviceReader interface { // Session leaves the device's exact authenticated delivery of one of // runIDs; without one it is the zero receipt. ArchivedCancellationReceipt(ctx context.Context, device, credentialHash string, runIDs []string) (runtimedevice.ArchivedCancellationReceipt, error) - // ListEnrolledRuntimeBindings lists the enrolled user-managed Runtimes of - // live Environments. - ListEnrolledRuntimeBindings(ctx context.Context) ([]EnrolledRuntimeBinding, error) // ListLiveSandboxResources lists the live Link resources. ListLiveSandboxResources(ctx context.Context) ([]SandboxResource, error) + // GetEnvironmentResource reads the live Link resource of the tenant's + // Environment; without one it is ErrNotFound. + GetEnvironmentResource(ctx context.Context, tenant, environment string) (runtimedevice.ServeAuthority, error) // GetSessionExecutionBinding reads the Runtime device that executes the // Session's Turns, with the native session that continues its history, // once its Environment preparation completed; before that, and without an // authorized bound device, it is ErrNotFound. GetSessionExecutionBinding(ctx context.Context, tenant, session string) (ExecutionBinding, error) - // ListExecutionDevices lists the tenant's unrevoked devices that belong - // to no Environment, in ID order. - ListExecutionDevices(ctx context.Context, tenant string) ([]ExecutionDevice, error) + // ListAgentHosts lists the unrevoked agent hosts that may run the + // tenant's Sessions, in ID order. + ListAgentHosts(ctx context.Context, tenant string) ([]ExecutionDevice, error) } // DeviceStorage stores Runtime devices. @@ -115,18 +92,15 @@ type DeviceStorage interface { // EnrollmentTx is the Session transaction EnrollRuntime runs in. type EnrollmentTx interface { // AuthorizeEnrollment rechecks, under the Session lock, that the - // credential still authorizes enrolling a Runtime for the live self_hosted - // Environment, and holds the key's lock until the transaction ends, so - // revocation cannot race enrollment. Without that authority it is - // ErrNotFound. - AuthorizeEnrollment(ctx context.Context) (EnrollmentAuthority, error) - // EnrollDevice creates the Environment's user-managed Runtime device for - // the key, or returns the device the same key already enrolled. A device - // of another key, or a revoked one, is ErrDeviceBindingConflict. - EnrollDevice(ctx context.Context, key string) (string, error) - // BindDevice binds the device to the Session. A Session bound to another - // device is ErrDeviceBindingConflict. - BindDevice(ctx context.Context, device string) error + // credential still authorizes enrolling a sandbox for the live + // self_hosted Environment, returns its key and holds the key's lock until + // the transaction ends, so revocation cannot race enrollment. Without + // that authority it is ErrNotFound. + AuthorizeEnrollment(ctx context.Context) (string, error) + // EnrollSandbox records the Environment's enrollment by the key and + // returns its Link resource, or the resource the same key already + // enrolled. An enrollment by another key is ErrDeviceBindingConflict. + EnrollSandbox(ctx context.Context, key string) (sandboxbootstrap.Resource, error) } // CreateDevice provisions a device for an operator. It is not a tenant-facing @@ -171,29 +145,24 @@ func heartbeatStatus(current bool) runtimedevice.HeartbeatStatus { return runtimedevice.HeartbeatStatus{Liveness: "online", Deleted: !current} } -// EnrollRuntime binds a user-managed Runtime to one self_hosted Environment -// with the executor credential whose digest is credentialHash. A retry keeps -// the same device and key; it cannot replace compute or adopt another native -// history. -func (s *Service) EnrollRuntime(ctx context.Context, environment, credentialHash string) (RuntimeEnrollment, error) { - var result RuntimeEnrollment - err := s.storage.WithEnrollment(ctx, environment, credentialHash, func(ctx context.Context, tx EnrollmentTx, current Environment, locked LockedSession) error { +// EnrollRuntime enrolls a user-managed sandbox for one self_hosted +// Environment with the executor credential whose digest is credentialHash, +// and returns the Link resource the sandbox serves with that credential. The +// first key to enroll keeps the Environment: a retry with it returns the +// same resource, and another key is ErrDeviceBindingConflict. Placement binds +// the Session to an agent host once the sandbox serves. +func (s *Service) EnrollRuntime(ctx context.Context, environment, credentialHash string) (sandboxbootstrap.Resource, error) { + var result sandboxbootstrap.Resource + err := s.storage.WithEnrollment(ctx, environment, credentialHash, func(ctx context.Context, tx EnrollmentTx, _ Environment, locked LockedSession) error { if err := locked.Public(); err != nil { return err } - authority, err := tx.AuthorizeEnrollment(ctx) - if err != nil { - return err - } - device, err := tx.EnrollDevice(ctx, authority.KeyID) + key, err := tx.AuthorizeEnrollment(ctx) if err != nil { return err } - if err := tx.BindDevice(ctx, device); err != nil { - return err - } - result = RuntimeEnrollment{DeviceID: device, SessionID: current.SessionID, EnvironmentID: current.ID, WorkspaceDirectory: authority.WorkspaceDirectory} - return nil + result, err = tx.EnrollSandbox(ctx, key) + return err }) return result, err } diff --git a/services/core/internal/sessions/devices_test.go b/services/core/internal/sessions/devices_test.go index 0ea72fcd4..9f62d31fd 100644 --- a/services/core/internal/sessions/devices_test.go +++ b/services/core/internal/sessions/devices_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" ) @@ -192,9 +193,10 @@ func TestDeviceUseCases(t *testing.T) { } } -func TestEnrollRuntimeBindsUnderTheSessionLock(t *testing.T) { +func TestEnrollRuntimeEnrollsUnderTheSessionLock(t *testing.T) { environment := Environment{ID: "environment", SessionID: "session"} - authorized := returns(EnrollmentAuthority{KeyID: "key", WorkspaceDirectory: "/workspace"}) + authorized := returns("key") + resource := sandboxbootstrap.Resource{TenantID: "tenant", EnvironmentID: "environment", Kind: "enrollment", ID: "enrollment", Generation: 1} for _, test := range []struct { name string tx fakeTx @@ -202,15 +204,15 @@ func TestEnrollRuntimeBindsUnderTheSessionLock(t *testing.T) { want error calls []string }{ - {"enrolls", fakeTx{authorizeEnrollment: authorized, enrollDevice: returns("device"), bindDevice: done}, - LockedSession{}, nil, []string{"AuthorizeEnrollment", "EnrollDevice key", "BindDevice device"}}, + {"enrolls", fakeTx{authorizeEnrollment: authorized, enrollSandbox: returns(resource)}, + LockedSession{}, nil, []string{"AuthorizeEnrollment", "EnrollSandbox key"}}, {"deleted Session", fakeTx{}, LockedSession{Deleted: true}, ErrNotFound, nil}, - {"revoked key", fakeTx{authorizeEnrollment: func() (EnrollmentAuthority, error) { return EnrollmentAuthority{}, ErrNotFound }}, + {"revoked key", fakeTx{authorizeEnrollment: func() (string, error) { return "", ErrNotFound }}, LockedSession{}, ErrNotFound, []string{"AuthorizeEnrollment"}}, - {"device of another key", fakeTx{authorizeEnrollment: authorized, enrollDevice: func() (string, error) { return "", ErrDeviceBindingConflict }}, - LockedSession{}, ErrDeviceBindingConflict, []string{"AuthorizeEnrollment", "EnrollDevice key"}}, - {"Session bound elsewhere", fakeTx{authorizeEnrollment: authorized, enrollDevice: returns("device"), bindDevice: func() error { return ErrDeviceBindingConflict }}, - LockedSession{}, ErrDeviceBindingConflict, []string{"AuthorizeEnrollment", "EnrollDevice key", "BindDevice device"}}, + {"enrollment of another key", fakeTx{authorizeEnrollment: authorized, enrollSandbox: func() (sandboxbootstrap.Resource, error) { + return sandboxbootstrap.Resource{}, ErrDeviceBindingConflict + }}, + LockedSession{}, ErrDeviceBindingConflict, []string{"AuthorizeEnrollment", "EnrollSandbox key"}}, } { t.Run(test.name, func(t *testing.T) { tx := test.tx @@ -220,10 +222,10 @@ func TestEnrollRuntimeBindsUnderTheSessionLock(t *testing.T) { if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) { t.Fatalf("got %v, want %v", err, test.want) } - if want := (RuntimeEnrollment{DeviceID: "device", SessionID: "session", EnvironmentID: "environment", WorkspaceDirectory: "/workspace"}); test.want == nil && enrolled != want { + if test.want == nil && enrolled != resource { t.Fatalf("enrollment %+v", enrolled) } - if test.want != nil && enrolled != (RuntimeEnrollment{}) { + if test.want != nil && enrolled != (sandboxbootstrap.Resource{}) { t.Fatalf("failed enrollment returned %+v", enrolled) } if storage.calls[0] != "WithEnrollment environment "+credentialDigest { diff --git a/services/core/internal/sessions/environment_device.go b/services/core/internal/sessions/environment_device.go index ee6377390..aab8cf5d3 100644 --- a/services/core/internal/sessions/environment_device.go +++ b/services/core/internal/sessions/environment_device.go @@ -9,17 +9,16 @@ type EnvironmentDeviceTx interface { // that still has authority. LoadBoundDevice(ctx context.Context) (bool, error) // InsertEnvironmentDevice inserts device, with the credential hash, as the - // dedicated Runtime device of the Session's hosted Environment and binds - // it to the Session. An Environment that already has a device or cannot - // take one is ErrDeviceBindingConflict. - InsertEnvironmentDevice(ctx context.Context, device ExecutionDevice, credentialHash string) error + // dedicated Runtime device of the Session's hosted Environment. An + // Environment that already has a device or cannot take one is + // ErrDeviceBindingConflict. + InsertEnvironmentDevice(ctx context.Context, environment string, device ExecutionDevice, credentialHash string) error } // CreateEnvironmentDevice creates the dedicated Runtime device of the -// Session's hosted Environment and binds it to the Session. A Session that is -// already bound to a device is ErrDeviceBindingConflict, so an existing -// credential is never widened. -func CreateEnvironmentDevice(ctx context.Context, tx EnvironmentDeviceTx, device ExecutionDevice, credentialHash string) error { +// Session's hosted Environment. A Session that is already bound to a device +// is ErrDeviceBindingConflict, so an existing credential is never widened. +func CreateEnvironmentDevice(ctx context.Context, tx EnvironmentDeviceTx, environment string, device ExecutionDevice, credentialHash string) error { bound, err := tx.LoadBoundDevice(ctx) if err != nil { return err @@ -27,5 +26,5 @@ func CreateEnvironmentDevice(ctx context.Context, tx EnvironmentDeviceTx, device if bound { return ErrDeviceBindingConflict } - return tx.InsertEnvironmentDevice(ctx, device, credentialHash) + return tx.InsertEnvironmentDevice(ctx, environment, device, credentialHash) } diff --git a/services/core/internal/sessions/environment_device_test.go b/services/core/internal/sessions/environment_device_test.go index 14e9e4088..01a12db12 100644 --- a/services/core/internal/sessions/environment_device_test.go +++ b/services/core/internal/sessions/environment_device_test.go @@ -7,15 +7,15 @@ import ( // A Session that is already bound to a device never gets a second one. func TestCreateEnvironmentDevice(t *testing.T) { - device := ExecutionDevice{ID: "device", Name: "runtime", EnvironmentID: "environment"} + device := ExecutionDevice{ID: "device", Name: "runtime"} free := &fakeTx{t: t, loadBoundDevice: returns(false), insertEnvironmentDevice: done} - if err := CreateEnvironmentDevice(t.Context(), free, device, "hash"); err != nil { + if err := CreateEnvironmentDevice(t.Context(), free, "environment", device, "hash"); err != nil { t.Fatal(err) } assertCalls(t, free, "LoadBoundDevice", "InsertEnvironmentDevice device runtime environment hash") bound := &fakeTx{t: t, loadBoundDevice: returns(true)} - if err := CreateEnvironmentDevice(t.Context(), bound, device, "hash"); !errors.Is(err, ErrDeviceBindingConflict) { + if err := CreateEnvironmentDevice(t.Context(), bound, "environment", device, "hash"); !errors.Is(err, ErrDeviceBindingConflict) { t.Fatal(err) } assertCalls(t, bound, "LoadBoundDevice") diff --git a/services/core/internal/sessions/execution_environment_test.go b/services/core/internal/sessions/execution_environment_test.go index d62ecad03..83a1bd155 100644 --- a/services/core/internal/sessions/execution_environment_test.go +++ b/services/core/internal/sessions/execution_environment_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" ) func (f *fakeTx) LoadSessionDevice(context.Context) (ExecutionDevice, bool, error) { @@ -71,14 +72,14 @@ func (f *fakeTx) BindDevice(_ context.Context, device string) error { return f.bindDevice() } -func (f *fakeTx) AuthorizeEnrollment(context.Context) (EnrollmentAuthority, error) { +func (f *fakeTx) AuthorizeEnrollment(context.Context) (string, error) { f.record("AuthorizeEnrollment", f.authorizeEnrollment != nil) return f.authorizeEnrollment() } -func (f *fakeTx) EnrollDevice(_ context.Context, key string) (string, error) { - f.record("EnrollDevice", f.enrollDevice != nil, key) - return f.enrollDevice() +func (f *fakeTx) EnrollSandbox(_ context.Context, key string) (sandboxbootstrap.Resource, error) { + f.record("EnrollSandbox", f.enrollSandbox != nil, key) + return f.enrollSandbox() } // environmentTx is the transaction the Environment family's With methods @@ -180,7 +181,7 @@ func TestInitializationOwnerAndDeviceRules(t *testing.T) { func TestInitializationTransitionsRunUnderTheListedDevice(t *testing.T) { owner := EnvironmentInitialization{TenantID: "tenant", SessionID: "session", EnvironmentID: "environment", DeviceID: "device"} listed := func() (ExecutionDevice, bool, error) { - return ExecutionDevice{ID: "device", EnvironmentID: "environment"}, true, nil + return ExecutionDevice{ID: "device"}, true, nil } checked := []string{"WithInitialization tenant session", "LoadEnvironment", "LoadSessionDevice"} for _, test := range []struct { @@ -200,7 +201,7 @@ func TestInitializationTransitionsRunUnderTheListedDevice(t *testing.T) { {"complete", fakeTx{loadEnvironment: returns(hostedEnvironment), loadSessionDevice: listed, completeInitialization: returns(true)}, LockedSession{}, func(o *ExecutionOperations) error { return o.CompleteEnvironmentInitialization(t.Context(), owner) }, nil, append(checked, "CompleteInitialization environment")}, - {"device moved", fakeTx{loadEnvironment: returns(hostedEnvironment), loadSessionDevice: loads(ExecutionDevice{ID: "other", EnvironmentID: "environment"}, true)}, + {"device moved", fakeTx{loadEnvironment: returns(hostedEnvironment), loadSessionDevice: loads(ExecutionDevice{ID: "other"}, true)}, LockedSession{}, func(o *ExecutionOperations) error { return o.CompleteEnvironmentInitialization(t.Context(), owner) }, ErrTurnConflict, checked}, {"deleted Session", fakeTx{}, diff --git a/services/core/internal/sessions/execution_file_writes.go b/services/core/internal/sessions/execution_file_writes.go index 3546d78af..e6e1fb68e 100644 --- a/services/core/internal/sessions/execution_file_writes.go +++ b/services/core/internal/sessions/execution_file_writes.go @@ -90,9 +90,9 @@ func (o *ExecutionOperations) ReserveEnvironmentFileWrite(ctx context.Context, t } // admitFileWrite admits a new write to the Session's live Environment through -// the device the key names, dedicated to that Environment, while the Session -// is idle with no pending input. A failed or expired Environment is -// ErrInvalidInput, a Session without a device ErrNotFound, another device +// the Session's bound Runtime, which the key names, while the Session is idle +// with no pending input. A failed or expired Environment is ErrInvalidInput, a +// Session without a bound Runtime ErrNotFound, another Runtime // ErrDeviceBindingConflict, and work in progress ErrTurnConflict. func admitFileWrite(ctx context.Context, tx FileWriteReservationTx, owner Environment, key FileWriteIdentity) error { current, err := tx.LoadEnvironment(ctx) @@ -109,7 +109,7 @@ func admitFileWrite(ctx context.Context, tx FileWriteReservationTx, owner Enviro if !found { return ErrNotFound } - if bound.ID != key.DeviceID || bound.EnvironmentID != owner.ID { + if bound.ID != key.DeviceID { return ErrDeviceBindingConflict } if err := CheckComputeAdmission(ctx, tx); err != nil { diff --git a/services/core/internal/sessions/execution_file_writes_test.go b/services/core/internal/sessions/execution_file_writes_test.go index 9fab5574f..01312c70d 100644 --- a/services/core/internal/sessions/execution_file_writes_test.go +++ b/services/core/internal/sessions/execution_file_writes_test.go @@ -53,7 +53,7 @@ func TestReserveEnvironmentFileWrite(t *testing.T) { reserve := "WithFileWriteReservation " + testTenant + " " + testEnvironment admission := []string{reserve, "LoadFileWrite " + testFileWrite, "LoadEnvironment", "LoadSessionDevice", "LoadComputeSuspension", "LoadPendingFileWrite", "LoadActiveTurn", "LoadPendingInput"} pending := EnvironmentFileWrite{Identity: testWriteKey, EnvironmentID: testEnvironment, SessionID: testSession, State: "pending"} - bound := loads(ExecutionDevice{ID: testDevice, EnvironmentID: testEnvironment}, true) + bound := loads(ExecutionDevice{ID: testDevice}, true) admitted := func(tx fakeTx) fakeTx { tx.loadFileWrite = loads(EnvironmentFileWrite{}, false) if tx.loadEnvironment == nil { @@ -88,7 +88,7 @@ func TestReserveEnvironmentFileWrite(t *testing.T) { {"a replaced Environment", admitted(fakeTx{loadEnvironment: returns(Environment{ID: testKey, Status: "ready"})}), LockedSession{}, ErrInvalidInput, false, admission[:3]}, {"a failed Environment", admitted(fakeTx{loadEnvironment: returns(Environment{ID: testEnvironment, Status: "failed"})}), LockedSession{}, ErrInvalidInput, false, admission[:3]}, {"no device", admitted(fakeTx{loadSessionDevice: loads(ExecutionDevice{}, false)}), LockedSession{}, ErrNotFound, false, admission[:4]}, - {"another device", admitted(fakeTx{loadSessionDevice: loads(ExecutionDevice{ID: testKey, EnvironmentID: testEnvironment}, true)}), LockedSession{}, ErrDeviceBindingConflict, false, admission[:4]}, + {"another device", admitted(fakeTx{loadSessionDevice: loads(ExecutionDevice{ID: testKey}, true)}), LockedSession{}, ErrDeviceBindingConflict, false, admission[:4]}, {"suspended compute", func() fakeTx { tx := admitted(fakeTx{}); tx.loadComputeSuspension = returns(true); return tx }(), LockedSession{}, ErrTurnConflict, false, admission[:5]}, {"pending input", idle(fakeTx{loadPendingInput: returns(true)}), LockedSession{}, ErrTurnConflict, false, admission}, } { diff --git a/services/core/internal/sessions/executor_credentials.go b/services/core/internal/sessions/executor_credentials.go index bfb38d8ff..68561f6ea 100644 --- a/services/core/internal/sessions/executor_credentials.go +++ b/services/core/internal/sessions/executor_credentials.go @@ -44,15 +44,15 @@ type ExecutorCredential struct { RevokedAt *time.Time `json:"revoked_at" extensions:"x-nullable"` } -// ExecutorConnectionState is an internal durable observation, never a wire payload. -// In particular the current credential digest must not be serialized. +// ExecutorConnectionState is the Environment's enrollment: the key that +// enrolled it and when, and the digest of the credential that may Serve its +// Link resource while that resource is live. It is internal and never a wire +// payload; in particular the credential digest must not be serialized. type ExecutorConnectionState struct { - DeviceID string `json:"-"` - BoundKeyID *string `json:"-"` - EnrolledAt *time.Time `json:"-"` - LastSeenAt *time.Time `json:"-"` - CredentialHash string `json:"-"` - EnvironmentStatus string `json:"-"` + Enrolled bool `json:"-"` + BoundKeyID *string `json:"-"` + EnrolledAt *time.Time `json:"-"` + CredentialHash string `json:"-"` } type ExecutorCredentialState struct { diff --git a/services/core/internal/sessions/transaction_test.go b/services/core/internal/sessions/transaction_test.go index 0fba5f086..fc3895cb8 100644 --- a/services/core/internal/sessions/transaction_test.go +++ b/services/core/internal/sessions/transaction_test.go @@ -10,6 +10,7 @@ import ( "time" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/items" ) @@ -51,8 +52,8 @@ type fakeTx struct { setConnectionStatus func() error loadDevice func() (bool, error) bindDevice func() error - authorizeEnrollment func() (EnrollmentAuthority, error) - enrollDevice func() (string, error) + authorizeEnrollment func() (string, error) + enrollSandbox func() (sandboxbootstrap.Resource, error) loadFileWrite func() (EnvironmentFileWrite, bool, error) loadPendingInput func() (bool, error) createFileWrite func() (EnvironmentFileWrite, error) @@ -216,8 +217,8 @@ func (f *fakeTx) LoadBoundDevice(context.Context) (bool, error) { return f.loadBoundDevice() } -func (f *fakeTx) InsertEnvironmentDevice(_ context.Context, device ExecutionDevice, credentialHash string) error { - f.record("InsertEnvironmentDevice", f.insertEnvironmentDevice != nil, device.ID, device.Name, device.EnvironmentID, credentialHash) +func (f *fakeTx) InsertEnvironmentDevice(_ context.Context, environment string, device ExecutionDevice, credentialHash string) error { + f.record("InsertEnvironmentDevice", f.insertEnvironmentDevice != nil, device.ID, device.Name, environment, credentialHash) return f.insertEnvironmentDevice() } diff --git a/services/core/internal/sessions/turn.go b/services/core/internal/sessions/turn.go index 6d9e76e77..290753dbc 100644 --- a/services/core/internal/sessions/turn.go +++ b/services/core/internal/sessions/turn.go @@ -84,14 +84,12 @@ type ExecutionEvent struct { type ExecutionWork struct{ TenantID, SessionID, TurnID, Status string } // ExecutionDevice contains safe identity only, never a device credential. -// EnvironmentID is the device's own Environment, empty for a device of none. // Assignment is the Session's bound assignment to the device, and // SessionEnvironmentID the Session's Environment, which that assignment binds; // it is empty for a Session without one. type ExecutionDevice struct { ID string Name string - EnvironmentID string Assignment proto.AssignmentRef SessionEnvironmentID string } diff --git a/services/core/migrations/000097_sandbox_link_authority.sql b/services/core/migrations/000097_sandbox_link_authority.sql index 7325afa51..375191145 100644 --- a/services/core/migrations/000097_sandbox_link_authority.sql +++ b/services/core/migrations/000097_sandbox_link_authority.sql @@ -13,7 +13,8 @@ CREATE TABLE sandbox_enrollments ( id uuid PRIMARY KEY, environment_id uuid NOT NULL UNIQUE REFERENCES environments(id), executor_key_id uuid NOT NULL REFERENCES environment_executor_credentials(key_id), - generation bigint NOT NULL DEFAULT 1 CHECK (generation > 0) + generation bigint NOT NULL DEFAULT 1 CHECK (generation > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() ); ALTER TABLE devices @@ -43,6 +44,23 @@ JOIN environments e ON e.id = n.environment_id JOIN sessions s ON s.id = e.session_id JOIN environment_executor_credentials c ON c.key_id = n.executor_key_id; +-- Sessions run on an agent host. A Session bound to an in-sandbox or +-- user-managed Runtime cannot move, so the upgrade waits until those Sessions +-- are deleted. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM session_runtime_assignments b + JOIN sessions s ON s.id = b.session_id + JOIN devices d ON d.id = b.runtime_id + WHERE b.desired_state = 'bound' AND s.deleted_at IS NULL AND NOT d.agent_host + ) THEN + RAISE EXCEPTION 'Sessions are bound to a Runtime that is not an agent host; delete those Sessions, then upgrade'; + END IF; +END $$; +-- +goose StatementEnd + -- +goose Down DROP VIEW sandbox_resources; DELETE FROM devices WHERE tenant_id IS NULL; diff --git a/services/core/tests/integration/agent_host_migration_test.go b/services/core/tests/integration/agent_host_migration_test.go new file mode 100644 index 000000000..2eacae462 --- /dev/null +++ b/services/core/tests/integration/agent_host_migration_test.go @@ -0,0 +1,42 @@ +package integration + +import ( + "database/sql" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +// TestAgentHostMigrationRefusesGuestBoundSessions upgrades a database where a +// Session is bound to a Runtime that is not an agent host: the upgrade refuses +// until that Session is deleted. +func TestAgentHostMigrationRefusesGuestBoundSessions(t *testing.T) { + s, pool := newManagedTestStore(t) + ctx := t.Context() + tenant, session := newTurnSession(t, s) + guest, _ := registerTestDevice(t, s, tenant) + if _, err := pool.Exec(ctx, `INSERT INTO session_runtime_assignments (session_id, runtime_id) VALUES ($1, $2)`, session.ID, guest.ID); err != nil { + t.Fatal(err) + } + db := sql.OpenDB(stdlib.GetConnector(*pool.Config().ConnConfig)) + t.Cleanup(func() { _ = db.Close() }) + provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(ctx, 96); err != nil { + t.Fatal(err) + } + if _, err := provider.Up(ctx); err == nil || !strings.Contains(err.Error(), "delete those Sessions, then upgrade") { + t.Fatal("the upgrade kept a Session bound to a guest Runtime", err) + } + if _, err := pool.Exec(ctx, `UPDATE sessions SET deleted_at = clock_timestamp() WHERE id = $1`, session.ID); err != nil { + t.Fatal(err) + } + if _, err := provider.Up(ctx); err != nil { + t.Fatal(err) + } +} diff --git a/services/core/tests/integration/archive_cancellation_test.go b/services/core/tests/integration/archive_cancellation_test.go deleted file mode 100644 index 445f70a9f..000000000 --- a/services/core/tests/integration/archive_cancellation_test.go +++ /dev/null @@ -1,328 +0,0 @@ -package integration - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - - "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtime" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" - "github.com/google/uuid" - "github.com/gorilla/websocket" -) - -// Controlled protocol fixtures, not native/model acceptance. Allocation setup -// uses the real execution lease; the real dispatcher consumes a function request -// through the real authenticated WebSocket and persists waiting before archive. -func TestArchiveWaitingCancellationReceipts(t *testing.T) { - for _, scenario := range []string{"receipt_without_heartbeat", "heartbeat_before_receipt", "done_heartbeat_ack", "ack_commit_blocked", "rotated", "expired", "transport_lost", "negative_ack", "missing_outcome", "revoke_before_archive", "cancel_revoke_archive", "revoke_after_archive", "revoke_concurrent_archive"} { - t.Run(scenario, func(t *testing.T) { - heartbeat := scenario != "receipt_without_heartbeat" - s, _ := newManagedTestStore(t) - leased := executionOwner(t, s) - t.Cleanup(func() { - if err := leased.Lease.Close(context.Background()); err != nil { - t.Error(err) - } - }) - installation := uuid.NewString() - if err := leased.Deployment.Claim(t.Context(), installation); err != nil { - t.Fatal(err) - } - if _, err := leased.Deployment.Initialize(t.Context(), installation, sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture", Template: "runtime:" + uuid.NewString()}}); err != nil { - t.Fatal(err) - } - projectID := uuid.NewString() - auditCtx := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) - _, management := fixtureProjects(t, s) - project, err := management.CreateProject(auditCtx, projects.CreateProject{ID: projectID, Name: "Archive diagnosis"}) - if err != nil { - t.Fatal(err) - } - configuration := strings.Replace(functionConfiguration, `"type":"none"`, `"type":"openai_hosted","network":{"access":"disabled"}`, 1) - session, err := s.CreateSession(t.Context(), project.TenantID, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(configuration)})) - if err != nil { - t.Fatal(err) - } - secret := uuid.NewString() - owner, err := leased.Deployment.ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: project.TenantID, EnvironmentID: session.Environment.ID}, installation, runtimedevice.HashCredential(secret), runtimedevice.HashCredential(secret)) - if err != nil { - t.Fatal(err) - } - for _, step := range []func(context.Context, deployment.Allocation) (deployment.Allocation, error){leased.Deployment.ObserveRunning, leased.Deployment.SettleCreation} { - owner, err = step(t.Context(), owner) - if err != nil { - t.Fatal(err) - } - } - if err := leased.Sessions.BindSessionDevice(t.Context(), project.TenantID, session.ID, owner.DeviceID); err != nil { - t.Fatal(err) - } - generation := uuid.NewString() - if err := leased.Sessions.ReplaceEnvironmentConnection(t.Context(), project.TenantID, session.Environment.ID, generation); err != nil { - t.Fatal(err) - } - if err := leased.Sessions.ObserveEnvironmentConnection(t.Context(), project.TenantID, session.Environment.ID, generation, 1, true); err != nil { - t.Fatal(err) - } - server := httptest.NewUnstartedServer(nil) - wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" - handler, registry, err := runtime.NewGateway(sessionAdapter(s), sessionService(t, s), sessionAdapter(s), runtimegateway.NewLinkAuthority(sessionAdapter(s)), wsURL) - if err != nil { - t.Fatal(err) - } - server.Config.Handler = handler - server.Start() - t.Cleanup(func() { runtime.CloseConnections(registry); server.Close() }) - u, _ := url.Parse(wsURL) - u.RawQuery = url.Values{"device_id": {owner.DeviceID}, "version": {proto.Version}}.Encode() - conn, _, err := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { conn.Close() }) - sessionStore := sessionAdapter(s) - service, err := newSessionService(s) - if err != nil { - t.Fatal(err) - } - h := &dispatchHarness{t: t, s: s, lease: leased.Lease, owned: &leased, tenant: project.TenantID, session: session, conn: conn, registry: registry, d: &execution.Dispatcher{Registry: registry, Observer: modelconfigurationpg.New(pgunit.NewPool(s.pool), s.credentialCipher), Sessions: service, SessionsReader: sessionStore}} - h.d = h.bound() - capabilities := workerEnvironmentCapabilities() - capabilities.FunctionTools = proto.CapabilitySupported - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{HomeRemoval: proto.CapabilityUnsupported, SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: capabilities}}}) - var peer *runtimegateway.Session - for deadline := time.Now().Add(3 * time.Second); ; { - peer, err = registry.LookupDevice(owner.DeviceID) - if err == nil { - info, _, known := peer.AgentKindStatus("codex") - if known && info.Capabilities.FunctionTools.IsSupported() { - break - } - } - if time.Now().After(deadline) { - t.Fatal("initial heartbeat missing") - } - time.Sleep(time.Millisecond) - } - pending, err := sessionService(t, s).ReserveEnvironmentInput(t.Context(), h.tenant, session.ID, "pending", []sessions.Input{messageInput("first"), messageInput("second")}) - if err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - result := runPreparedDispatch(h, ctx, pending) - frame := h.read(proto.TypeExecutionPrepare) - handle := acknowledgePreparation(h, frame.ID) - start := readyPreparedDispatch(t, h, frame.ID, handle) - h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) - input := sessions.InputReceipt{TurnID: start.RunID} - h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "pending", Name: "lookup_ticket", Arguments: json.RawMessage(`{"ticket":"42"}`)}) - state := functionState(t, h, 1) - if state.LastTurn.Status != sessions.TurnWaiting { - t.Fatal(state.LastTurn) - } - - if scenario == "cancel_revoke_archive" { - if _, err := requestCancel(t.Context(), s, h.tenant, session.ID, "ordinary-cancel"); err != nil { - t.Fatal(err) - } - } - if scenario == "revoke_before_archive" || scenario == "cancel_revoke_archive" { - if err := sessionService(t, s).RevokeDevice(t.Context(), h.tenant, owner.DeviceID); err != nil { - t.Fatal(err) - } - } - var revokeDone chan error - if scenario == "revoke_concurrent_archive" { - revokeDone = make(chan error, 1) - go func() { revokeDone <- sessionService(t, s).RevokeDevice(t.Context(), h.tenant, owner.DeviceID) }() - } - - archived, err := leased.Deployment.ArchiveSession(auditCtx, h.tenant, session.ID, 1) - if err != nil || archived.State != "cleanup_pending" { - t.Fatal(archived, err) - } - if revokeDone != nil { - if err := <-revokeDone; err != nil { - t.Fatal(err) - } - } - if scenario == "revoke_after_archive" { - if err := sessionService(t, s).RevokeDevice(t.Context(), h.tenant, owner.DeviceID); err != nil { - t.Fatal(err) - } - } - // A repeat archive and cleanup must not recreate an explicitly - // cleared marker, nor erase the marker from a fresh archive. - repeatAudit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) - if _, err := leased.Deployment.ArchiveSession(repeatAudit, h.tenant, session.ID, 1); err != nil { - t.Fatal(err) - } - if _, err := leased.Deployment.RequestCleanup(t.Context(), owner); err != nil { - t.Fatal(err) - } - - current, err := sessionAdapter(s).GetTurn(t.Context(), h.tenant, session.ID, input.TurnID) - if err != nil || current.Status != sessions.TurnWaiting || current.CancelRequestedAt.IsZero() { - t.Fatal("archive must request rather than invent cancellation", current, err) - } - if _, err := runtimegateway.NewAuthenticator(sessionAdapter(s)).AuthenticateBearer(t.Context(), owner.DeviceID, secret); !errors.Is(err, runtimegateway.ErrAuthUnknownDevice) { - t.Fatal("archive allowed renewed authority", err) - } - rejected, response, dialErr := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) - if rejected != nil { - rejected.Close() - } - if response != nil { - response.Body.Close() - } - if dialErr == nil || response == nil || response.StatusCode != http.StatusUnauthorized { - t.Fatal("revoked Runtime reconnected") - } - drain, err := sessionAdapter(s).ArchivedCancellationReceipt(t.Context(), owner.DeviceID, secret, nil) - if err != nil || drain.RunID != "" { - t.Fatal("unowned delivery got receipt permission", drain, err) - } - drain, err = sessionAdapter(s).ArchivedCancellationReceipt(t.Context(), owner.DeviceID, runtimedevice.HashCredential(secret), []string{input.TurnID}) - if err != nil || (drain.RunID == input.TurnID) == strings.Contains(scenario, "revoke") { - t.Fatal("archive revocation causality lost", drain, err) - } - if drain.RunID != "" && !drain.Deadline.Equal(current.CancelRequestedAt.Add(runtimedevice.ArchivedCancellationReceiptLimit)) { - t.Fatal("archive renewed cancellation deadline") - } - // Explicitly observe cancel delivery before inducing transport loss. This - // proves even a sent cancellation can lose its receipt; no ticker timing guess. - var request proto.PromptCancelPayload - if err := h.read(proto.TypePromptCancel).DecodePayload(&request); err != nil { - t.Fatal(err) - } - if request.DeliveryID == "" { - t.Fatal("missing cancel delivery identity") - } - if scenario == "rotated" { - if _, err := s.pool.Exec(t.Context(), "UPDATE devices SET credential_hash=$2 WHERE id=$1", owner.DeviceID, runtimedevice.HashCredential(uuid.NewString())); err != nil { - t.Fatal(err) - } - } - if scenario == "expired" { - if _, err := s.pool.Exec(t.Context(), "UPDATE turns SET cancel_requested_at=clock_timestamp()-interval '21 seconds' WHERE id=$1", input.TurnID); err != nil { - t.Fatal(err) - } - } - var unlockCommit func() - if scenario == "ack_commit_blocked" { - tx, err := s.pool.Begin(t.Context()) - if err != nil { - t.Fatal(err) - } - if _, err := tx.Exec(t.Context(), "SELECT session_id FROM session_runtime_assignments WHERE session_id=$1 FOR UPDATE", session.ID); err != nil { - t.Fatal(err) - } - unlockCommit = func() { - if err := tx.Rollback(context.Background()); err != nil { - t.Fatal(err) - } - } - defer func() { _ = tx.Rollback(context.Background()) }() - } - - if scenario == "done_heartbeat_ack" { - h.write(input.TurnID, proto.TypeDone, proto.DonePayload{}) - } - closedCase := scenario == "rotated" || scenario == "expired" || scenario == "transport_lost" || strings.Contains(scenario, "revoke") - if scenario == "transport_lost" { - h.conn.Close() - } else if heartbeat && scenario != "ack_commit_blocked" { - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{HomeRemoval: proto.CapabilityUnsupported}) - } - if !closedCase { - ack := proto.InteractionDecisionAckPayload{DeliveryID: request.DeliveryID, Applied: true, Outcome: &proto.DonePayload{Usage: proto.Usage{InputTokens: 17}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "cancelled-native"}}} - if scenario == "negative_ack" { - ack.Applied = false - ack.ErrorCode = "cancel_failed" - ack.Outcome = nil - } - if scenario == "missing_outcome" { - ack.Outcome = nil - } - h.write(input.TurnID, proto.TypeInteractionDecisionAck, ack) - } - if unlockCommit != nil { - // Observe actual SQL lock contention, not an assumed timing delay. - for deadline := time.Now().Add(3 * time.Second); ; { - var blocked bool - if err := s.pool.QueryRow(t.Context(), "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND query ILIKE '%session_runtime_assignments%')").Scan(&blocked); err != nil { - t.Fatal(err) - } - if blocked { - break - } - if time.Now().After(deadline) { - t.Fatal("terminal commit never blocked") - } - time.Sleep(time.Millisecond) - } - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{HomeRemoval: proto.CapabilityUnsupported}) - draining, err := peer.DrainArchivedCancellation(t.Context()) - if err != nil || !draining || peer.IsClosed() { - t.Fatal("ACK lost drain before terminal commit", draining, err) - } - unlockCommit() - } - - got := awaitPreparedDispatch(t, result) - wantStatus := sessions.TurnCancelled - if closedCase || scenario == "negative_ack" || scenario == "missing_outcome" { - wantStatus = sessions.TurnFailed - } - if got.err != nil || got.run.Turn.Status != wantStatus { - t.Fatal(got.run.Turn.Status, got.err, string(got.run.Turn.Outcome)) - } - var outcome execution.Result - if err := json.Unmarshal(got.run.Turn.Outcome, &outcome); err != nil || (wantStatus == sessions.TurnCancelled && outcome.Done.Usage.InputTokens != 17) { - t.Fatal("receipt lost usage", string(got.run.Turn.Outcome), err) - } - - var receipts int - if err := s.pool.QueryRow(t.Context(), "SELECT count(*) FROM turn_events WHERE turn_id=$1 AND kind='cancel_receipt'", input.TurnID).Scan(&receipts); err != nil { - t.Fatal(err) - } - wantReceipts := 1 - if closedCase { - wantReceipts = 0 - } - if receipts != wantReceipts { - t.Fatal("durable cancellation receipt mismatch", receipts, wantReceipts) - } - // The original cleanup owner survives every delivery outcome; only - // provider receipts can release its resources. - allocation, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: h.tenant, EnvironmentID: session.Environment.ID}) - if err != nil || allocation.State != "cleanup_pending" { - t.Fatal(allocation, err) - } - var revoked bool - if err := s.pool.QueryRow(t.Context(), "SELECT revoked_at IS NOT NULL FROM devices WHERE id=$1", owner.DeviceID).Scan(&revoked); err != nil || !revoked { - t.Fatal(revoked, err) - } - }) - } -} diff --git a/services/core/tests/integration/credential_matrix_http_test.go b/services/core/tests/integration/credential_matrix_http_test.go index 74b49cfdc..c07b09f40 100644 --- a/services/core/tests/integration/credential_matrix_http_test.go +++ b/services/core/tests/integration/credential_matrix_http_test.go @@ -10,6 +10,7 @@ import ( "sync" "testing" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/relay" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" @@ -42,8 +43,12 @@ func TestCredentialNamespaceMatrix(t *testing.T) { } // The server composition: daemon transport beside the API handler. mux := http.NewServeMux() - mux.Handle("/api/v1/agent-daemon/enroll", runtimeenrollment.EnrollmentHandler(sessionService(t, s))) - mux.Handle("/api/v1/agent-daemon/connection", runtimeenrollment.ConnectionHandler(sessionAdapter(s), runtimegateway.NewRegistry())) + origin, err := deployment.NewPublicOrigin("https://core.example") + if err != nil { + t.Fatal(err) + } + mux.Handle("/api/v1/agent-daemon/enroll", runtimeenrollment.EnrollmentHandler(sessionService(t, s), origin)) + mux.Handle("/api/v1/agent-daemon/connection", runtimeenrollment.ConnectionHandler(sessionAdapter(s), relay.New(runtimegateway.NewLinkAuthority(sessionAdapter(s))))) mux.Handle("/", handler) server := api.CanonicalPaths(mux) call := func(method, path, token, body string) *httptest.ResponseRecorder { diff --git a/services/core/tests/integration/deployment_model_providers_http_test.go b/services/core/tests/integration/deployment_model_providers_http_test.go index aa0260f38..266be70f0 100644 --- a/services/core/tests/integration/deployment_model_providers_http_test.go +++ b/services/core/tests/integration/deployment_model_providers_http_test.go @@ -200,8 +200,7 @@ func TestDeploymentModelProvidersHTTP(t *testing.T) { t.Fatal("a changed default reached an existing Session") } - // Every Environment type accepts every source and freezes it, except that - // a self_hosted guest never receives the deployment key. + // Every Environment type accepts every source and freezes it. agentID := text(call("POST", "/v1/agents", projectKey, `{"model":"agent-model","x_agents_core":{"model_provider":{"protocol":"responses","base_url":"https://agent.example/v1","api_key":"agent-canary"}}}`, 201)["id"]) for name, environment := range environments { for _, tc := range []struct{ source, body, key string }{ @@ -209,13 +208,6 @@ func TestDeploymentModelProvidersHTTP(t *testing.T) { {"agent", `{"agent_id":"` + agentID + `",` + environment + `}`, "agent-canary"}, {"deployment", `{"agent":{"model":"m"},` + environment + `}`, "changed-canary"}, } { - if name == "self_hosted" && tc.source == "deployment" { - failure := call("POST", "/v1/agents/sessions", projectKey, tc.body, 400) - if !strings.Contains(string(failure["error"]), `"code":"model_provider_required","param":"x_agents_core.model_provider"`) { - t.Fatalf("self_hosted accepted the deployment default: %s", failure["error"]) - } - continue - } id := text(call("POST", "/v1/agents/sessions", projectKey, tc.body, 201)["id"]) projection, err := sessionAdapter(st).GetSessionExecutionConfiguration(t.Context(), tenant, id) if providerOf(id) != tc.key || err != nil || projection.ModelProvider.Source != tc.source { diff --git a/services/core/tests/integration/device_bootstrap_binding_test.go b/services/core/tests/integration/device_bootstrap_binding_test.go index d1d261d4d..1d4598350 100644 --- a/services/core/tests/integration/device_bootstrap_binding_test.go +++ b/services/core/tests/integration/device_bootstrap_binding_test.go @@ -70,22 +70,10 @@ func TestDeviceCredentialWithoutManagedNodeRetainsPublicRouteIdentity(t *testing if err != nil { t.Fatal(err) } - principal := FixtureExecutorPrincipal(t, s, uuid.NewString()) - _, selfhost, key := runtimeEnrollmentFixture(t, s, principal) - enrolled, err := sessionService(t, s).EnrollRuntime(t.Context(), selfhost.ID, executorDigest(key.Token)) - if err != nil { - t.Fatal(err) - } - for _, id := range []string{ordinary.ID, allocation.DeviceID, enrolled.DeviceID} { + for _, id := range []string{ordinary.ID, allocation.DeviceID} { credential, found, err := sessionAdapter(s).GetDeviceCredential(t.Context(), id) if err != nil || !found || credential.RuntimeNodeID != "" { t.Fatalf("non-node credential acquired allocation route: found=%v node=%s error=%v", found, credential.RuntimeNodeID, err) } } - if err := sessionService(t, s).RevokeExecutorCredential(t.Context(), principal, key.KeyID); err != nil { - t.Fatal(err) - } - if _, found, err := sessionAdapter(s).GetDeviceCredential(t.Context(), enrolled.DeviceID); err != nil || found { - t.Fatal("LEFT JOIN revived revoked executor key", err) - } } diff --git a/services/core/tests/integration/devices_test.go b/services/core/tests/integration/devices_test.go index 6461f34e1..80c7e31f1 100644 --- a/services/core/tests/integration/devices_test.go +++ b/services/core/tests/integration/devices_test.go @@ -39,9 +39,8 @@ func TestDeviceBindingIsTenantScopedStableAndDurable(t *testing.T) { ctx := context.Background() tenant, session := newTurnSession(t, s) otherTenant, otherSession := newTurnSession(t, s) - a, _ := registerTestDevice(t, s, tenant) - b, _ := registerTestDevice(t, s, tenant) - foreign, _ := registerTestDevice(t, s, otherTenant) + a, b := registerAgentHost(t, s, tenant), registerAgentHost(t, s, tenant) + ordinary, _ := registerTestDevice(t, s, tenant) // The binds run on an execution lease of their own, which closes before // the pool does. lease, err := pgunit.AcquireLease(ctx, pool) @@ -53,7 +52,8 @@ func TestDeviceBindingIsTenantScopedStableAndDurable(t *testing.T) { if err != nil { t.Fatal(err) } - for _, args := range [][3]string{{tenant, session.ID, foreign.ID}, {otherTenant, session.ID, foreign.ID}, {tenant, otherSession.ID, a.ID}} { + // Sessions bind only agent hosts, and only within their own tenant. + for _, args := range [][3]string{{tenant, session.ID, ordinary.ID}, {otherTenant, session.ID, a.ID}, {tenant, otherSession.ID, a.ID}} { if err := execution.BindSessionDevice(ctx, args[0], args[1], args[2]); !errors.Is(err, sessions.ErrNotFound) { t.Fatalf("foreign binding: %v", err) } @@ -98,18 +98,13 @@ func TestDeviceBindingIsTenantScopedStableAndDurable(t *testing.T) { t.Fatal(err) } pool.Close() - restarted, _ := testStore(t) + restarted, restartedPool := testStore(t) got, err := sessionAdapter(restarted).GetSessionDevice(ctx, tenant, session.ID) if err != nil || got != winner { t.Fatalf("binding after restart: %+v %v", got, err) } - if err := sessionService(t, restarted).RevokeDevice(ctx, otherTenant, winner.ID); !errors.Is(err, sessions.ErrNotFound) { - t.Fatalf("foreign revocation: %v", err) - } - for range 2 { - if err := sessionService(t, restarted).RevokeDevice(ctx, tenant, winner.ID); err != nil { - t.Fatal(err) - } + if _, err := restartedPool.Exec(ctx, `UPDATE devices SET revoked_at = clock_timestamp() WHERE id = $1`, winner.ID); err != nil { + t.Fatal(err) } if _, err := sessionAdapter(restarted).GetSessionDevice(ctx, tenant, session.ID); !errors.Is(err, sessions.ErrNotFound) { t.Fatalf("revoked device remains dispatchable: %v", err) diff --git a/services/core/tests/integration/dispatch_test.go b/services/core/tests/integration/dispatch_test.go index 3fd4d30c1..934e57fc3 100644 --- a/services/core/tests/integration/dispatch_test.go +++ b/services/core/tests/integration/dispatch_test.go @@ -13,12 +13,13 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/sandboxlinktest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtime" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" @@ -43,14 +44,22 @@ type dispatchHarness struct { url string credential string environments map[string]*dispatchHarness + link *sandboxlinktest.Server // the relay the Session's Environment Serves at + resource sandboxbootstrap.Resource // the Environment's Link resource, if the Session has one + serve []byte // the resource's Serve credential + served *linkServe // the fake sandbox Serving resource } func newDispatchHarness(t *testing.T) *dispatchHarness { t.Helper() - return newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model","instructions":"Keep this instruction."},"environment":{"type":"none"}}`), false) + return newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model","instructions":"Keep this instruction."},"environment":{"type":"none"}}`)) } -func newDispatchHarnessForSession(t *testing.T, configuration []byte, local bool) *dispatchHarness { +// newDispatchHarnessForSession binds a Session of configuration to the +// deployment's agent host, which the harness connects as. A Session with an +// Environment first gets a live Link resource that a fake sandbox Serves at +// the harness's relay: an enrollment for self_hosted, an allocation otherwise. +func newDispatchHarnessForSession(t *testing.T, configuration []byte) *dispatchHarness { t.Helper() s, _ := testStore(t) h := &dispatchHarness{t: t, s: s, tenant: uuid.NewString(), environments: map[string]*dispatchHarness{}} @@ -60,28 +69,14 @@ func newDispatchHarnessForSession(t *testing.T, configuration []byte, local bool if err != nil { t.Fatal(err) } - secret := uuid.NewString() - h.credential = secret - var snapshot struct { - Environment struct { - Type string `json:"type"` - } `json:"environment"` - } - _ = json.Unmarshal(configuration, &snapshot) - if snapshot.Environment.Type == "self_hosted" { - h.device, h.credential = enrollFixtureSession(t, s, h.tenant, h.session) - secret = h.credential - } else if local { - environment, getErr := sessionAdapter(s).GetSessionEnvironment(ctx, h.tenant, h.session.ID) - if getErr != nil { - t.Fatal(getErr) - } - h.device, err = FixtureEnvironmentDevice(t, ctx, s.pool, h.tenant, environment.ID, "local runtime", runtimedevice.HashCredential(secret)) - } else { - h.device, err = sessionService(t, s).CreateDevice(ctx, h.tenant, "isolated executor", runtimedevice.HashCredential(secret)) - } - if err != nil { - t.Fatal(err) + host := registerAgentHost(t, s, h.tenant) + h.device, h.credential = sessions.ExecutionDevice{ID: host.ID, Name: "agent host"}, host.Credential + secret := h.credential + h.link = sandboxlinktest.StartRelay(t, runtimegateway.NewLinkAuthority(sessionAdapter(s))) + if h.session.Environment != nil { + h.resource, h.serve = fixtureLinkResource(t, s, h.tenant, h.session) + h.served = startLinkServe(t, h.link, h.serve, h.resource.Ref()) + within(t, h.served.connected) } if err = bindSessionDevice(t, s, h.tenant, h.session.ID, h.device.ID); err != nil { t.Fatal(err) @@ -121,10 +116,18 @@ func newDispatchHarnessForSession(t *testing.T, configuration []byte, local bool if err != nil { t.Fatal(err) } - h.d = &execution.Dispatcher{Registry: h.registry, Observer: modelconfigurationpg.New(pgunit.NewPool(s.pool), s.credentialCipher), Sessions: sessionService, SessionsReader: sessionStore} + h.d = &execution.Dispatcher{Registry: h.registry, Links: h.link.Relay, Observer: modelconfigurationpg.New(pgunit.NewPool(s.pool), s.credentialCipher), Sessions: sessionService, SessionsReader: sessionStore} return h } +// stopServing stops the fake sandbox and waits until the relay no longer +// holds its serve peer. +func (h *dispatchHarness) stopServing() { + h.t.Helper() + h.served.stop() + awaitDaemonRemoteCondition(h.t, h.t.Context(), linkWait, "the relay to drop the serve peer", func() bool { return !h.link.Relay.Serving(h.resource.Ref()) }) +} + func (h *dispatchHarness) message(key, text string) sessions.InputReceipt { h.t.Helper() r, err := sendMessage(context.Background(), h.s, h.tenant, h.session.ID, key, messageText(text)) @@ -433,7 +436,7 @@ func TestExecutionRejectsRuntimeMissingCapabilityBeforeClaim(t *testing.T) { missing := tc.missing t.Run(missing, func(t *testing.T) { // Only an explicit non-medium verbosity needs text_verbosity. - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model","instructions":"Keep this instruction.","text":{"verbosity":"high"}},"environment":{"type":"none"}}`), false) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model","instructions":"Keep this instruction.","text":{"verbosity":"high"}},"environment":{"type":"none"}}`)) caps := prototest.Capabilities(proto.AgentKindCapabilities{TextVerbosity: proto.CapabilityFromBool(missing != "text_verbosity"), EnvironmentNone: proto.CapabilityFromBool(missing != "environment_none")}) h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{HomeRemoval: proto.CapabilityUnsupported, SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: caps}}}) deadline := time.Now().Add(3 * time.Second) diff --git a/services/core/tests/integration/environment_directory_test.go b/services/core/tests/integration/environment_directory_test.go index b18fbe23c..a5276c5a4 100644 --- a/services/core/tests/integration/environment_directory_test.go +++ b/services/core/tests/integration/environment_directory_test.go @@ -20,7 +20,7 @@ type directoryResult struct { func directoryWorker(t *testing.T) (*dispatchHarness, *execution.Worker, sessions.Environment) { t.Helper() - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"unavailable-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`), true) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"unavailable-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`)) environment, err := sessionAdapter(h.s).GetSessionEnvironment(t.Context(), h.tenant, h.session.ID) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/environment_executor_management_test.go b/services/core/tests/integration/environment_executor_management_test.go index a2f5a0d84..9afbc8b6a 100644 --- a/services/core/tests/integration/environment_executor_management_test.go +++ b/services/core/tests/integration/environment_executor_management_test.go @@ -207,11 +207,10 @@ func TestProjectExecutorConnectionState(t *testing.T) { project := binding.Project session, env, key := runtimeEnrollmentFixture(t, s, binding.Principal) state, err := sessionAdapter(s).ProjectExecutorCredentialState(ctx, binding.Principal, env.ID) - if err != nil || state.Connection.DeviceID != "" || state.Connection.EnrolledAt != nil || state.Connection.BoundKeyID != nil || state.Connection.LastSeenAt != nil { + if err != nil || state.Connection.Enrolled || state.Connection.EnrolledAt != nil || state.Connection.BoundKeyID != nil { t.Fatal("never enrolled", state, err) } - enrolled, err := sessionService(t, s).EnrollRuntime(ctx, env.ID, executorDigest(key.Token)) - if err != nil { + if _, err := sessionService(t, s).EnrollRuntime(ctx, env.ID, executorDigest(key.Token)); err != nil { t.Fatal(err) } check := func() sessions.ExecutorCredentialState { @@ -223,16 +222,16 @@ func TestProjectExecutorConnectionState(t *testing.T) { return v } state = check() - if state.Connection.DeviceID != enrolled.DeviceID || state.Connection.BoundKeyID == nil || *state.Connection.BoundKeyID != key.KeyID || state.Connection.EnrolledAt == nil || state.Connection.LastSeenAt != nil || state.Connection.CredentialHash != executorDigest(key.Token) { + if !state.Connection.Enrolled || state.Connection.BoundKeyID == nil || *state.Connection.BoundKeyID != key.KeyID || state.Connection.EnrolledAt == nil || state.Connection.CredentialHash != executorDigest(key.Token) { t.Fatal("binding", state) } for _, spelling := range []string{env.ID, strings.ToUpper(env.ID), strings.ReplaceAll(env.ID, "-", "")} { resolved, err := sessionAdapter(s).ProjectExecutorCredentialState(ctx, binding.Principal, spelling) - if err != nil || resolved.EnvironmentID != env.ID || resolved.Connection.DeviceID != enrolled.DeviceID || resolved.Connection.CredentialHash != executorDigest(key.Token) { + if err != nil || resolved.EnvironmentID != env.ID || !resolved.Connection.Enrolled || resolved.Connection.CredentialHash != executorDigest(key.Token) { t.Fatal("equivalent target did not retain canonical identity and binding", spelling, resolved, err) } } - // An additional credential never changes the enrolled device's bound key. + // An additional credential never changes the enrolled key. if _, err = sessionService(t, s).IssueProjectExecutorCredential(keyAdminContext(ctx, project.ID), binding.Principal, env.ID, uuid.NewString(), false); err != nil { t.Fatal(err) } @@ -240,13 +239,6 @@ func TestProjectExecutorConnectionState(t *testing.T) { if *state.Connection.BoundKeyID != key.KeyID || len(state.Credentials) != 2 { t.Fatal("second key changed binding") } - if _, err = pool.Exec(ctx, "UPDATE devices SET last_seen_at=clock_timestamp() WHERE id=$1", enrolled.DeviceID); err != nil { - t.Fatal(err) - } - state = check() - if state.Connection.LastSeenAt == nil { - t.Fatal("heartbeat history missing") - } rotated, err := sessionService(t, s).IssueProjectExecutorCredential(keyAdminContext(ctx, project.ID), binding.Principal, env.ID, key.KeyID, true) if err != nil { t.Fatal(err) @@ -285,7 +277,7 @@ func TestProjectExecutorConnectionState(t *testing.T) { t.Fatal(err) } state = check() - if state.Connection.EnvironmentStatus != "expired" || state.Connection.CredentialHash != "" { + if !state.Connection.Enrolled || state.Connection.CredentialHash != "" { t.Fatal("expired authority") } foreign := createTestProject(t, pool) diff --git a/services/core/tests/integration/environment_expiry_dispatch_test.go b/services/core/tests/integration/environment_expiry_dispatch_test.go index 78dc02e61..bcceb0c32 100644 --- a/services/core/tests/integration/environment_expiry_dispatch_test.go +++ b/services/core/tests/integration/environment_expiry_dispatch_test.go @@ -21,7 +21,7 @@ func TestWorkerEnvironmentExpiryAtFullExecutionCapacity(t *testing.T) { var requests []proto.Envelope var active []sessions.Session for _, key := range []string{"one", "two", "three", "four"} { - session := publicSession(t, h, key) + session := ordinarySession(t, h, key) if _, err := worker.SubmitInputs(t.Context(), h.tenant, session.ID, key, []sessions.Input{messageInput("remain active")}); err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestWorkerEnvironmentExpirySkipsBusySessionAndAllowsDispatch(t *testing.T) t.Fatal(err) } worker, stop := startEnvironmentExpiryWorker(t, h.s, h.d) - h.session = publicSession(t, h, "unrelated") + h.session = ordinarySession(t, h, "unrelated") receipt, err := worker.SubmitInputs(t.Context(), h.tenant, h.session.ID, "work", []sessions.Input{messageInput("make normal progress")}) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/environment_file_write_semantics_public_test.go b/services/core/tests/integration/environment_file_write_semantics_public_test.go index 03398dc75..dd1da7f53 100644 --- a/services/core/tests/integration/environment_file_write_semantics_public_test.go +++ b/services/core/tests/integration/environment_file_write_semantics_public_test.go @@ -49,7 +49,7 @@ func serveFileWrite(h *dispatchHarness, final proto.WorkspaceWriteResultPayload) } func TestEnvironmentFileCreateRejectionsLeaveNoReceiptOrConsumption(t *testing.T) { - h, w, environment := localWorker(t, true, false) + h, w, environment := localWorker(t, false) _, pool := testStore(t) if _, err := pool.Exec(t.Context(), `UPDATE environments SET status='connected' WHERE id=$1`, environment.ID); err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/environment_file_writes_test.go b/services/core/tests/integration/environment_file_writes_test.go index 276e346ad..e0d36ee1a 100644 --- a/services/core/tests/integration/environment_file_writes_test.go +++ b/services/core/tests/integration/environment_file_writes_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) @@ -24,14 +23,12 @@ type fileWriteFixture struct { func newFileWriteFixture(t *testing.T) fileWriteFixture { t.Helper() - s, pool := testStore(t) + s, _ := testStore(t) lease := executionWriter(t, s).lease tenant := uuid.NewString() session, env := localEnvironment(t, s, tenant) - host, err := FixtureEnvironmentDevice(t, t.Context(), pool, tenant, env.ID, "file owner", runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } + host := registerAgentHost(t, s, tenant) + assignSession(t, s, session.ID, host.ID) return fileWriteFixture{s: s, lease: lease, writer: sessionExecution(t, lease), tenant: tenant, session: session, env: env, key: sessions.FileWriteIdentity{ID: uuid.NewString(), DeviceID: host.ID, RequestSHA256: strings.Repeat("a", 64)}} } diff --git a/services/core/tests/integration/environment_initialization_test.go b/services/core/tests/integration/environment_initialization_test.go index 994add168..aeee8fd9c 100644 --- a/services/core/tests/integration/environment_initialization_test.go +++ b/services/core/tests/integration/environment_initialization_test.go @@ -13,6 +13,8 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/sandboxlinktest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" @@ -47,7 +49,7 @@ func awaitInitialization(t *testing.T, s *Store, tenant, environment, state stri } func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *testing.T) { - for _, outcome := range []string{"completed", "failed", "unknown", "unavailable", "revoked"} { + for _, outcome := range []string{"completed", "failed", "unknown", "unavailable"} { t.Run(outcome, func(t *testing.T) { _, pool := newManagedTestStore(t) cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) @@ -81,7 +83,8 @@ func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *test handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: registry}) server := httptest.NewServer(http.HandlerFunc(handler.WS)) defer server.Close() - worker := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry}) + link := sandboxlinktest.StartRelay(t, runtimegateway.NewLinkAuthority(sessionAdapter(s))) + worker := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, Links: link.Relay}) ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { done <- worker.Run(ctx) }() @@ -93,8 +96,8 @@ func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *test }() var mu sync.Mutex var actions []string - peer := &initializationPeer{unavailable: outcome == "unavailable"} - peer.setRuntimeGateway(t, "ws"+strings.TrimPrefix(server.URL, "http"), registry, nil) + peer := &initializationPeer{unavailable: outcome == "unavailable", tenant: principal.TenantID} + peer.setRuntimeGateway(t, s, "ws"+strings.TrimPrefix(server.URL, "http"), registry, link) peer.apply = func(request proto.RuntimePreparePayload, data []byte) proto.RuntimePrepareResultPayload { if request.EnvironmentID != environment.ID || request.SessionID != session.ID { t.Error("wrong authorization binding") @@ -109,12 +112,6 @@ func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *test mu.Lock() actions = append(actions, action) mu.Unlock() - if outcome == "revoked" { - if err := sessionService(t, s).RevokeDevice(t.Context(), principal.TenantID, enrolled.DeviceID); err != nil { - t.Error(err) - } - return completedInitialization(request, data) - } if outcome != "completed" { return proto.RuntimePrepareResultPayload{Outcome: outcome, ErrorCode: "runtime_preparation_unconfirmed"} } @@ -124,7 +121,8 @@ func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *test if initializationState(t, s, principal.TenantID, environment.ID) != "pending" { t.Fatal("unconnected preparation was consumed") } - bootstrap := sandbox.Bootstrap{DeviceID: enrolled.DeviceID, Credential: key.Token} + // The enrolled machine Serves; the agent host runs the initialization. + bootstrap := sandbox.Bootstrap{SandboxIO: sandboxbootstrap.Input{Credential: key.Token, Resource: enrolled}} if err := peer.connect(bootstrap); err != nil { t.Fatal(err) } @@ -156,7 +154,7 @@ func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *test expected = 0 } if peer.writes.Load() != expected { - t.Fatal("failed, unavailable or revoked effect replayed", peer.writes.Load()) + t.Fatal("failed or unavailable effect replayed", peer.writes.Load()) } value, err := sessionAdapter(s).GetSession(t.Context(), principal.TenantID, session.ID) if err != nil || value.EnvironmentFailure == nil { @@ -172,6 +170,8 @@ func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *test func TestEnvironmentInitializationRevocationBeforeClaim(t *testing.T) { s, _ := newManagedTestStore(t) principal := FixtureExecutorPrincipal(t, s, uuid.NewString()) + owned := executionOwner(t, s).Sessions + // Each Environment is enrolled and its Session bound to an agent host of its own. create := func() sessions.EnvironmentInitialization { t.Helper() session, err := s.CreateSession(t.Context(), principal.TenantID, sessions.CreateSession{ @@ -190,15 +190,17 @@ func TestEnvironmentInitializationRevocationBeforeClaim(t *testing.T) { if err != nil { t.Fatal(err) } - enrolled, err := sessionService(t, s).EnrollRuntime(t.Context(), environment.ID, runtimedevice.HashCredential(key.Token)) - if err != nil { + if _, err := sessionService(t, s).EnrollRuntime(t.Context(), environment.ID, runtimedevice.HashCredential(key.Token)); err != nil { t.Fatal(err) } - return sessions.EnvironmentInitialization{EnvironmentID: environment.ID, SessionID: session.ID, TenantID: principal.TenantID, DeviceID: enrolled.DeviceID, State: "pending", Engine: "codex"} + host := registerAgentHost(t, s, principal.TenantID) + if err := owned.BindSessionDevice(t.Context(), principal.TenantID, session.ID, host.ID); err != nil { + t.Fatal(err) + } + return sessions.EnvironmentInitialization{EnvironmentID: environment.ID, SessionID: session.ID, TenantID: principal.TenantID, DeviceID: host.ID, State: "pending", Engine: "codex"} } revoked, other := create(), create() - owned := executionOwner(t, s).Sessions - if err := sessionService(t, s).RevokeDevice(t.Context(), principal.TenantID, revoked.DeviceID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE devices SET revoked_at = clock_timestamp() WHERE id = $1", revoked.DeviceID); err != nil { t.Fatal(err) } if err := owned.ClaimEnvironmentInitialization(t.Context(), revoked); !errors.Is(err, sessions.ErrNotFound) { diff --git a/services/core/tests/integration/environment_runtime_fixture_test.go b/services/core/tests/integration/environment_runtime_fixture_test.go index 31dde7bfd..0798a9518 100644 --- a/services/core/tests/integration/environment_runtime_fixture_test.go +++ b/services/core/tests/integration/environment_runtime_fixture_test.go @@ -1,46 +1,69 @@ package integration import ( + "encoding/json" "net/http" "net/url" "testing" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" "github.com/gorilla/websocket" ) -func enrollFixtureSession(t *testing.T, s *Store, tenant string, session sessions.Session) (sessions.ExecutionDevice, string) { +// fixtureLinkResource gives the Session's Environment a live Link resource +// and returns it with its Serve credential: for self_hosted, the enrollment of +// a new executor key, whose token Serves; otherwise a running allocation. +func fixtureLinkResource(t *testing.T, s *Store, tenant string, session sessions.Session) (sandboxbootstrap.Resource, []byte) { t.Helper() - environment, err := sessionAdapter(s).GetSessionEnvironment(t.Context(), tenant, session.ID) - if err != nil { - t.Fatal(err) + environment := session.Environment.ID + var snapshot struct { + Environment struct{ Type string } `json:"environment"` } - principal := FixtureExecutorPrincipal(t, s, tenant) - key, err := sessionService(t, s).IssueExecutorCredential(t.Context(), principal, uuid.NewString(), environment.ID) - if err != nil { + if err := json.Unmarshal(session.Configuration, &snapshot); err != nil { t.Fatal(err) } - enrolled, err := sessionService(t, s).EnrollRuntime(t.Context(), environment.ID, runtimedevice.HashCredential(key.Token)) - if err != nil || enrolled.EnvironmentID != environment.ID || enrolled.SessionID != session.ID || enrolled.WorkspaceDirectory != "/workspace" { - t.Fatalf("Runtime enrollment: %+v %v", enrolled, err) + if snapshot.Environment.Type == "self_hosted" { + key, err := sessionService(t, s).IssueExecutorCredential(t.Context(), FixtureExecutorPrincipal(t, s, tenant), uuid.NewString(), environment) + if err != nil { + t.Fatal(err) + } + resource, err := sessionService(t, s).EnrollRuntime(t.Context(), environment, runtimedevice.HashCredential(key.Token)) + if err != nil { + t.Fatal(err) + } + return resource, []byte(key.Token) } - bound, err := sessionAdapter(s).GetSessionDevice(t.Context(), tenant, session.ID) - if err != nil || bound.ID != enrolled.DeviceID || bound.EnvironmentID != environment.ID { - t.Fatalf("Runtime binding: %+v %v", bound, err) + device, err := FixtureEnvironmentDevice(t, t.Context(), s.pool, tenant, environment, "sandbox", runtimedevice.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) } - return bound, key.Token + resource := sandboxbootstrap.Resource{TenantID: tenant, EnvironmentID: environment, Kind: "allocation", ID: uuid.NewString(), Generation: 1} + serve := []byte(uuid.NewString()) + insertAllocation(t, s, resource, device.ID, serve) + return resource, serve } +// connectFixtureRuntime connects another agent host with session placed on +// it, and has the Session's Environment Serve a Link resource of its own at +// h's relay. func connectFixtureRuntime(t *testing.T, h *dispatchHarness, session sessions.Session) *dispatchHarness { t.Helper() // The Runtime shares the harness's Core, not its connection or write lock. other := &dispatchHarness{t: h.t, s: h.s, lease: h.lease, owned: h.owned, d: h.d, tenant: h.tenant, session: session, registry: h.registry, url: h.url, - admissions: h.admissions, environments: h.environments} - other.device, other.credential = enrollFixtureSession(t, h.s, h.tenant, session) + admissions: h.admissions, environments: h.environments, link: h.link} + host := registerAgentHost(t, h.s, h.tenant) + other.device, other.credential = sessions.ExecutionDevice{ID: host.ID, Name: "agent host"}, host.Credential + other.resource, other.serve = fixtureLinkResource(t, h.s, h.tenant, session) + // The placement precedes Serve, so a running Worker cannot place the + // Session on another connected agent host first. + assignSession(t, h.s, session.ID, host.ID) + other.served = startLinkServe(t, h.link, other.serve, other.resource.Ref()) + within(t, other.served.connected) u, err := url.Parse(h.url) if err != nil { t.Fatal(err) @@ -49,7 +72,7 @@ func connectFixtureRuntime(t *testing.T, h *dispatchHarness, session sessions.Se u.RawQuery = url.Values{"device_id": {other.device.ID}, "version": {proto.Version}}.Encode() other.conn, _, err = websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + other.credential}}) if err != nil { - t.Fatal("enrolled Runtime connection failed") + t.Fatal("agent host connection failed") } t.Cleanup(func() { _ = other.conn.Close() }) enableWorkerEnvironment(t, other) diff --git a/services/core/tests/integration/environment_worker_helpers_test.go b/services/core/tests/integration/environment_worker_helpers_test.go index 6cd55c03f..9a4e48c98 100644 --- a/services/core/tests/integration/environment_worker_helpers_test.go +++ b/services/core/tests/integration/environment_worker_helpers_test.go @@ -26,6 +26,16 @@ func enableWorkerEnvironment(t *testing.T, h *dispatchHarness) { }) } +// ordinarySession creates a Session without an Environment placed on h's agent +// host: each Environment's fixture Runtime is another agent host of the +// tenant, which the Worker could otherwise choose. +func ordinarySession(t *testing.T, h *dispatchHarness, key string) sessions.Session { + t.Helper() + session := publicSession(t, h, key) + assignSession(t, h.s, session.ID, h.device.ID) + return session +} + func workerEnvironmentCapabilities() proto.AgentKindCapabilities { return prototest.Capabilities(proto.AgentKindCapabilities{EnvironmentNone: proto.CapabilitySupported, TextVerbosity: proto.CapabilitySupported, LocalEnvironment: proto.CapabilitySupported}) } @@ -90,7 +100,7 @@ func workerRuntimeForPreparation(t *testing.T, h *dispatchHarness, frame proto.E t.Fatal("invalid worker preparation") } for _, candidate := range h.environments { - if input.SessionID == candidate.session.ID && input.Configuration.LocalEnvironment != nil && input.Configuration.LocalEnvironment.ID == candidate.device.EnvironmentID { + if input.SessionID == candidate.session.ID && input.Configuration.LocalEnvironment != nil && input.Configuration.LocalEnvironment.ID == candidate.session.Environment.ID { return candidate } } diff --git a/services/core/tests/integration/environment_worker_scan_test.go b/services/core/tests/integration/environment_worker_scan_test.go index a81294f38..3bf46eef9 100644 --- a/services/core/tests/integration/environment_worker_scan_test.go +++ b/services/core/tests/integration/environment_worker_scan_test.go @@ -18,7 +18,7 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { awaitFixtureCapabilities(t, runtime, caps) frames := workerFrames(t, h, runtime) - h.session = publicSession(t, h, "scan-barrier") + h.session = ordinarySession(t, h, "scan-barrier") receipt := h.message("barrier", "ordinary work") scanned := time.Now() _, stop := startEnvironmentExpiryWorker(t, h.s, h.d) @@ -66,7 +66,7 @@ func TestWorkerEnvironmentPaginationReachesReadyTail(t *testing.T) { runtime := connectFixtureRuntime(t, h, session) h.environments[last.SessionID] = runtime frames := workerFrames(t, h, runtime) - h.session = publicSession(t, h, "page-barrier") + h.session = ordinarySession(t, h, "page-barrier") receipt := h.message("barrier", "ordinary work") scanned := time.Now() _, stop := startEnvironmentExpiryWorker(t, h.s, h.d) diff --git a/services/core/tests/integration/environment_worker_test.go b/services/core/tests/integration/environment_worker_test.go index dc0cf75da..de3a762ee 100644 --- a/services/core/tests/integration/environment_worker_test.go +++ b/services/core/tests/integration/environment_worker_test.go @@ -25,7 +25,7 @@ func TestWorkerEnvironmentSharesCapacityThroughClaimAndCleanup(t *testing.T) { frames := workerFrames(t, runtimes...) ordinary := map[string]sessions.Session{} for _, key := range []string{"one", "two", "three"} { - session := publicSession(t, h, key) + session := ordinarySession(t, h, key) h.session = session receipt := h.message(key, "ordinary") ordinary[receipt.TurnID] = session @@ -137,7 +137,7 @@ func TestWorkerEnvironmentRetriesPendingWithoutExtendingDeadline(t *testing.T) { handle := acknowledgePreparation(runtime, first.ID) runtime.write(first.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "failed"}) nextWorkerFrame(t, frames, proto.TypeExecutionRelease) - h.session = publicSession(t, h, "unrelated") + h.session = ordinarySession(t, h, "unrelated") receipt := h.message("ordinary", "make progress after preparation failure") request := nextWorkerFrame(t, frames, testExecutionRequest) if request.ID != receipt.TurnID { diff --git a/services/core/tests/integration/execution_test.go b/services/core/tests/integration/execution_test.go index 8e8f81218..a6765d983 100644 --- a/services/core/tests/integration/execution_test.go +++ b/services/core/tests/integration/execution_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" @@ -17,7 +16,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/sessionpg" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -114,13 +112,11 @@ func TestExecutionLeaseLossFencesAllLifecycleWrites(t *testing.T) { tenant, active := newTurnSession(t, s) input := submitMessage(t, s, tenant, active.ID, "active") transition(t, writer, tenant, active.ID, input.TurnID, sessions.TurnQueued, sessions.TurnInProgress) - host, err := sessionService(t, s).CreateDevice(t.Context(), tenant, "owner test", runtimedevice.HashCredential(uuid.NewString())) + host := registerAgentHost(t, s, tenant) + err := sessionExecution(t, writer.lease).BindSessionDevice(t.Context(), tenant, active.ID, host.ID) if err != nil { t.Fatal(err) } - if err = sessionExecution(t, writer.lease).BindSessionDevice(t.Context(), tenant, active.ID, host.ID); err != nil { - t.Fatal(err) - } queued, err := s.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: "queued"}) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/hosted_initialization_failure_public_test.go b/services/core/tests/integration/hosted_initialization_failure_public_test.go index 04ec8f9ca..9d9b0b37d 100644 --- a/services/core/tests/integration/hosted_initialization_failure_public_test.go +++ b/services/core/tests/integration/hosted_initialization_failure_public_test.go @@ -65,8 +65,8 @@ type hostedFailureProvider struct { steps []string } -func (p *hostedFailureProvider) setRuntimeGateway(t *testing.T, endpoint string, registry *runtimegateway.Registry, link *sandboxlinktest.Server) { - p.initializationPeer.setRuntimeGateway(t, endpoint, registry, link) +func (p *hostedFailureProvider) setRuntimeGateway(t *testing.T, s *Store, endpoint string, registry *runtimegateway.Registry, link *sandboxlinktest.Server) { + p.initializationPeer.setRuntimeGateway(t, s, endpoint, registry, link) p.apply = p.prepare } func (p *hostedFailureProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { diff --git a/services/core/tests/integration/link_authority_test.go b/services/core/tests/integration/link_authority_test.go index 5a31fc3b0..2488a5edf 100644 --- a/services/core/tests/integration/link_authority_test.go +++ b/services/core/tests/integration/link_authority_test.go @@ -40,37 +40,20 @@ import ( const linkWait = 10 * time.Second -// linkHarness is a hosted Session bound to h.device whose Environment has an -// allocation with a Serve credential, and Core's Link Authority behind the -// Link route. +// linkHarness is a dispatch harness whose Worker runs with the harness's +// relay: the Session is bound to the agent host, and its Environment's Link +// resource Serves at the relay. type linkHarness struct { *dispatchHarness - relay *sandboxlinktest.Server - resource sandboxbootstrap.Resource - serve []byte } -// newLinkHarness binds the Session to an unmarked operator device, or, for a -// guest, to the allocation's own device, as an in-sandbox daemon is bound, and -// runs a Worker with the Link route's relay. -func newLinkHarness(t *testing.T, guest bool) *linkHarness { +const hostedLinkSession = `{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}` + +func newLinkHarness(t *testing.T, configuration string) *linkHarness { t.Helper() - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), guest) - device := h.device.ID - if !guest { - allocated, err := sessionService(t, h.s).CreateDevice(t.Context(), h.tenant, "sandbox", runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } - device = allocated.ID - } - l := &linkHarness{dispatchHarness: h, relay: startLinkRoute(t, h.s), serve: []byte(uuid.NewString()), - resource: sandboxbootstrap.Resource{TenantID: h.tenant, EnvironmentID: h.session.Environment.ID, Kind: "allocation", ID: uuid.NewString(), Generation: 1}} - insertAllocation(t, h.s, l.resource, device, l.serve) - dispatcher := *h.d - dispatcher.Links = l.relay.Relay - runWorker(t, startWorker(t, t.Context(), h.s, &dispatcher)) - return l + h := newDispatchHarnessForSession(t, []byte(configuration)) + runWorker(t, startWorker(t, t.Context(), h.s, h.d)) + return &linkHarness{dispatchHarness: h} } // insertAllocation inserts a running allocation of device that is resource @@ -129,11 +112,10 @@ func (l *linkHarness) exec(query string, args ...any) { } } -// bind has the Worker bind the Session's current assignment to h.device and -// returns it with the payload the Runtime received. Until Sessions are placed -// on an agent host, the Environment's initialization is its one production -// bind, so bind reopens the initialization, which has nothing to install. The -// Worker claims it only while the Environment's resource is Serving. +// bind has the Worker bind the Session's current assignment to the agent host +// and returns it with the payload the agent host received. It reopens the +// Environment's initialization, which has nothing to install and binds first; +// the Worker claims it only while the Environment's resource is Serving. func (l *linkHarness) bind() (proto.AssignmentRef, proto.AssignmentBindPayload) { t := l.t t.Helper() @@ -155,7 +137,7 @@ func (l *linkHarness) bind() (proto.AssignmentRef, proto.AssignmentBindPayload) } func (l *linkHarness) attach(runtime string, credential []byte) (*sandboxlink.AttachLink, error) { - return attachLink(l.t, l.relay, runtime, credential) + return attachLink(l.t, l.link, runtime, credential) } func attachLink(t *testing.T, srv *sandboxlinktest.Server, runtime string, credential []byte) (*sandboxlink.AttachLink, error) { @@ -253,9 +235,8 @@ func within[T any](t *testing.T, ch <-chan T) T { // from a marked agent host, and checks that each part of the grant's // authority is current at every Open and renewal. func TestLinkAuthorityAgentHost(t *testing.T) { - l := newLinkHarness(t, false) - p := startLinkServe(t, l.relay, l.serve, l.resource.Ref()) - within(t, p.connected) + l := newLinkHarness(t, hostedLinkSession) + p := l.served otherID, otherEnvironment := l.resource, l.resource otherID.ID, otherEnvironment.EnvironmentID = uuid.NewString(), uuid.NewString() @@ -263,15 +244,18 @@ func TestLinkAuthorityAgentHost(t *testing.T) { resource sandboxbootstrap.Resource want sandboxlink.Code }{{otherID, sandboxlink.AuthenticationFailed}, {otherEnvironment, sandboxlink.PermissionDenied}} { - if got := startLinkServe(t, l.relay, l.serve, test.resource.Ref()).refused(t); got != test.want { + if got := startLinkServe(t, l.link, l.serve, test.resource.Ref()).refused(t); got != test.want { t.Fatalf("Serve of another resource refused with %v, want %v", got, test.want) } } - if _, err := l.attach(l.device.ID, []byte(l.credential)); linkCode(err) != sandboxlink.AuthenticationFailed { + operator := uuid.NewString() + unmarked, err := sessionService(t, l.s).CreateDevice(t.Context(), l.tenant, "operator", runtimedevice.HashCredential(operator)) + if err != nil { + t.Fatal(err) + } + if _, err := l.attach(unmarked.ID, []byte(operator)); linkCode(err) != sandboxlink.AuthenticationFailed { t.Fatal("an unmarked device attached", err) } - - l.exec("UPDATE devices SET agent_host = true WHERE id = $1", l.device.ID) link, err := l.attach(l.device.ID, []byte(l.credential)) if err != nil { t.Fatal(err) @@ -330,30 +314,11 @@ func TestLinkAuthorityAgentHost(t *testing.T) { } } -// TestLinkAuthorityGuest checks that an in-sandbox daemon's assignment -// carries no grant and that its device can neither attach nor be marked. -func TestLinkAuthorityGuest(t *testing.T) { - l := newLinkHarness(t, true) - within(t, startLinkServe(t, l.relay, l.serve, l.resource.Ref()).connected) - if _, payload := l.bind(); payload.Resource != nil || payload.AttachGrant != nil { - t.Fatalf("guest bind = %+v", payload) - } - if _, err := l.attach(l.device.ID, []byte(l.credential)); linkCode(err) != sandboxlink.AuthenticationFailed { - t.Fatal("a guest attached", err) - } - if _, err := l.s.pool.Exec(t.Context(), "UPDATE devices SET agent_host = true WHERE id = $1", l.device.ID); err == nil || !strings.Contains(err.Error(), "devices_agent_host") { - t.Fatal("a guest device was marked as an agent host", err) - } -} - // TestLinkAuthorityReleaseRevokesBeforeSend checks that a released // assignment's grant opens nothing from the commit on, and that the Worker // has the relay close its attachments before it sends the release. func TestLinkAuthorityReleaseRevokesBeforeSend(t *testing.T) { - l := newLinkHarness(t, false) - p := startLinkServe(t, l.relay, l.serve, l.resource.Ref()) - within(t, p.connected) - l.exec("UPDATE devices SET agent_host = true WHERE id = $1", l.device.ID) + l := newLinkHarness(t, hostedLinkSession) ref, payload := l.bind() link, err := l.attach(l.device.ID, []byte(l.credential)) if err != nil { @@ -411,26 +376,15 @@ func TestLinkAuthorityEnrollment(t *testing.T) { } // TestLinkAuthorityEnrollmentRotation checks that rotating an executor key -// advances its enrollment's generation: the serve peer of the old secret stays -// connected but no Open or renewal for the old generation is authorized, and -// the new secret serves the new generation. +// advances its enrollment's generation and the epoch of the Session's bound +// assignment. No Open or renewal for the old generation is authorized, the +// Worker's pass ends the old secret's Serve, and after the machine re-enrolls +// and Serves the new generation, the agent host's next bind opens on it. func TestLinkAuthorityEnrollmentRotation(t *testing.T) { - l := newLinkHarness(t, false) - // The Session's Link resource becomes an enrollment of an executor key. - l.exec("UPDATE runtime_allocations SET serve_credential_hash = NULL WHERE id = $1", l.resource.ID) - l.exec(`UPDATE sessions SET configuration = jsonb_set(configuration, '{environment,type}', '"self_hosted"') WHERE id = $1`, l.session.ID) - principal := FixtureExecutorPrincipal(t, l.s, l.tenant) - key, err := sessionService(t, l.s).IssueExecutorCredential(t.Context(), principal, uuid.NewString(), l.session.Environment.ID) - if err != nil { - t.Fatal(err) - } - l.resource.Kind, l.resource.ID = "enrollment", uuid.NewString() - l.exec("INSERT INTO sandbox_enrollments(id, environment_id, executor_key_id) VALUES($1, $2, $3)", l.resource.ID, l.resource.EnvironmentID, key.KeyID) - p := startLinkServe(t, l.relay, []byte(key.Token), l.resource.Ref()) - within(t, p.connected) - l.exec("UPDATE devices SET agent_host = true WHERE id = $1", l.device.ID) + l := newLinkHarness(t, `{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`) + p := l.served ref, payload := l.bind() - if payload.Resource == nil || *payload.Resource != l.resource { + if payload.Resource == nil || *payload.Resource != l.resource || len(payload.AttachGrant) == 0 { t.Fatalf("agent host bind = %+v", payload) } link, err := l.attach(l.device.ID, []byte(l.credential)) @@ -443,19 +397,41 @@ func TestLinkAuthorityEnrollmentRotation(t *testing.T) { } within(t, p.binds) - rotated, err := sessionService(t, l.s).RotateExecutorCredential(t.Context(), principal, key.KeyID) + var key string + if err := l.s.pool.QueryRow(t.Context(), "SELECT executor_key_id::text FROM sandbox_enrollments WHERE id = $1", l.resource.ID).Scan(&key); err != nil { + t.Fatal(err) + } + principal := FixtureExecutorPrincipal(t, l.s, l.tenant) + rotated, err := sessionService(t, l.s).RotateExecutorCredential(t.Context(), principal, key) if err != nil { t.Fatal(err) } - if _, err := l.open(link, sandboxlink.ServiceFile, ref, payload.AttachGrant); linkCode(err) != sandboxlink.StaleGeneration { - t.Fatal("an Open reached the old secret's serve peer", err) + var epoch uint64 + if err := l.s.pool.QueryRow(t.Context(), "SELECT epoch FROM session_runtime_assignments WHERE session_id = $1", l.session.ID).Scan(&epoch); err != nil || epoch != ref.Epoch+1 { + t.Fatal("rotation left the assignment at epoch", epoch, err) + } + if _, err := l.open(link, sandboxlink.ServiceFile, ref, payload.AttachGrant); err == nil { + t.Fatal("an Open reached the old secret's serve peer") + } + if err := l.renew(link, file, payload.AttachGrant); err == nil { + t.Fatal("an attachment to the old secret's serve peer renewed") + } + if got := p.refused(t); got != sandboxlink.AuthenticationFailed { + t.Fatal("the old secret's Serve ended with", got) } - if err := l.renew(link, file, payload.AttachGrant); linkCode(err) != sandboxlink.StaleGeneration { - t.Fatal("an attachment to the old secret's serve peer renewed", err) + resource, err := sessionService(t, l.s).EnrollRuntime(t.Context(), l.resource.EnvironmentID, runtimedevice.HashCredential(rotated.Token)) + if next := l.resource; err != nil || resource != func() sandboxbootstrap.Resource { next.Generation++; return next }() { + t.Fatalf("re-enrollment = %+v %v", resource, err) + } + l.resource = resource + within(t, startLinkServe(t, l.link, []byte(rotated.Token), resource.Ref()).connected) + next, payload := l.bind() + if next.Epoch != ref.Epoch+1 || payload.Resource == nil || *payload.Resource != resource { + t.Fatalf("bind after rotation = %+v %+v", next, payload) + } + if _, err := l.open(link, sandboxlink.ServiceFile, next, payload.AttachGrant); err != nil { + t.Fatal("the new generation did not open", err) } - next := l.resource - next.Generation++ - within(t, startLinkServe(t, l.relay, []byte(rotated.Token), next.Ref()).connected) } // TestLinkAuthorityDestroyedAllocation checks that the Worker's cleanup of an @@ -493,7 +469,7 @@ func TestLinkAuthorityDestroyedAllocation(t *testing.T) { // accept its credential, a second startup changes nothing, and another // device's ID is never taken over. func TestRegisteredAgentHostAuthenticates(t *testing.T) { - s, _ := testStore(t) + s, _ := newManagedTestStore(t) dir := t.TempDir() runtime, credential := uuid.NewString(), uuid.NewString() identity, _ := json.Marshal(map[string]string{"runtime_id": runtime, "credential": credential}) @@ -551,62 +527,57 @@ func TestRegisteredAgentHostAuthenticates(t *testing.T) { } } -// TestInitializationBindsAgentHost binds a hosted Session to the registered -// agent host and runs the Environment's initialization once its Link resource -// is Serving. The bind carries the resource and an attach grant. A bind that -// fails before any effect, here because the agent host's connection closes, -// leaves the initialization unclaimed, and a later pass completes it. +// TestInitializationBindsAgentHost places a hosted and a self_hosted Session +// on the registered agent host and runs each Environment's initialization once +// its Link resource is Serving. The bind carries the resource and an attach +// grant. A bind that fails before any effect, here because the agent host's +// connection closes, leaves the initialization unclaimed, and a later pass +// completes it. func TestInitializationBindsAgentHost(t *testing.T) { - s, _ := newManagedTestStore(t) - tenant, host, credential := uuid.NewString(), uuid.NewString(), uuid.NewString() - if err := sessionAdapter(s).RegisterAgentHost(t.Context(), host, runtimedevice.HashCredential(credential)); err != nil { - t.Fatal(err) - } - session, err := s.CreateSession(t.Context(), tenant, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), - Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted"}}`), - InitialFiles: []environmentconfig.InitialFile{{Type: "inline", Path: "/workspace/input", Data: []byte("frozen")}}})) - if err != nil { - t.Fatal(err) - } - // Placement does not choose the agent host yet. - if _, err := s.pool.Exec(t.Context(), "INSERT INTO session_runtime_assignments(session_id, runtime_id) VALUES($1, $2)", session.ID, host); err != nil { - t.Fatal(err) - } - resource := sandboxbootstrap.Resource{TenantID: tenant, EnvironmentID: session.Environment.ID, Kind: "allocation", ID: uuid.NewString(), Generation: 1} - device, err := sessionService(t, s).CreateDevice(t.Context(), tenant, "sandbox", runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } - serve := []byte(uuid.NewString()) - insertAllocation(t, s, resource, device.ID, serve) - server := httptest.NewUnstartedServer(nil) - endpoint := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" - handler, registry, err := runtime.NewGateway(sessionAdapter(s), sessionService(t, s), sessionAdapter(s), runtimegateway.NewLinkAuthority(sessionAdapter(s)), endpoint) - if err != nil { - t.Fatal(err) - } - server.Config.Handler = handler - server.Start() - t.Cleanup(func() { server.Close(); runtime.CloseConnections(registry) }) - link := startLinkRoute(t, s) - runWorker(t, startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, Links: link.Relay})) - within(t, startLinkServe(t, link, serve, resource.Ref()).connected) - - dropped := &initializationPeer{apply: completedInitialization, binds: make(chan proto.AssignmentBindPayload, 1), closeOnBind: true} - dropped.setRuntimeGateway(t, endpoint, registry, nil) - if err := dropped.connect(sandbox.Bootstrap{DeviceID: host, Credential: credential}); err != nil { - t.Fatal(err) - } - within(t, dropped.binds) - awaitInitialization(t, s, tenant, session.Environment.ID, "pending") - - peer := &initializationPeer{apply: completedInitialization, binds: make(chan proto.AssignmentBindPayload, 1)} - peer.setRuntimeGateway(t, endpoint, registry, nil) - if err := peer.connect(sandbox.Bootstrap{DeviceID: host, Credential: credential}); err != nil { - t.Fatal(err) - } - if bind := within(t, peer.binds); bind.EnvironmentID != session.Environment.ID || bind.Resource == nil || *bind.Resource != resource || len(bind.AttachGrant) == 0 { - t.Fatalf("agent host bind = %+v", bind) + for _, environment := range []string{`{"type":"openai_hosted"}`, `{"type":"self_hosted","workspace_directory":"/workspace"}`} { + t.Run(environment, func(t *testing.T) { + s, _ := newManagedTestStore(t) + tenant, host := uuid.NewString(), registerAgentHost(t, s, "") + session, err := s.CreateSession(t.Context(), tenant, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), + Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":` + environment + `}`), + InitialFiles: []environmentconfig.InitialFile{{Type: "inline", Path: "/workspace/input", Data: []byte("frozen")}}})) + if err != nil { + t.Fatal(err) + } + resource, serve := fixtureLinkResource(t, s, tenant, session) + server := httptest.NewUnstartedServer(nil) + endpoint := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" + handler, registry, err := runtime.NewGateway(sessionAdapter(s), sessionService(t, s), sessionAdapter(s), runtimegateway.NewLinkAuthority(sessionAdapter(s)), endpoint) + if err != nil { + t.Fatal(err) + } + server.Config.Handler = handler + server.Start() + t.Cleanup(func() { server.Close(); runtime.CloseConnections(registry) }) + link := startLinkRoute(t, s) + runWorker(t, startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, Links: link.Relay})) + within(t, startLinkServe(t, link, serve, resource.Ref()).connected) + + dropped := &initializationPeer{apply: completedInitialization, binds: make(chan proto.AssignmentBindPayload, 1), closeOnBind: true, host: host} + dropped.setRuntimeGateway(t, s, endpoint, registry, nil) + if err := dropped.connect(sandbox.Bootstrap{}); err != nil { + t.Fatal(err) + } + within(t, dropped.binds) + awaitInitialization(t, s, tenant, session.Environment.ID, "pending") + + peer := &initializationPeer{apply: completedInitialization, binds: make(chan proto.AssignmentBindPayload, 1), host: host} + peer.setRuntimeGateway(t, s, endpoint, registry, nil) + if err := peer.connect(sandbox.Bootstrap{}); err != nil { + t.Fatal(err) + } + if bind := within(t, peer.binds); bind.EnvironmentID != session.Environment.ID || bind.Resource == nil || *bind.Resource != resource || len(bind.AttachGrant) == 0 { + t.Fatalf("agent host bind = %+v", bind) + } + awaitInitialization(t, s, tenant, session.Environment.ID, "complete") + if bound, err := sessionAdapter(s).GetSessionDevice(t.Context(), tenant, session.ID); err != nil || bound.ID != host.ID { + t.Fatal("Session placed on", bound.ID, err) + } + }) } - awaitInitialization(t, s, tenant, session.Environment.ID, "complete") } diff --git a/services/core/tests/integration/list_cursor_public_test.go b/services/core/tests/integration/list_cursor_public_test.go index 0586fc74a..dffcb2710 100644 --- a/services/core/tests/integration/list_cursor_public_test.go +++ b/services/core/tests/integration/list_cursor_public_test.go @@ -95,10 +95,7 @@ func seedCursorFixture(t *testing.T, s *Store, leased execution.Owner, skillServ if err != nil { t.Fatal(err) } - host, err := sessionService.CreateDevice(ctx, tenant, "cursor "+key, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } + host := registerAgentHost(t, s, tenant) if err = leased.Sessions.BindSessionDevice(ctx, tenant, created.ID, host.ID); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/local_environment_devices_test.go b/services/core/tests/integration/local_environment_devices_test.go index 27e3a22f5..9db95e081 100644 --- a/services/core/tests/integration/local_environment_devices_test.go +++ b/services/core/tests/integration/local_environment_devices_test.go @@ -3,6 +3,7 @@ package integration import ( "encoding/json" "errors" + "slices" "sync" "testing" @@ -38,23 +39,22 @@ func TestEnvironmentDeviceAuthorityAndLifecycle(t *testing.T) { t.Fatalf("foreign provisioning: %v", err) } bound, err := FixtureEnvironmentDevice(t, t.Context(), pool, tenant, environment.ID, "dedicated", digest) - if err != nil || bound.EnvironmentID != environment.ID { + if err != nil { t.Fatalf("provision: %+v %v", bound, err) } + // Sessions are placed on agent hosts only: the device binds no Session. execution := sessionExecution(t, executionWriter(t, s).lease) - for _, other := range []sessions.Session{sibling, foreign} { + for _, other := range []sessions.Session{session, sibling, foreign} { if err := execution.BindSessionDevice(t.Context(), other.TenantID, other.ID, bound.ID); err == nil { - t.Fatal("dedicated credential bound to another Session") + t.Fatal("an Environment device was bound to a Session") } } - devices, err := sessionAdapter(s).ListExecutionDevices(t.Context(), tenant) - if err != nil || len(devices) != 0 { - t.Fatalf("dedicated device entered general selection: %v %v", devices, err) + if _, err := sessionAdapter(s).GetSessionDevice(t.Context(), tenant, session.ID); !errors.Is(err, sessions.ErrNotFound) { + t.Fatalf("provisioning bound the Session: %v", err) } - reopened, _ := testStore(t) - got, err := sessionAdapter(reopened).GetSessionDevice(t.Context(), tenant, session.ID) - if err != nil || got.ID != bound.ID || got.EnvironmentID != bound.EnvironmentID || got.Assignment.SessionID != session.ID || got.Assignment.Epoch != 1 || !got.Assignment.Valid() { - t.Fatalf("durable exact binding: %+v %v", got, err) + hosts, err := sessionAdapter(s).ListAgentHosts(t.Context(), tenant) + if err != nil || slices.ContainsFunc(hosts, func(host sessions.ExecutionDevice) bool { return host.ID == bound.ID }) { + t.Fatalf("an Environment device entered placement: %v %v", hosts, err) } if _, ok, err := sessionAdapter(s).GetDeviceCredential(t.Context(), bound.ID); err != nil || !ok { t.Fatalf("valid credential unavailable: %v", err) @@ -74,7 +74,7 @@ func TestEnvironmentDeviceAuthorityAndLifecycle(t *testing.T) { func TestEnvironmentDeviceProvisioningHasOneWinner(t *testing.T) { s, pool := testStore(t) tenant := uuid.NewString() - session, environment := localEnvironment(t, s, tenant) + _, environment := localEnvironment(t, s, tenant) var wg sync.WaitGroup results := make(chan error, 6) for range 6 { @@ -98,11 +98,11 @@ func TestEnvironmentDeviceProvisioningHasOneWinner(t *testing.T) { if winners != 1 { t.Fatalf("provisioned %d devices", winners) } - bound, err := sessionAdapter(s).GetSessionDevice(t.Context(), tenant, session.ID) - if err != nil { + var device string + if err := pool.QueryRow(t.Context(), "SELECT id::text FROM devices WHERE environment_id = $1", environment.ID).Scan(&device); err != nil { t.Fatal(err) } - if err := sessionService(t, s).RevokeDevice(t.Context(), tenant, bound.ID); err != nil { + if err := sessionService(t, s).RevokeDevice(t.Context(), tenant, device); err != nil { t.Fatal(err) } if _, err := FixtureEnvironmentDevice(t, t.Context(), pool, tenant, environment.ID, "replacement", runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, sessions.ErrDeviceBindingConflict) { diff --git a/services/core/tests/integration/local_environment_file_write_test.go b/services/core/tests/integration/local_environment_file_write_test.go index 3f01b683d..251617bbf 100644 --- a/services/core/tests/integration/local_environment_file_write_test.go +++ b/services/core/tests/integration/local_environment_file_write_test.go @@ -38,7 +38,7 @@ func awaitLocalWrite(t *testing.T, done <-chan localWriteResult) localWriteResul } func TestLocalEnvironmentFileWriteOwnsMutationBeforeDispatch(t *testing.T) { - h, w, environment := localWorker(t, true, false) + h, w, environment := localWorker(t, false) foreign := environment foreign.TenantID = uuid.NewString() if _, err := w.WriteEnvironmentFile(t.Context(), foreign, "input", nil); !errors.Is(err, sessions.ErrNotFound) { @@ -92,7 +92,7 @@ func TestLocalEnvironmentFileWriteOwnsMutationBeforeDispatch(t *testing.T) { } func TestLocalEnvironmentFileWriteLostReceiptRemainsPending(t *testing.T) { - h, w, environment := localWorker(t, true, false) + h, w, environment := localWorker(t, false) done := startLocalWrite(t.Context(), w, environment) begin := h.read(proto.TypeWorkspaceWrite) if err := h.conn.Close(); err != nil { @@ -111,7 +111,7 @@ func TestLocalEnvironmentFileWriteLostReceiptRemainsPending(t *testing.T) { } func TestLocalEnvironmentFileWriteKnownRejectionReleasesMutation(t *testing.T) { - h, w, environment := localWorker(t, true, false) + h, w, environment := localWorker(t, false) for range 2 { done := startLocalWrite(t.Context(), w, environment) begin := h.read(proto.TypeWorkspaceWrite) @@ -126,9 +126,10 @@ func TestLocalEnvironmentFileWriteKnownRejectionReleasesMutation(t *testing.T) { } } -func TestLocalEnvironmentFileWriteRejectsUnscopedDevice(t *testing.T) { - _, w, environment := localWorker(t, false, false) +func TestLocalEnvironmentFileWriteRejectsUnservedEnvironment(t *testing.T) { + h, w, environment := localWorker(t, false) + h.stopServing() if _, err := w.WriteEnvironmentFile(t.Context(), environment, "input", nil); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatal("unscoped writer selected", err) + t.Fatal("an Environment that is not Serving was written", err) } } diff --git a/services/core/tests/integration/local_environment_worker_test.go b/services/core/tests/integration/local_environment_worker_test.go index 937af981f..420d71d92 100644 --- a/services/core/tests/integration/local_environment_worker_test.go +++ b/services/core/tests/integration/local_environment_worker_test.go @@ -13,13 +13,9 @@ import ( "github.com/google/uuid" ) -func localWorker(t *testing.T, scoped, execute bool) (*dispatchHarness, *execution.Worker, sessions.Environment) { +func localWorker(t *testing.T, execute bool) (*dispatchHarness, *execution.Worker, sessions.Environment) { t.Helper() - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), scoped) - if scoped { - _, pool := testStore(t) - insertWorkerRuntimeAllocation(t, pool, h, "disabled") - } + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`)) environment, err := sessionAdapter(h.s).GetSessionEnvironment(t.Context(), h.tenant, h.session.ID) if err != nil { t.Fatal(err) @@ -53,7 +49,7 @@ func localWorker(t *testing.T, scoped, execute bool) (*dispatchHarness, *executi } func TestLocalEnvironmentWorkerDirectoryUsesExactAuthorityWithoutModel(t *testing.T) { - h, w, environment := localWorker(t, true, false) + h, w, environment := localWorker(t, false) foreign := environment foreign.TenantID = uuid.NewString() if _, err := w.ReadEnvironmentDirectory(t.Context(), foreign, "reports"); !errors.Is(err, sessions.ErrNotFound) { @@ -82,10 +78,13 @@ func TestLocalEnvironmentWorkerDirectoryUsesExactAuthorityWithoutModel(t *testin } } -func TestLocalEnvironmentWorkerRejectsGeneralDeviceDespiteCapability(t *testing.T) { - h, w, environment := localWorker(t, false, false) +// An Environment whose Link resource is not Serving, or that has none, has no +// directory to read. +func TestLocalEnvironmentWorkerRejectsUnservedEnvironment(t *testing.T) { + h, w, environment := localWorker(t, false) + h.stopServing() if _, err := w.ReadEnvironmentDirectory(t.Context(), environment, "reports"); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatal("general device used as local authority", err) + t.Fatal("an Environment that is not Serving was read", err) } other, err := h.s.CreateSession(t.Context(), h.tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: "unassigned", Configuration: h.session.Configuration}) if err != nil { @@ -96,7 +95,7 @@ func TestLocalEnvironmentWorkerRejectsGeneralDeviceDespiteCapability(t *testing. t.Fatal(err) } if _, err := w.ReadEnvironmentDirectory(t.Context(), unassigned, "reports"); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatal("unassigned environment selected general device", err) + t.Fatal("an Environment without a Link resource was read", err) } if _, err := sessionAdapter(h.s).GetSessionDevice(t.Context(), h.tenant, other.ID); !errors.Is(err, sessions.ErrNotFound) { t.Fatal("read persisted an unauthorized placement", err) @@ -104,7 +103,7 @@ func TestLocalEnvironmentWorkerRejectsGeneralDeviceDespiteCapability(t *testing. } func TestLocalEnvironmentWorkerSchedulesPreparationWithoutRemoteResolver(t *testing.T) { - h, worker, environment := localWorker(t, true, true) + h, worker, environment := localWorker(t, true) reservation, err := sessionService(t, h.s).ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "local-input", []sessions.Input{messageInput("first")}) if err != nil { t.Fatal(err) @@ -145,7 +144,7 @@ func TestLocalEnvironmentWorkerSchedulesPreparationWithoutRemoteResolver(t *test t.Fatal("local reservation did not settle", err) } bound, err := sessionAdapter(h.s).GetSessionExecutionBinding(t.Context(), h.tenant, h.session.ID) - if err != nil || bound.Device.EnvironmentID != environment.ID || bound.NativeSessionID != "local-native-history" { + if err != nil || bound.Device.ID != h.device.ID || bound.NativeSessionID != "local-native-history" { t.Fatal("local native identity was not retained", err) } artifacts, err := sessionAdapter(h.s).ListSessionArtifacts(t.Context(), h.tenant, h.session.ID, environment.ID, "", 20, false) diff --git a/services/core/tests/integration/native_recovery_test.go b/services/core/tests/integration/native_recovery_test.go index db4d7d859..9d0207f59 100644 --- a/services/core/tests/integration/native_recovery_test.go +++ b/services/core/tests/integration/native_recovery_test.go @@ -11,7 +11,7 @@ func TestSessionExecutionBindingRetainsStartedExecutionRequirement(t *testing.T) s, pool := testStore(t) tenant, session := newTurnSession(t, s) foreign, _ := newTurnSession(t, s) - device, _ := registerTestDevice(t, s, tenant) + device := registerAgentHost(t, s, tenant) // The bind runs on an execution lease of its own, which closes before the // pool does. writer := executionWriter(t, s) diff --git a/services/core/tests/integration/prepared_dispatch_test.go b/services/core/tests/integration/prepared_dispatch_test.go index 413db2cd6..ca9a7a7a7 100644 --- a/services/core/tests/integration/prepared_dispatch_test.go +++ b/services/core/tests/integration/prepared_dispatch_test.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) @@ -20,7 +21,7 @@ type preparedDispatchResult struct { func preparedDispatchHarness(t *testing.T) (*dispatchHarness, sessions.EnvironmentInputReservation) { t.Helper() - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model","instructions":"Keep this instruction.","x_agents_core":{"harness_config":{"model_reasoning_effort":"low"}}},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`), true) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model","instructions":"Keep this instruction.","x_agents_core":{"harness_config":{"model_reasoning_effort":"low"}}},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`)) assertNoRuntimeAllocation(t, h) h.d, h.lease = h.bound(), h.owner().Lease enableWorkerEnvironment(t, h) @@ -72,12 +73,29 @@ func readyPreparedDispatch(t *testing.T, h *dispatchHarness, request, handle str return start } +// TestSelfHostedProviderBundleStaysOnTheAgentHost checks that a self_hosted +// Session's model provider bundle reaches only the agent host. The machine's +// executor credential Serves the enrollment's Link resource and authenticates +// no Runtime, so no Runtime frame reaches the machine. +func TestSelfHostedProviderBundleStaysOnTheAgentHost(t *testing.T) { + h, pending := preparedDispatchHarness(t) + runPreparedDispatch(h, t.Context(), pending) + var prepare proto.ExecutionPreparePayload + if frame := h.read(proto.TypeExecutionPrepare); frame.DecodePayload(&prepare) != nil || prepare.Configuration.ModelProvider == nil || prepare.Configuration.ModelProvider.APIKey != FixtureModelProvider("codex").APIKey { + t.Fatal("the agent host did not receive the provider bundle", prepare.Configuration.ModelProvider) + } + var runtimes int + if err := h.s.pool.QueryRow(t.Context(), "SELECT count(*) FROM devices WHERE credential_hash = $1", runtimedevice.HashCredential(string(h.serve))).Scan(&runtimes); err != nil || runtimes != 0 { + t.Fatal("the executor credential authenticates a Runtime", runtimes, err) + } +} + func TestPreparedDispatchPromotesOriginalBatchAndPersistsCompletion(t *testing.T) { h, pending := preparedDispatchHarness(t) result := runPreparedDispatch(h, t.Context(), pending) frame := h.read(proto.TypeExecutionPrepare) var prepare proto.ExecutionPreparePayload - if frame.DecodePayload(&prepare) != nil || prepare.Configuration.LocalEnvironment == nil || prepare.Configuration.LocalEnvironment.ID != h.device.EnvironmentID || prepare.Configuration.DisableExecutionEnvironment { + if frame.DecodePayload(&prepare) != nil || prepare.Configuration.LocalEnvironment == nil || prepare.Configuration.LocalEnvironment.ID != h.session.Environment.ID || prepare.Configuration.DisableExecutionEnvironment { t.Fatal("invalid preparation configuration", prepare) } session, err := sessionAdapter(h.s).GetSession(t.Context(), h.tenant, h.session.ID) diff --git a/services/core/tests/integration/runtime_allocations_test.go b/services/core/tests/integration/runtime_allocations_test.go index 165f31b85..ec0efc372 100644 --- a/services/core/tests/integration/runtime_allocations_test.go +++ b/services/core/tests/integration/runtime_allocations_test.go @@ -24,9 +24,8 @@ func TestRuntimeAllocationAtomicOwnershipAndRecovery(t *testing.T) { if err != nil || owner.Replayed || owner.State != "creating" || owner.CreateSettled { t.Fatalf("reservation: %+v %v", owner, err) } - bound, err := sessionAdapter(s).GetSessionDevice(t.Context(), tenant, session.ID) - if err != nil || bound.ID != owner.DeviceID || bound.EnvironmentID != environment.ID { - t.Fatalf("binding not committed with allocation: %+v %v", bound, err) + if _, err := sessionAdapter(s).GetSessionDevice(t.Context(), tenant, session.ID); !errors.Is(err, sessions.ErrNotFound) { + t.Fatalf("the reservation bound the Session: %v", err) } if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: uuid.NewString(), EnvironmentID: environment.ID}, provider, runtimedevice.HashCredential(secret), runtimedevice.HashCredential(secret)); !errors.Is(err, sessions.ErrNotFound) { t.Fatalf("foreign allocation accepted: %v", err) diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 163b5983b..762b29919 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -15,6 +15,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/sandboxlinktest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -26,38 +27,99 @@ import ( // The controlled provider records external effects independently of DB phases. // Lost replies retain those effects so recovery must use observation, not replay. +// Each running compute Serves its allocation's Link resource at link, and the +// deployment's agent host, which runs the Sessions, connects to the gateway. type fakeCheckpointProvider struct { + t *testing.T preparation *initializationPeer lifecycleProvider computes map[string]sandbox.ComputeState snapshots map[string]sandbox.SnapshotIdentity bootstraps map[string]sandbox.Bootstrap - peers map[string]*websocket.Conn + serving map[string]*linkServe // by allocation + host agentHost + hostConn *websocket.Conn registry *runtimegateway.Registry + link *sandboxlinktest.Server endpoint string captures, restores, captureObservations, restoreObservations int - computeKills, snapshotDeletes, wakeCommands int + computeKills, snapshotDeletes int promptFrames atomic.Int32 quiesces, resumes atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() } +func newFakeCheckpointProvider(t *testing.T, s *Store) *fakeCheckpointProvider { + t.Helper() + p := &fakeCheckpointProvider{t: t, lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, serving: map[string]*linkServe{}, + host: registerAgentHost(t, s, ""), registry: runtimegateway.NewRegistry(), link: sandboxlinktest.StartRelay(t, runtimegateway.NewLinkAuthority(sessionAdapter(s)))} + handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: p.registry}) + server := httptest.NewServer(http.HandlerFunc(handler.WS)) + p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") + t.Cleanup(func() { + p.mu.Lock() + if p.hostConn != nil { + p.hostConn.Close() + } + p.mu.Unlock() + server.Close() + }) + return p +} + func (p *fakeCheckpointProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{Name: r.AllocationID + "-g0"}, nil } func (p *fakeCheckpointProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.SnapshotIdentity) (sandbox.Compute, error) { return sandbox.Compute{Generation: generation, Name: fmt.Sprintf("%s-g%d", r.AllocationID, generation), RestoredFrom: parent}, nil } + +// Create starts the compute, which Serves its Link resource, and connects +// the agent host unless it is connected. func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { info, err := p.lifecycleProvider.Create(ctx, b) + if err != nil { + return info, err + } p.mu.Lock() - defer p.mu.Unlock() current, _ := p.Initial(ctx, b.Reference) current.ID = uuid.NewString() p.computes[current.Name] = sandbox.ComputeState{Compute: current, Status: "running", BootstrapComplete: true} p.bootstraps[b.AllocationID] = b - return info, err + p.mu.Unlock() + if served := p.serve(b.AllocationID); served != nil { + select { + case <-served.connected: + case <-ctx.Done(): + return info, ctx.Err() + } + } + return info, p.connectHost(ctx, false) +} + +// serve starts the allocation's Serve unless it runs, and returns the new one. +func (p *fakeCheckpointProvider) serve(allocation string) *linkServe { + p.mu.Lock() + defer p.mu.Unlock() + if p.serving[allocation] != nil { + return nil + } + io := p.bootstraps[allocation].SandboxIO + served := startLinkServe(p.t, p.link, []byte(io.Credential), io.Resource.Ref()) + p.serving[allocation] = served + return served +} + +// stopServe ends the allocation's Serve, as a paused or killed sandbox does. +func (p *fakeCheckpointProvider) stopServe(allocation string) { + p.mu.Lock() + served := p.serving[allocation] + delete(p.serving, allocation) + p.mu.Unlock() + if served != nil { + served.stop() + } } func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { p.mu.Lock() @@ -72,6 +134,12 @@ func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Referen return state, nil } func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + paused := false + defer func() { + if paused { + p.stopServe(q.Reference.AllocationID) + } + }() p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[q.Source.Name] @@ -89,7 +157,7 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq return sandbox.ComputeState{}, errors.New("capture replayed") } p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} - state.Status = "paused" + state.Status, paused = "paused", true p.computes[q.Source.Name] = state if p.loseCapture { p.loseCapture = false @@ -102,6 +170,12 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq return state, nil } func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + restored := false + defer func() { + if restored { + p.serve(q.Reference.AllocationID) + } + }() p.mu.Lock() defer p.mu.Unlock() if q.ObserveOnly { @@ -120,13 +194,20 @@ func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeReque target.ID = uuid.NewString() state := sandbox.ComputeState{Compute: target, Status: "running", BootstrapComplete: true} p.computes[target.Name] = state + restored = true if p.loseRestore { p.loseRestore = false return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed } return state, nil } -func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { +func (p *fakeCheckpointProvider) KillCompute(_ context.Context, r sandbox.Reference, c sandbox.Compute) error { + killed := false + defer func() { + if killed { + p.stopServe(r.AllocationID) + } + }() p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -138,6 +219,7 @@ func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Refere } p.computeKills++ delete(p.computes, c.Name) + killed = true return nil } func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Reference, s sandbox.SnapshotIdentity) error { @@ -160,27 +242,32 @@ func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Re return state, err } p.mu.Lock() - defer p.mu.Unlock() state.Status = "running" p.computes[c.Name] = state + p.mu.Unlock() + p.serve(r.AllocationID) return state, nil } -func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { - if _, err := p.GetCompute(ctx, r, c); err != nil { - return sandbox.CommandResult{}, err +func (p *fakeCheckpointProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { + return sandbox.CommandResult{}, errors.New("unexpected compute command") +} + +// connectHost connects the agent host unless it is connected; restart drops +// its connection first, as a restarted agent host does. +func (p *fakeCheckpointProvider) connectHost(ctx context.Context, restart bool) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.hostConn != nil && !restart { + return nil } - if len(command.Args) != 8 || command.Args[0] != "oac-daemon" || command.Args[1] != "resume" || command.Args[5] != r.EnvironmentID { - return sandbox.CommandResult{}, errors.New("unexpected wake command") + var previous *runtimegateway.Session + if p.hostConn != nil { + previous, _ = p.registry.LookupDevice(p.host.ID) + p.hostConn.Close() + p.hostConn = nil } - p.mu.Lock() - p.wakeCommands++ - b := p.bootstraps[r.AllocationID] - p.mu.Unlock() - return sandbox.CommandResult{}, p.connect(ctx, b) -} -func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { - header := http.Header{"Authorization": []string{"Bearer " + b.Credential}} - conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, header) + header := http.Header{"Authorization": []string{"Bearer " + p.host.Credential}} + conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+p.host.ID+"&version="+proto.Version, header) if err != nil { return err } @@ -189,74 +276,70 @@ func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstra conn.Close() return err } - p.mu.Lock() - p.peers[b.AllocationID] = conn - p.mu.Unlock() - if _, err = p.registry.WaitForDevice(ctx, b.DeviceID, time.Second); err != nil { - conn.Close() - return err + p.hostConn = conn + for deadline := time.Now().Add(time.Second); ; time.Sleep(time.Millisecond) { + if current, err := p.registry.LookupDevice(p.host.ID); err == nil && current != previous { + break + } + if time.Now().After(deadline) { + return context.DeadlineExceeded + } } - go func() { - defer conn.Close() - transfer := initializationTransfer{peer: p.preparation} - for { - var env proto.Envelope - if conn.ReadJSON(&env) != nil { - return - } - if reply, ok := assignmentReply(env); ok { - if conn.WriteJSON(reply) != nil { - return - } - continue - } - if env.Type == proto.TypeRuntimePrepare && p.preparation != nil { - reply, err := transfer.receive(env) - if err != nil { - p.preparation.t.Error(err) - return - } - if conn.WriteJSON(reply) != nil { - return - } - continue - } - if env.Type != proto.TypeEnvironmentQuiesce && env.Type != proto.TypeEnvironmentResume { - p.mu.Lock() - p.promptFrames.Add(1) - p.mu.Unlock() - continue - } - var request proto.EnvironmentSuspendPayload - if env.DecodePayload(&request) != nil { - return - } - p.mu.Lock() - reject := p.rejectQuiesce && env.Type == proto.TypeEnvironmentQuiesce - if env.Type == proto.TypeEnvironmentQuiesce { - p.quiesces.Add(1) - } else { - p.resumes.Add(1) - } - beforeQuiesce := p.beforeQuiesce - p.mu.Unlock() - if env.Type == proto.TypeEnvironmentQuiesce && beforeQuiesce != nil { - beforeQuiesce() - } - kind := proto.TypeEnvironmentResumed - if env.Type == proto.TypeEnvironmentQuiesce { - kind = proto.TypeEnvironmentQuiesced - } - reply, _ := env.Reply(kind, proto.EnvironmentSuspendResultPayload{EnvironmentID: request.EnvironmentID, SuspendID: request.SuspendID, Accepted: !reject}) - if conn.WriteJSON(reply) != nil { + go p.answer(conn) + return nil +} + +// answer replies to binds, preparation and suspension control on conn. +func (p *fakeCheckpointProvider) answer(conn *websocket.Conn) { + defer conn.Close() + transfer := newInitializationTransfer(p.preparation, conn) + for { + var env proto.Envelope + if conn.ReadJSON(&env) != nil { + return + } + if reply, ok := assignmentReply(env); ok { + if transfer.write(reply) != nil { return } - if env.Type == proto.TypeEnvironmentQuiesce && !reject { + continue + } + if env.Type == proto.TypeRuntimePrepare && p.preparation != nil { + if err := transfer.receive(env); err != nil { + p.preparation.t.Error(err) return } + continue } - }() - return nil + if env.Type != proto.TypeEnvironmentQuiesce && env.Type != proto.TypeEnvironmentResume { + p.promptFrames.Add(1) + continue + } + var request proto.EnvironmentSuspendPayload + if env.DecodePayload(&request) != nil { + return + } + p.mu.Lock() + reject := p.rejectQuiesce && env.Type == proto.TypeEnvironmentQuiesce + if env.Type == proto.TypeEnvironmentQuiesce { + p.quiesces.Add(1) + } else { + p.resumes.Add(1) + } + beforeQuiesce := p.beforeQuiesce + p.mu.Unlock() + if env.Type == proto.TypeEnvironmentQuiesce && beforeQuiesce != nil { + beforeQuiesce() + } + kind := proto.TypeEnvironmentResumed + if env.Type == proto.TypeEnvironmentQuiesce { + kind = proto.TypeEnvironmentQuiesced + } + reply, _ := env.Reply(kind, proto.EnvironmentSuspendResultPayload{EnvironmentID: request.EnvironmentID, SuspendID: request.SuspendID, Accepted: !reject}) + if transfer.write(reply) != nil { + return + } + } } type computeLifecycleFixture struct { @@ -273,20 +356,7 @@ type computeLifecycleFixture struct { func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *computeLifecycleFixture { t.Helper() s, _ := newManagedTestStore(t) - registry := runtimegateway.NewRegistry() - p := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} - handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: registry}) - server := httptest.NewServer(http.HandlerFunc(handler.WS)) - p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") - t.Cleanup(func() { - p.mu.Lock() - defer p.mu.Unlock() - for _, peer := range p.peers { - peer.Close() - } - server.Close() - }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour}} + f := &computeLifecycleFixture{t: t, store: s, provider: newFakeCheckpointProvider(t, s), key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour}} view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) @@ -298,7 +368,7 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu func (f *computeLifecycleFixture) start() { t := f.t t.Helper() - w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) + w := startWebWorker(t, f.store, f.provider.registry, f.provider.link.Relay, f.key, f.provider, &f.policy) // The Worker's claim starts a new owner epoch, in which the node reconnects. onlineManagerNode(t, f.store, f.node) var once sync.Once @@ -318,17 +388,10 @@ func (f *computeLifecycleFixture) create() (string, sessions.Session, sessions.E t := f.t t.Helper() tenant, session, environment := managedSession(t, f.store) - owner, err := f.worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, f.key) - if err != nil { + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, f.key); err != nil { t.Fatal(err) } - f.provider.mu.Lock() - b := f.provider.bootstraps[owner.ID] - f.provider.mu.Unlock() - if err := f.provider.connect(t.Context(), b); err != nil { - t.Fatal(err) - } - owner = f.phase(tenant, environment.ID, "running") + owner := f.phase(tenant, environment.ID, "running") return tenant, session, environment, owner } func (f *computeLifecycleFixture) phase(tenant, environment, phase string) deployment.Allocation { @@ -352,8 +415,12 @@ func (f *computeLifecycleFixture) phase(tenant, environment, phase string) deplo f.t.Fatalf("compute phase=%s, want %s state=%s", owner.ComputePhase, phase, owner.ComputeState) return owner } + +// complete records a Turn that ran on the agent host, where placement bound +// the Session. func (f *computeLifecycleFixture) complete(owner deployment.Allocation) string { id := uuid.NewString() + assignSession(f.t, f.store, owner.SessionID, f.provider.host.ID) f.sql(`INSERT INTO turns(id,session_id,status,completed_at) VALUES($1,$2,'completed',clock_timestamp()-interval '2 minutes')`, id, owner.SessionID) f.sql(`UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes' WHERE id=$1`, owner.ID) return id @@ -378,9 +445,14 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T if f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.computes) != 0 || len(f.provider.snapshots) != 1 { t.Fatal("capture did not release source compute") } + // The agent host restarts while the Environment is suspended; the wake + // resumes it on the new connection. + if err := f.provider.connectHost(t.Context(), true); err != nil { + t.Fatal(err) + } queued := f.queued(suspended) awake := f.phase(tenant, env.ID, "running") - if awake.SessionID != session.ID || awake.ID != owner.ID || awake.DeviceID != owner.DeviceID || f.provider.creates != 1 || f.provider.restores != 1 || f.provider.snapshotDeletes != 1 { + if awake.SessionID != session.ID || awake.ID != owner.ID || awake.DeviceID != owner.DeviceID || f.provider.creates != 1 || f.provider.restores != 1 || f.provider.snapshotDeletes != 1 || f.provider.resumes.Load() != 1 { t.Fatal("wake replaced Session or replayed allocation") } var completedCount, queuedCount int diff --git a/services/core/tests/integration/runtime_enrollment_connection_test.go b/services/core/tests/integration/runtime_enrollment_connection_test.go index effbdf6a9..9114aa906 100644 --- a/services/core/tests/integration/runtime_enrollment_connection_test.go +++ b/services/core/tests/integration/runtime_enrollment_connection_test.go @@ -5,24 +5,24 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "net/http" "net/http/httptest" - "net/url" "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/sandboxlinktest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtime" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeenrollment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" - "github.com/gorilla/websocket" ) -func TestEnrolledDaemonConnectionRevocationAndRestart(t *testing.T) { +// A self_hosted Environment is connected while its enrollment is Serving. A +// rotation or revocation ends the old secret's Serve through the Worker's +// revocation pass, and a restarted Worker observes the same Serve. +func TestEnrolledSandboxConnectionRevocationAndRestart(t *testing.T) { s, _ := testStore(t) principal := FixtureExecutorPrincipal(t, s, uuid.NewString()) session, err := s.CreateSession(t.Context(), principal.TenantID, sessions.CreateSession{ @@ -45,13 +45,8 @@ func TestEnrolledDaemonConnectionRevocationAndRestart(t *testing.T) { if err != nil { t.Fatal(err) } - server := httptest.NewUnstartedServer(nil) - wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" - handler, registry, err := runtime.NewGateway(sessionAdapter(s), sessionService(t, s), sessionAdapter(s), runtimegateway.NewLinkAuthority(sessionAdapter(s)), wsURL) - if err != nil { - t.Fatal(err) - } - connection := runtimeenrollment.ConnectionHandler(sessionAdapter(s), registry) + link := sandboxlinktest.StartRelay(t, runtimegateway.NewLinkAuthority(sessionAdapter(s))) + connection := runtimeenrollment.ConnectionHandler(sessionAdapter(s), link.Relay) assertConnection := func(target, token, status string, code int) { t.Helper() request := httptest.NewRequest("GET", "/api/v1/agent-daemon/connection?environment_id="+target, nil) @@ -81,11 +76,8 @@ func TestEnrolledDaemonConnectionRevocationAndRestart(t *testing.T) { t.Fatal(err) } assertConnection(environment.ID, foreignKey.Token, "", 401) - server.Config.Handler = handler - server.Start() - t.Cleanup(func() { server.Close(); runtime.CloseConnections(registry) }) start := func() func() { - worker := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry}) + worker := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), Links: link.Relay}) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- worker.Run(ctx) }() @@ -104,19 +96,6 @@ func TestEnrolledDaemonConnectionRevocationAndRestart(t *testing.T) { stop() } }() - connect := func(token string) *websocket.Conn { - u, _ := url.Parse(wsURL) - u.RawQuery = url.Values{"device_id": {bound.DeviceID}, "version": {proto.Version}}.Encode() - conn, resp, err := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + token}}) - if resp != nil { - resp.Body.Close() - } - if err != nil { - t.Fatal("daemon connection rejected") - } - t.Cleanup(func() { conn.Close() }) - return conn - } await := func(status string) { t.Helper() deadline := time.Now().Add(5 * time.Second) @@ -129,7 +108,8 @@ func TestEnrolledDaemonConnectionRevocationAndRestart(t *testing.T) { } t.Fatalf("environment did not become %s", status) } - first := connect(key.Token) + first := startLinkServe(t, link, []byte(key.Token), bound.Ref()) + within(t, first.connected) await("connected") assertConnection(environment.ID, key.Token, "connected", 200) rotated, err := sessionService(t, s).RotateExecutorCredential(t.Context(), principal, key.KeyID) @@ -138,21 +118,24 @@ func TestEnrolledDaemonConnectionRevocationAndRestart(t *testing.T) { } assertConnection(environment.ID, key.Token, "", 401) assertConnection(environment.ID, rotated.Token, "disconnected", 200) - // No heartbeat is sent: the Worker's authority check must fence the old socket. + // The Worker's pass revokes the previous generation, which closes the old + // secret's Serve; its redial is refused. await("disconnected") - _ = first.SetReadDeadline(time.Now().Add(time.Second)) - if _, _, err = first.ReadMessage(); err == nil { - t.Fatal("rotated socket retained authority") + if got := first.refused(t); got != sandboxlink.AuthenticationFailed { + t.Fatal("the rotated-out secret's Serve ended with", got) } - second := connect(rotated.Token) + next := bound + next.Generation++ + second := startLinkServe(t, link, []byte(rotated.Token), next.Ref()) + within(t, second.connected) await("connected") assertConnection(environment.ID, rotated.Token, "connected", 200) awaitRelease := pgtest.ObserveExecutionLeaseRelease(t, s.pool) stop() stop = nil awaitRelease() - // A new Core owner clears prior transport evidence, then observes the same - // live, authorized daemon. No compute allocation or native execution is made. + // A new Core owner clears prior connection evidence, then observes the + // same Serving resource. No compute allocation or native execution is made. stop = start() await("connected") if err = sessionService(t, s).RevokeExecutorCredential(t.Context(), principal, key.KeyID); err != nil { @@ -160,13 +143,12 @@ func TestEnrolledDaemonConnectionRevocationAndRestart(t *testing.T) { } assertConnection(environment.ID, rotated.Token, "", 401) await("disconnected") - _ = second.SetReadDeadline(time.Now().Add(time.Second)) - if _, _, err = second.ReadMessage(); err == nil { - t.Fatal("revoked socket retained authority") + if got := second.refused(t); got != sandboxlink.AuthenticationFailed { + t.Fatal("the revoked key's Serve ended with", got) } var allocations int if err = s.pool.QueryRow(t.Context(), "SELECT count(*) FROM runtime_allocations WHERE environment_id=$1", environment.ID).Scan(&allocations); err != nil || allocations != 0 { - t.Fatal("user Runtime acquired managed allocation", allocations, err) + t.Fatal("self_hosted Environment acquired a managed allocation", allocations, err) } current, err := sessionAdapter(s).GetSession(t.Context(), principal.TenantID, session.ID) if err != nil || current.LastTurn != nil { diff --git a/services/core/tests/integration/runtime_enrollment_test.go b/services/core/tests/integration/runtime_enrollment_test.go index 7a22c4c04..68635fad3 100644 --- a/services/core/tests/integration/runtime_enrollment_test.go +++ b/services/core/tests/integration/runtime_enrollment_test.go @@ -5,9 +5,8 @@ import ( "sync" "testing" + "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxbootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) @@ -47,56 +46,42 @@ func TestRuntimeEnrollmentAuthorityAndRotation(t *testing.T) { } } bound, err := sessionService(t, s).EnrollRuntime(ctx, environment.ID, executorDigest(key.Token)) - if err != nil || bound.SessionID != session.ID || bound.EnvironmentID != environment.ID || bound.WorkspaceDirectory != "/workspace" { + if err != nil || bound.TenantID != p.TenantID || bound.EnvironmentID != environment.ID || bound.Kind != "enrollment" || bound.ID == "" || bound.Generation != 1 { t.Fatalf("enrollment: %+v %v", bound, err) } if again, err := sessionService(t, s).EnrollRuntime(ctx, environment.ID, executorDigest(key.Token)); err != nil || again != bound { - t.Fatalf("retry changed binding: %+v %v", again, err) + t.Fatalf("retry changed the resource: %+v %v", again, err) } - if devices, err := sessionAdapter(s).ListExecutionDevices(ctx, p.TenantID); err != nil || len(devices) != 0 { - t.Fatalf("enrolled Runtime entered general selection: %v", err) + if _, err := sessionAdapter(s).GetSessionDevice(ctx, p.TenantID, session.ID); !errors.Is(err, sessions.ErrNotFound) { + t.Fatalf("enrollment bound the Session: %v", err) } - auth := runtimegateway.NewAuthenticator(sessionAdapter(s)) - if _, err := auth.AuthenticateBearer(ctx, bound.DeviceID, key.Token); err != nil { - t.Fatal(err) + if live, err := sessionAdapter(s).GetEnvironmentResource(ctx, p.TenantID, environment.ID); err != nil || live.Resource != bound || live.CredentialHash != executorDigest(key.Token) { + t.Fatalf("live resource: %+v %v", live, err) } otherKey, err := sessionService(t, s).IssueExecutorCredential(ctx, p, uuid.NewString(), environment.ID) if err != nil { t.Fatal(err) } if _, err := sessionService(t, s).EnrollRuntime(ctx, environment.ID, executorDigest(otherKey.Token)); !errors.Is(err, sessions.ErrDeviceBindingConflict) { - t.Fatalf("another key replaced binding: %v", err) + t.Fatalf("another key replaced the enrollment: %v", err) } rotated, err := sessionService(t, s).RotateExecutorCredential(ctx, p, key.KeyID) if err != nil { t.Fatal(err) } - if _, err := auth.AuthenticateBearer(ctx, bound.DeviceID, key.Token); !errors.Is(err, runtimegateway.ErrAuthBadCredential) { - t.Fatalf("old key after rotation: %v", err) + if _, err := sessionService(t, s).EnrollRuntime(ctx, environment.ID, executorDigest(key.Token)); !errors.Is(err, sessions.ErrNotFound) { + t.Fatalf("the rotated-out token enrolled: %v", err) } - if _, err := auth.AuthenticateBearer(ctx, bound.DeviceID, rotated.Token); err != nil { - t.Fatal(err) - } - for _, check := range []struct { - token string - denied bool - }{{key.Token, true}, {rotated.Token, false}} { - status, err := sessionService(t, s).TouchAgentDaemonHeartbeat(ctx, runtimedevice.Heartbeat{RuntimeID: bound.DeviceID, CredentialHash: executorDigest(check.token)}) - if err != nil || status.Deleted != check.denied { - t.Fatalf("rotation heartbeat: %+v %v", status, err) - } - } - if again, err := sessionService(t, s).EnrollRuntime(ctx, environment.ID, executorDigest(rotated.Token)); err != nil || again != bound { - t.Fatalf("rotation replaced identity: %+v %v", again, err) + next := bound + next.Generation++ + if again, err := sessionService(t, s).EnrollRuntime(ctx, environment.ID, executorDigest(rotated.Token)); err != nil || again != next { + t.Fatalf("re-enrollment after rotation: %+v %v", again, err) } if err := sessionService(t, s).RevokeExecutorCredential(ctx, p, key.KeyID); err != nil { t.Fatal(err) } - if _, ok, err := sessionAdapter(s).GetDeviceCredential(ctx, bound.DeviceID); err != nil || ok { - t.Fatalf("revoked key authenticates: %v", err) - } - if _, err := sessionAdapter(s).GetSessionDevice(ctx, p.TenantID, session.ID); !errors.Is(err, sessions.ErrNotFound) { - t.Fatalf("revoked binding dispatchable: %v", err) + if _, err := sessionAdapter(s).GetEnvironmentResource(ctx, p.TenantID, environment.ID); !errors.Is(err, sessions.ErrNotFound) { + t.Fatalf("a revoked key's enrollment is live: %v", err) } } @@ -105,7 +90,7 @@ func TestRuntimeEnrollmentConcurrentAndDeletion(t *testing.T) { p := FixtureExecutorPrincipal(t, s, uuid.NewString()) session, environment, key := runtimeEnrollmentFixture(t, s, p) var wg sync.WaitGroup - results := make(chan sessions.RuntimeEnrollment, 6) + results := make(chan sandboxbootstrap.Resource, 6) failures := make(chan error, 6) for range 6 { wg.Add(1) @@ -124,13 +109,13 @@ func TestRuntimeEnrollmentConcurrentAndDeletion(t *testing.T) { t.Fatal(err) } } - var bound sessions.RuntimeEnrollment + var bound sandboxbootstrap.Resource for got := range results { - if bound.DeviceID == "" { + if bound.ID == "" { bound = got } if got != bound { - t.Fatal("concurrent enrollment created multiple identities") + t.Fatal("concurrent enrollment created multiple resources") } } var allocations int @@ -143,11 +128,7 @@ func TestRuntimeEnrollmentConcurrentAndDeletion(t *testing.T) { if _, err := sessionService(t, s).EnrollRuntime(t.Context(), environment.ID, executorDigest(key.Token)); !errors.Is(err, sessions.ErrNotFound) { t.Fatalf("deleted enrollment: %v", err) } - if _, ok, err := sessionAdapter(s).GetDeviceCredential(t.Context(), bound.DeviceID); err != nil || ok { - t.Fatalf("deleted Session authenticates: %v", err) - } - status, err := sessionService(t, s).TouchAgentDaemonHeartbeat(t.Context(), runtimedevice.Heartbeat{RuntimeID: bound.DeviceID, CredentialHash: executorDigest(key.Token)}) - if err != nil || !status.Deleted { - t.Fatalf("deleted heartbeat: %+v %v", status, err) + if _, err := sessionAdapter(s).GetEnvironmentResource(t.Context(), p.TenantID, environment.ID); !errors.Is(err, sessions.ErrNotFound) { + t.Fatalf("a deleted Session's enrollment is live: %v", err) } } diff --git a/services/core/tests/integration/runtime_file_admission_test.go b/services/core/tests/integration/runtime_file_admission_test.go index 676c9e9d9..564e4c8de 100644 --- a/services/core/tests/integration/runtime_file_admission_test.go +++ b/services/core/tests/integration/runtime_file_admission_test.go @@ -13,17 +13,26 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func runtimeFileWriteKey(owner deployment.Allocation) sessions.FileWriteIdentity { - return sessions.FileWriteIdentity{ID: uuid.NewString(), DeviceID: owner.DeviceID, RequestSHA256: strings.Repeat("a", 64)} +// runtimeFileWriteHost places the allocation's Session on an agent host, the +// Runtime its file writes name. +func runtimeFileWriteHost(t *testing.T, s *Store, owner deployment.Allocation) string { + t.Helper() + host := registerAgentHost(t, s, owner.TenantID) + assignSession(t, s, owner.SessionID, host.ID) + return host.ID +} + +func runtimeFileWriteKey(host string) sessions.FileWriteIdentity { + return sessions.FileWriteIdentity{ID: uuid.NewString(), DeviceID: host, RequestSHA256: strings.Repeat("a", 64)} } func TestRuntimeFileWriteRequiresRunningComputeBeforeNewIntent(t *testing.T) { for _, phase := range []string{"disabled", "running", "quiescing", "suspending", "suspended", "restoring", "waking"} { t.Run(phase, func(t *testing.T) { - _, w, pool, owner := runtimeSuspensionFixture(t) + s, w, pool, owner := runtimeSuspensionFixture(t) writes := sessionExecution(t, w.lease) runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_phase=$2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID, phase) - key := runtimeFileWriteKey(owner) + key := runtimeFileWriteKey(runtimeFileWriteHost(t, s, owner)) write, err := writes.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key) blocked := phase != "disabled" && phase != "running" if blocked { @@ -47,9 +56,10 @@ func TestRuntimeFileWriteRequiresRunningComputeBeforeNewIntent(t *testing.T) { } func TestRuntimeFileWritePhaseFencePreservesExistingReceipts(t *testing.T) { - _, w, pool, owner := runtimeSuspensionFixture(t) + s, w, pool, owner := runtimeSuspensionFixture(t) writes := sessionExecution(t, w.lease) - key := runtimeFileWriteKey(owner) + host := runtimeFileWriteHost(t, s, owner) + key := runtimeFileWriteKey(host) first, err := writes.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key) if err != nil { t.Fatal(err) @@ -67,7 +77,7 @@ func TestRuntimeFileWritePhaseFencePreservesExistingReceipts(t *testing.T) { if _, err := writes.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, changed); !errors.Is(err, sessions.ErrIdempotencyConflict) { t.Fatal("phase fence masked changed retry identity", err) } - if _, err := writes.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, runtimeFileWriteKey(owner)); !errors.Is(err, sessions.ErrTurnConflict) { + if _, err := writes.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, runtimeFileWriteKey(host)); !errors.Is(err, sessions.ErrTurnConflict) { t.Fatal("receipt authorized a successor while waking", err) } if state == "pending" { @@ -84,7 +94,7 @@ func TestRuntimeFileWriteAndQuiesceSerializeBothOrders(t *testing.T) { s, w, pool, owner := runtimeSuspensionFixture(t) runtimeSuspensionCompleted(t, pool, owner) writes := sessionExecution(t, w.lease) - key := runtimeFileWriteKey(owner) + key := runtimeFileWriteKey(runtimeFileWriteHost(t, s, owner)) until := time.Now().Add(time.Hour) ctx, tx, blocker := runtimeSuspensionLockedSession(t, pool, owner.SessionID) done := make(chan error, 1) diff --git a/services/core/tests/integration/runtime_initialization_peer_test.go b/services/core/tests/integration/runtime_initialization_peer_test.go index fd5facab1..4579d0d6b 100644 --- a/services/core/tests/integration/runtime_initialization_peer_test.go +++ b/services/core/tests/integration/runtime_initialization_peer_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "net/http" + "sync" "sync/atomic" "testing" "time" @@ -18,14 +19,18 @@ import ( "github.com/gorilla/websocket" ) -// initializationPeer exercises the real authenticated gateway and chunk receipts. -// The provider fixture bootstraps its socket, and Serves the bootstrap's Link -// resource at link; all initialization runs on that peer. +// initializationPeer exercises the real authenticated gateway and chunk +// receipts. It connects as the deployment's agent host, on which all +// initialization runs, and has a fake sandbox Serve each bootstrap's Link +// resource at link. type initializationPeer struct { t *testing.T endpoint string registry *runtimegateway.Registry link *sandboxlinktest.Server + host agentHost + tenant string // scopes the agent host setRuntimeGateway registers; see registerAgentHost + serving map[string]*linkServe // by Link resource ID apply func(proto.RuntimePreparePayload, []byte) proto.RuntimePrepareResultPayload writes atomic.Int32 commandCalls atomic.Int32 @@ -36,22 +41,35 @@ type initializationPeer struct { closeOnBind bool // close the socket at a bind instead of replying } -func (p *initializationPeer) setRuntimeGateway(t *testing.T, endpoint string, registry *runtimegateway.Registry, link *sandboxlinktest.Server) { - p.t, p.endpoint, p.registry, p.link = t, endpoint, registry, link +// setRuntimeGateway points the peer at the gateway and the relay, and +// registers its agent host on s unless the peer already has one. +func (p *initializationPeer) setRuntimeGateway(t *testing.T, s *Store, endpoint string, registry *runtimegateway.Registry, link *sandboxlinktest.Server) { + p.t, p.endpoint, p.registry, p.link, p.serving = t, endpoint, registry, link, nil + if p.host.ID == "" { + p.host = registerAgentHost(t, s, p.tenant) + } } + +// connect has a fake sandbox Serve the bootstrap's Link resource, once per +// resource, and connects the agent host again. func (p *initializationPeer) connect(b sandbox.Bootstrap) error { p.bootstrap = b if p.deferred { return nil } - if p.link != nil && b.SandboxIO.Credential != "" { + if resource := b.SandboxIO.Resource; p.link != nil && b.SandboxIO.Credential != "" && p.serving[resource.ID] == nil { + served := startLinkServe(p.t, p.link, []byte(b.SandboxIO.Credential), resource.Ref()) select { - case <-startLinkServe(p.t, p.link, []byte(b.SandboxIO.Credential), b.SandboxIO.Resource.Ref()).connected: + case <-served.connected: case <-time.After(linkWait): return context.DeadlineExceeded } + if p.serving == nil { + p.serving = map[string]*linkServe{} + } + p.serving[resource.ID] = served } - c, _, err := websocket.DefaultDialer.Dial(p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, http.Header{"Authorization": {"Bearer " + b.Credential}}) + c, _, err := websocket.DefaultDialer.Dial(p.endpoint+"?device_id="+p.host.ID+"&version="+proto.Version, http.Header{"Authorization": {"Bearer " + p.host.Credential}}) if err != nil { return err } @@ -62,7 +80,7 @@ func (p *initializationPeer) connect(b sandbox.Bootstrap) error { } go func() { - transfer := initializationTransfer{peer: p} + transfer := newInitializationTransfer(p, c) for { var env proto.Envelope if c.ReadJSON(&env) != nil { @@ -77,7 +95,7 @@ func (p *initializationPeer) connect(b sandbox.Bootstrap) error { _ = c.Close() return } - if c.WriteJSON(reply) != nil { + if transfer.write(reply) != nil { return } continue @@ -85,19 +103,15 @@ func (p *initializationPeer) connect(b sandbox.Bootstrap) error { if env.Type != proto.TypeRuntimePrepare { continue } - reply, err := transfer.receive(env) - if err != nil { + if err := transfer.receive(env); err != nil { p.t.Error(err) return } - if c.WriteJSON(reply) != nil { - return - } } }() end := time.Now().Add(time.Second) for time.Now().Before(end) { - if _, err := p.registry.LookupDevice(b.DeviceID); err == nil { + if _, err := p.registry.LookupDevice(p.host.ID); err == nil { return nil } time.Sleep(time.Millisecond) @@ -112,41 +126,77 @@ func completedInitialization(proto.RuntimePreparePayload, []byte) proto.RuntimeP return proto.RuntimePrepareResultPayload{Outcome: "completed"} } -// Each authenticated socket owns its own bounded preparation transfer. +// initializationTransfer answers preparation frames on one agent host socket, +// which initializes many Environments. Every step of one preparation shares +// its envelope ID. A commit applies in the background, as the Runtime does, so +// a blocked preparation holds up no other Session's. type initializationTransfer struct { peer *initializationPeer + conn *websocket.Conn + writeMu sync.Mutex + pending map[string]*preparationTransfer // by envelope ID +} + +type preparationTransfer struct { request proto.RuntimePreparePayload data []byte } -func (x *initializationTransfer) receive(env proto.Envelope) (proto.Envelope, error) { +func newInitializationTransfer(peer *initializationPeer, conn *websocket.Conn) *initializationTransfer { + return &initializationTransfer{peer: peer, conn: conn, pending: map[string]*preparationTransfer{}} +} + +func (x *initializationTransfer) write(env proto.Envelope) error { + x.writeMu.Lock() + defer x.writeMu.Unlock() + return x.conn.WriteJSON(env) +} + +func (x *initializationTransfer) receive(env proto.Envelope) error { var frame proto.RuntimePreparePayload if env.DecodePayload(&frame) != nil || !proto.ValidRuntimePrepareRequest(frame) { - return proto.Envelope{}, errors.New("invalid Runtime frame") + return errors.New("invalid Runtime frame") + } + if frame.Step == "begin" { + x.pending[env.ID] = &preparationTransfer{request: frame} + } + current := x.pending[env.ID] + if current == nil { + return errors.New("initialization step without begin") } result := proto.RuntimePrepareResultPayload{} switch frame.Step { case "begin": - x.request, x.data = frame, nil result.Outcome = "ready" case "chunk": - if frame.Offset != len(x.data) { - return proto.Envelope{}, errors.New("unordered initialization bytes") + if frame.Offset != len(current.data) { + return errors.New("unordered initialization bytes") } - x.data = append(x.data, frame.Data...) - result.Outcome, result.Offset = "received", len(x.data) + current.data = append(current.data, frame.Data...) + result.Outcome, result.Offset = "received", len(current.data) case "commit": - if x.request.Action == "file" || x.request.Action == "skill" || x.request.Action == "plugin" { - sum := sha256.Sum256(x.data) - if x.request.SizeBytes != len(x.data) || x.request.SHA256 != hex.EncodeToString(sum[:]) { - return proto.Envelope{}, errors.New("initialization digest changed") + if current.request.Action == "file" || current.request.Action == "skill" || current.request.Action == "plugin" { + sum := sha256.Sum256(current.data) + if current.request.SizeBytes != len(current.data) || current.request.SHA256 != hex.EncodeToString(sum[:]) { + return errors.New("initialization digest changed") } } - result = x.peer.apply(x.request, x.data) - x.peer.writes.Add(1) - if result.Outcome == "completed" { - result.SizeBytes = len(x.data) - } + delete(x.pending, env.ID) + go func() { + result := x.peer.apply(current.request, current.data) + x.peer.writes.Add(1) + if result.Outcome == "completed" { + result.SizeBytes = len(current.data) + } + if reply, err := env.Reply(proto.TypeRuntimePrepareResult, result); err == nil { + _ = x.write(reply) + } + }() + return nil + } + reply, err := env.Reply(proto.TypeRuntimePrepareResult, result) + if err != nil { + return err } - return env.Reply(proto.TypeRuntimePrepareResult, result) + return x.write(reply) } diff --git a/services/core/tests/integration/runtime_initialization_test.go b/services/core/tests/integration/runtime_initialization_test.go index 51fc6b646..18c7cfa4a 100644 --- a/services/core/tests/integration/runtime_initialization_test.go +++ b/services/core/tests/integration/runtime_initialization_test.go @@ -5,10 +5,10 @@ import ( "bytes" "context" "encoding/json" - "errors" "reflect" "strings" "sync" + "sync/atomic" "testing" "time" @@ -41,7 +41,7 @@ func (p *initializingProvider) RunCommand(ctx context.Context, r sandbox.Referen } func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { - for _, mode := range []string{"complete", "restart", "uncertain", "setup-complete", "setup-restart", "setup-uncertain"} { + for _, mode := range []string{"complete", "restart", "uncertain", "unavailable", "setup-complete", "setup-restart", "setup-uncertain"} { t.Run(mode, func(t *testing.T) { setupOnly := strings.HasPrefix(mode, "setup-") mode = strings.TrimPrefix(mode, "setup-") @@ -74,16 +74,16 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { } } p := &initializingProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, initializationPeer: initializationPeer{deferred: true}} + var rejected atomic.Bool p.apply = func(_ proto.RuntimePreparePayload, _ []byte) proto.RuntimePrepareResultPayload { - if _, err := sessionAdapter(s).GetSessionDevice(t.Context(), tenant, session.ID); !errors.Is(err, sessions.ErrNotFound) { - t.Error("premature file access", err) - } - if _, err := sessionAdapter(s).GetSessionExecutionBinding(t.Context(), tenant, session.ID); !errors.Is(err, sessions.ErrNotFound) { - t.Error("premature execution", err) - } if mode == "uncertain" { return proto.RuntimePrepareResultPayload{Outcome: "unknown", ErrorCode: "runtime_preparation_unconfirmed"} } + // A first step the Runtime rejects as unavailable took no + // effect, so a later pass starts the preparation again. + if mode == "unavailable" && rejected.CompareAndSwap(false, true) { + return proto.RuntimePrepareResultPayload{Outcome: "rejected", ErrorCode: "resource_unavailable"} + } return completedInitialization(proto.RuntimePreparePayload{}, nil) } w, stop := managedWorkerMode(t, s, key, p, true) @@ -114,7 +114,11 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { want = "failed" } awaitInitialization(t, s, tenant, env.ID, want) - if mode == "complete" { + if mode == "unavailable" { + if int(p.writes.Load()) != expectedSteps+1 { + t.Fatal("rejected first step was not retried once", p.writes.Load()) + } + } else if mode == "complete" { if int(p.writes.Load()) != expectedSteps { t.Fatal("missing operations", p.writes.Load()) } diff --git a/services/core/tests/integration/runtime_lifecycle_test.go b/services/core/tests/integration/runtime_lifecycle_test.go index a9dab0db4..3dc82740b 100644 --- a/services/core/tests/integration/runtime_lifecycle_test.go +++ b/services/core/tests/integration/runtime_lifecycle_test.go @@ -88,13 +88,13 @@ func managedWorkerMode(t *testing.T, s *Store, key string, p sandbox.SandboxProv t.Helper() dispatcher := &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: webRuntimes(t, s, key, p, nil)} if peer, ok := p.(interface { - setRuntimeGateway(*testing.T, string, *runtimegateway.Registry, *sandboxlinktest.Server) + setRuntimeGateway(*testing.T, *Store, string, *runtimegateway.Registry, *sandboxlinktest.Server) }); ok { handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: dispatcher.Registry}) server := httptest.NewServer(http.HandlerFunc(handler.WS)) t.Cleanup(server.Close) link := sandboxlinktest.StartRelay(t, runtimegateway.NewLinkAuthority(sessionAdapter(s))) - peer.setRuntimeGateway(t, "ws"+strings.TrimPrefix(server.URL, "http"), dispatcher.Registry, link) + peer.setRuntimeGateway(t, s, "ws"+strings.TrimPrefix(server.URL, "http"), dispatcher.Registry, link) dispatcher.Links = link.Relay } w, err := startNextWorker(t.Context(), s, dispatcher) diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index 4dd787771..818a3f69a 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -5,8 +5,6 @@ import ( "context" "encoding/json" "errors" - "net/http" - "net/http/httptest" "strings" "sync" "sync/atomic" @@ -14,23 +12,18 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/internal/sandboxlink/sandboxlinktest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" - "github.com/gorilla/websocket" "github.com/jackc/pgx/v5/pgxpool" ) -// nodeIsolationProvider Serves each allocation it creates at link. type nodeIsolationProvider struct { *fakeCheckpointProvider - link *sandboxlinktest.Server blockMu sync.Mutex blocked map[string]bool mode string @@ -53,20 +46,6 @@ func (p *nodeIsolationProvider) block(ctx context.Context, r sandbox.Reference, p.returned.Add(1) return ctx.Err() } -func (p *nodeIsolationProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { - info, err := p.fakeCheckpointProvider.Create(ctx, b) - if err == nil { - err = p.connect(ctx, b) - } - if err == nil { - select { - case <-startLinkServe(p.preparation.t, p.link, []byte(b.SandboxIO.Credential), b.SandboxIO.Resource.Ref()).connected: - case <-ctx.Done(): - err = ctx.Err() - } - } - return info, err -} func (p *nodeIsolationProvider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { if err := p.block(ctx, r, "observe"); err != nil { return sandbox.Info{}, err @@ -106,9 +85,8 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { t.Fatal(err) } s := NewWithCredentialCipher(pool, cipher) - registry := runtimegateway.NewRegistry() - cp := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} - p := &nodeIsolationProvider{fakeCheckpointProvider: cp, link: sandboxlinktest.StartRelay(t, runtimegateway.NewLinkAuthority(sessionAdapter(s))), blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} + cp := newFakeCheckpointProvider(t, s) + p := &nodeIsolationProvider{fakeCheckpointProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} preparationContext, cancelPreparation := context.WithCancel(t.Context()) t.Cleanup(cancelPreparation) cp.preparation = &initializationPeer{t: t, apply: func(request proto.RuntimePreparePayload, data []byte) proto.RuntimePrepareResultPayload { @@ -122,21 +100,10 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { p.writes.Add(1) return completedInitialization(request, data) }} - handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: registry}) - server := httptest.NewServer(http.HandlerFunc(handler.WS)) - cp.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") - t.Cleanup(func() { - cp.mu.Lock() - for _, peer := range cp.peers { - peer.Close() - } - cp.mu.Unlock() - server.Close() - }) f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, nodes: deploymentService(t, s), pool: pool, provider: p, key: webDeployment(t, s, "microsandbox"), nodeA: uuid.NewString(), nodeB: uuid.NewString()} // Keep restored compute awake throughout the isolation assertions. // The suspension setup explicitly dates its activity two minutes in the past. - f.worker, err = startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: registry, Links: p.link.Relay, ManagedRuntimes: webRuntimes(t, s, f.key, p, &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour})}) + f.worker, err = startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: cp.registry, Links: cp.link.Relay, ManagedRuntimes: webRuntimes(t, s, f.key, p, &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour})}) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_node_lifecycle_test.go b/services/core/tests/integration/runtime_node_lifecycle_test.go index 545e87866..14eb9c336 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_test.go @@ -21,6 +21,7 @@ func TestManagedNodesIsolateBlockedProviderAndInitialization(t *testing.T) { wakeTenant, _, wakeEnv := f.session(f.nodeB, false) wakeOwner := f.provision(wakeTenant, wakeEnv) f.phase(wakeTenant, wakeEnv.ID, "running") + assignSession(t, f.store, wakeOwner.SessionID, f.provider.host.ID) if _, err := f.pool.Exec(t.Context(), "INSERT INTO turns(id,session_id,status,completed_at) VALUES($1,$2,'completed',clock_timestamp()-interval '2 minutes')", uuid.NewString(), wakeOwner.SessionID); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_pending_test.go b/services/core/tests/integration/runtime_pending_test.go index 06af06ef7..af238926d 100644 --- a/services/core/tests/integration/runtime_pending_test.go +++ b/services/core/tests/integration/runtime_pending_test.go @@ -26,7 +26,7 @@ func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) t.Fatal(err) } p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - start := func() *execution.Worker { return startWebWorker(t, s, runtimegateway.NewRegistry(), key, p, nil) } + start := func() *execution.Worker { return startWebWorker(t, s, runtimegateway.NewRegistry(), nil, key, p, nil) } stop := func(w *execution.Worker) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index aedeca68c..f163bc884 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -73,7 +73,7 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } - worker := startWebWorker(t, f.store, f.provider.registry, f.key, provider, &f.policy) + worker := startWebWorker(t, f.store, f.provider.registry, f.provider.link.Relay, f.key, provider, &f.policy) onlineManagerNode(t, f.store, f.node) ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) diff --git a/services/core/tests/integration/runtime_worker_recovery_test.go b/services/core/tests/integration/runtime_worker_recovery_test.go index e7dd3e87b..cf1496bb4 100644 --- a/services/core/tests/integration/runtime_worker_recovery_test.go +++ b/services/core/tests/integration/runtime_worker_recovery_test.go @@ -13,21 +13,21 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) -func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { +// setWorkerComputePhase puts the harness's allocation in phase, with +// compute retained for an hour. +func setWorkerComputePhase(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { t.Helper() - _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation) - VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment))`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) - if err != nil { + if _, err := pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase = $2, compute_retained_until = clock_timestamp() + interval '1 hour' WHERE id = $1`, h.resource.ID, phase); err != nil { t.Fatal(err) } } func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { t.Helper() - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`), true) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`)) enableWorkerEnvironment(t, h) _, pool := testStore(t) - insertWorkerRuntimeAllocation(t, pool, h, "waking") + setWorkerComputePhase(t, pool, h, "waking") return h, pool } diff --git a/services/core/tests/integration/session_devices_fixture_test.go b/services/core/tests/integration/session_devices_fixture_test.go index 9dc9380e5..54acf9726 100644 --- a/services/core/tests/integration/session_devices_fixture_test.go +++ b/services/core/tests/integration/session_devices_fixture_test.go @@ -5,13 +5,51 @@ import ( "errors" "testing" + "github.com/google/uuid" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/sessionpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// bindSessionDevice binds the tenant's device to the Session through the +// agentHost is a Runtime registered as Core's startup registers the +// deployment's agent host, the Runtime Sessions are placed on. +type agentHost struct{ ID, Credential string } + +// registerAgentHost registers an agent host. Tests on the shared database pass +// their tenant: a deployment-wide host would let the test's Worker list and +// place every other test's Sessions, so the host serves only that tenant, as +// the test's own deployment. A test on an isolated database passes "". +func registerAgentHost(t testing.TB, s *Store, tenant string) agentHost { + t.Helper() + host := agentHost{ID: uuid.NewString(), Credential: uuid.NewString()} + if err := sessionAdapter(s).RegisterAgentHost(t.Context(), host.ID, runtimedevice.HashCredential(host.Credential)); err != nil { + t.Fatal(err) + } + if tenant != "" { + if _, err := s.pool.Exec(t.Context(), `UPDATE devices SET tenant_id = $2 WHERE id = $1`, host.ID, tenant); err != nil { + t.Fatal(err) + } + } + return host +} + +// assignSession places the Session on the agent host as a Worker's placement +// does, without the execution lease a running Worker holds. A Session already +// placed elsewhere fails the test. +func assignSession(t testing.TB, s *Store, session, host string) { + t.Helper() + var runtime string + err := s.pool.QueryRow(t.Context(), `INSERT INTO session_runtime_assignments (session_id, runtime_id) VALUES ($1, $2) + ON CONFLICT (session_id) DO UPDATE SET runtime_id = session_runtime_assignments.runtime_id RETURNING runtime_id::text`, session, host).Scan(&runtime) + if err != nil || runtime != host { + t.Fatalf("Session placed on %q, want %s: %v", runtime, host, err) + } +} + +// bindSessionDevice binds the agent host to the Session through the // Session execution operations on an execution lease of its own, and returns // once the server released that lease, so a Worker can take it next. func bindSessionDevice(t *testing.T, s *Store, tenant, session, device string) error { diff --git a/services/core/tests/integration/session_reads_fixture_test.go b/services/core/tests/integration/session_reads_fixture_test.go index 3aa8e060e..f2cdc8308 100644 --- a/services/core/tests/integration/session_reads_fixture_test.go +++ b/services/core/tests/integration/session_reads_fixture_test.go @@ -12,9 +12,9 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// FixtureEnvironmentDevice provisions the dedicated Runtime device of the -// tenant's hosted Environment on pool through the procedure the managed -// Runtime allocation runs, without the allocation. +// FixtureEnvironmentDevice creates the per-allocation device of the tenant's +// hosted Environment on pool through the procedure the managed Runtime +// allocation runs, without the allocation. func FixtureEnvironmentDevice(t testing.TB, ctx context.Context, pool *pgxpool.Pool, tenant, environment, name, credentialHash string) (sessions.ExecutionDevice, error) { s := New(t, pool) current, err := sessionAdapter(s).GetEnvironment(ctx, tenant, environment) @@ -29,12 +29,12 @@ func FixtureEnvironmentDevice(t testing.TB, ctx context.Context, pool *pgxpool.P if err != nil { return sessions.ExecutionDevice{}, err } - device := sessions.ExecutionDevice{ID: uuid.NewString(), Name: registration.Name, EnvironmentID: current.ID} + device := sessions.ExecutionDevice{ID: uuid.NewString(), Name: registration.Name} err = sessionpg.WithSession(ctx, s.pooled, lookup.TenantID, lookup.ID, func(ctx context.Context, q *sqlc.Queries, locked sessions.LockedSession) error { if err := locked.Public(); err != nil { return err } - return sessions.CreateEnvironmentDevice(ctx, sessionpg.BindSession(q, lookup.TenantID, lookup.ID), device, registration.CredentialHash) + return sessions.CreateEnvironmentDevice(ctx, sessionpg.BindSession(q, lookup.TenantID, lookup.ID), current.ID, device, registration.CredentialHash) }) if err != nil { return sessions.ExecutionDevice{}, err diff --git a/services/core/tests/integration/subagent_identities_test.go b/services/core/tests/integration/subagent_identities_test.go index 4b2d47ce4..8537baacd 100644 --- a/services/core/tests/integration/subagent_identities_test.go +++ b/services/core/tests/integration/subagent_identities_test.go @@ -9,7 +9,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) @@ -26,13 +25,11 @@ func TestSubagentIdentityIsAtomicScopedAndImmutable(t *testing.T) { journal := sessionExecution(t, w.lease) ctx := t.Context() tenant, session := newSubagentSession(t, s) - host, err := sessionService(t, s).CreateDevice(ctx, tenant, "identity test", runtimedevice.HashCredential(uuid.NewString())) + host := registerAgentHost(t, s, tenant) + err := sessionExecution(t, w.lease).BindSessionDevice(ctx, tenant, session.ID, host.ID) if err != nil { t.Fatal(err) } - if err = sessionExecution(t, w.lease).BindSessionDevice(ctx, tenant, session.ID, host.ID); err != nil { - t.Fatal(err) - } input := submitMessage(t, s, tenant, session.ID, "first") transition(t, w, tenant, session.ID, input.TurnID, sessions.TurnQueued, sessions.TurnInProgress) a, b := subagentIdentityEvent("child-a", "root", 102), subagentIdentityEvent("child-b", "root", 101) diff --git a/services/core/tests/integration/subagent_native_outputs_test.go b/services/core/tests/integration/subagent_native_outputs_test.go index ad1238c1b..b6116a893 100644 --- a/services/core/tests/integration/subagent_native_outputs_test.go +++ b/services/core/tests/integration/subagent_native_outputs_test.go @@ -7,22 +7,18 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" - "github.com/google/uuid" ) func TestSubagentNativeFunctionResultDoesNotConsumeOutputIndex(t *testing.T) { s, pool := testStore(t) owner := executionWriter(t, s) tenant, session := newSubagentSession(t, s) - host, err := sessionService(t, s).CreateDevice(t.Context(), tenant, "child outputs", runtimedevice.HashCredential(uuid.NewString())) + host := registerAgentHost(t, s, tenant) + err := sessionExecution(t, owner.lease).BindSessionDevice(t.Context(), tenant, session.ID, host.ID) if err != nil { t.Fatal(err) } - if err = sessionExecution(t, owner.lease).BindSessionDevice(t.Context(), tenant, session.ID, host.ID); err != nil { - t.Fatal(err) - } input := submitMessage(t, s, tenant, session.ID, "start") transition(t, owner, tenant, session.ID, input.TurnID, sessions.TurnQueued, sessions.TurnInProgress) call := json.RawMessage(`{"id":"native-file-change","stage":"after","observation":{"status":"completed","kind":"function","name":"apply_patch","arguments":{"count":9007199254740993,"scale":1e2},"content":[{"type":"input_text","text":"file written"}]}}`) @@ -90,13 +86,11 @@ func TestSubagentCancelledPartialMessageSurvivesHistoryReplay(t *testing.T) { s, _ := testStore(t) owner := executionWriter(t, s) tenant, session := newSubagentSession(t, s) - host, err := sessionService(t, s).CreateDevice(t.Context(), tenant, "cancelled child", runtimedevice.HashCredential(uuid.NewString())) + host := registerAgentHost(t, s, tenant) + err := sessionExecution(t, owner.lease).BindSessionDevice(t.Context(), tenant, session.ID, host.ID) if err != nil { t.Fatal(err) } - if err = sessionExecution(t, owner.lease).BindSessionDevice(t.Context(), tenant, session.ID, host.ID); err != nil { - t.Fatal(err) - } input := submitMessage(t, s, tenant, session.ID, "start") transition(t, owner, tenant, session.ID, input.TurnID, sessions.TurnQueued, sessions.TurnInProgress) message := subagentFact(proto.TypeSubagentItem, proto.SubagentItemPayload{ diff --git a/services/core/tests/integration/subagent_resources_test.go b/services/core/tests/integration/subagent_resources_test.go index 1cfbf33c8..ad2f964b7 100644 --- a/services/core/tests/integration/subagent_resources_test.go +++ b/services/core/tests/integration/subagent_resources_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) @@ -33,13 +32,11 @@ func TestSubagentResourcesNativeOwnershipLifecycleAndRecovery(t *testing.T) { journal := sessionExecution(t, owner.lease) ctx := t.Context() tenant, session := newSubagentSession(t, s) - host, err := sessionService(t, s).CreateDevice(ctx, tenant, "child resources", runtimedevice.HashCredential(uuid.NewString())) + host := registerAgentHost(t, s, tenant) + err := sessionExecution(t, owner.lease).BindSessionDevice(ctx, tenant, session.ID, host.ID) if err != nil { t.Fatal(err) } - if err = sessionExecution(t, owner.lease).BindSessionDevice(ctx, tenant, session.ID, host.ID); err != nil { - t.Fatal(err) - } root := submitMessage(t, s, tenant, session.ID, "first") transition(t, owner, tenant, session.ID, root.TurnID, sessions.TurnQueued, sessions.TurnInProgress) ordinal := int32(1) diff --git a/services/core/tests/integration/subagent_visibility_public_test.go b/services/core/tests/integration/subagent_visibility_public_test.go index 26cfe2d29..cf07a75ab 100644 --- a/services/core/tests/integration/subagent_visibility_public_test.go +++ b/services/core/tests/integration/subagent_visibility_public_test.go @@ -121,10 +121,7 @@ func TestSubagentVisibilityPublic(t *testing.T) { t.Fatal(page, err) } root := page.Turns[0].ID - host, err := sessionService(t, s).CreateDevice(ctx, tenant, "subagent visibility", runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } + host := registerAgentHost(t, s, tenant) if err = leased.Sessions.BindSessionDevice(ctx, tenant, session, host.ID); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/unified_model_configuration_http_test.go b/services/core/tests/integration/unified_model_configuration_http_test.go index 7836effd1..435c6fb7e 100644 --- a/services/core/tests/integration/unified_model_configuration_http_test.go +++ b/services/core/tests/integration/unified_model_configuration_http_test.go @@ -141,15 +141,10 @@ func TestUnifiedModelConfigurationHTTP(t *testing.T) { {"explicit empty inline parameters", `{"agent":{"x_agents_core":{"harness_config":{}}},"environment":{"type":"openai_hosted"}}`, "model-replacement", "deployment", "deployment", "deployment-canary"}, {"explicit empty Session parameters", `{"agent":{},"environment":{"type":"openai_hosted"},"x_agents_core":{"harness_config":{}}}`, "model-replacement", "deployment", "deployment", "deployment-canary"}, } { - // self_hosted resolves native defaults exactly as openai_hosted does, - // but its guest never receives the deployment key. + // self_hosted resolves native defaults exactly as openai_hosted does. for _, environment := range []string{`{"type":"openai_hosted"}`, `{"type":"self_hosted","workspace_directory":"/workspace"}`} { t.Run(tc.name+" "+environment, func(t *testing.T) { body := strings.Replace(tc.body, `{"type":"openai_hosted"}`, environment, 1) - if strings.Contains(environment, "self_hosted") && tc.providerSource == "deployment" { - call("POST", "/v1/agents/sessions", token, body, uuid.NewString(), 400) - return - } id := create(body, uuid.NewString()) assertSession(id, tc.model, `{}`, tc.modelSource, "session", tc.providerSource, tc.key) }) diff --git a/services/core/tests/integration/worker_fixture_test.go b/services/core/tests/integration/worker_fixture_test.go index 055feb420..757d274de 100644 --- a/services/core/tests/integration/worker_fixture_test.go +++ b/services/core/tests/integration/worker_fixture_test.go @@ -172,10 +172,11 @@ func unusedPreparation(t testing.TB) execution.RuntimeDeploymentPreparer { } } -// startWebWorker starts the Worker on webRuntimes. -func startWebWorker(t *testing.T, s *Store, registry *runtimegateway.Registry, installation string, p sandbox.SandboxProvider, suspension *execution.RuntimeSuspensionPolicy) *execution.Worker { +// startWebWorker starts the Worker on webRuntimes, with links as its Link +// relay or a new one when links is nil. +func startWebWorker(t *testing.T, s *Store, registry *runtimegateway.Registry, links *relay.Relay, installation string, p sandbox.SandboxProvider, suspension *execution.RuntimeSuspensionPolicy) *execution.Worker { t.Helper() - w, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: webRuntimes(t, s, installation, p, suspension)}) + w, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: registry, Links: links, ManagedRuntimes: webRuntimes(t, s, installation, p, suspension)}) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/worker_preparation_failure_test.go b/services/core/tests/integration/worker_preparation_failure_test.go index 027a51939..2bfd77610 100644 --- a/services/core/tests/integration/worker_preparation_failure_test.go +++ b/services/core/tests/integration/worker_preparation_failure_test.go @@ -11,7 +11,7 @@ import ( func TestWorkerSettlesConfirmedPreparationFailureAndAcceptsNewInput(t *testing.T) { for _, code := range []string{"preparation_failed", "invalid_configuration", "unsupported_configuration", "unsupported_preparation"} { t.Run(code, func(t *testing.T) { - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`), false) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`)) enableWorkerEnvironment(t, h) frames := workerFrames(t, h) pending, err := sessionService(t, h.s).ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "first", []sessions.Input{messageInput("first")}) @@ -94,7 +94,7 @@ func TestWorkerRetriesUncertainPreparationFailure(t *testing.T) { {"run_present", "rejected", proto.TypeExecutionPrepare, "unsupported_configuration", "unconfirmed-run"}, } { t.Run(response.name, func(t *testing.T) { - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`), false) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`)) enableWorkerEnvironment(t, h) frames := workerFrames(t, h) pending, err := sessionService(t, h.s).ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "retry", []sessions.Input{messageInput("retry")}) @@ -126,7 +126,7 @@ func TestWorkerRetriesUncertainPreparationFailure(t *testing.T) { } func TestWorkerPreparationRejectionPreservesCancellationAndNewerInput(t *testing.T) { - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`), false) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`)) enableWorkerEnvironment(t, h) frames := workerFrames(t, h) first, err := sessionService(t, h.s).ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "first", []sessions.Input{messageInput("first")}) diff --git a/services/core/tests/integration/worker_wakeup_test.go b/services/core/tests/integration/worker_wakeup_test.go index 5ab16df8d..6f03ce2b7 100644 --- a/services/core/tests/integration/worker_wakeup_test.go +++ b/services/core/tests/integration/worker_wakeup_test.go @@ -98,7 +98,7 @@ func TestWorkerSchedulerCommittedAdmissionWakesBeforeMaintenance(t *testing.T) { } func TestWorkerSchedulerHintBypassesEnvironmentScanThrottle(t *testing.T) { - h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`), false) + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`)) enableWorkerEnvironment(t, h) frames := workerFrames(t, h) worker, stop := startEnvironmentExpiryWorker(t, h.s, h.d)