diff --git a/src/core/batchEvaluationResults.test.ts b/src/core/batchEvaluationResults.test.ts index c39f34214..d3e77d229 100644 --- a/src/core/batchEvaluationResults.test.ts +++ b/src/core/batchEvaluationResults.test.ts @@ -1,47 +1,21 @@ import { test, expect } from "bun:test"; -import type { CloudWatchLogsClient, OutputLogEvent } from "@aws-sdk/client-cloudwatch-logs"; -import { ResultTruncationError } from "../errors"; import { createSilentLogger } from "../testing"; import { isTerminalStatus, parseEvaluationLogEvent, readEvaluationResults, } from "./batchEvaluationResults"; +import type { CloudWatchLogEvent } from "./observability/index"; -// fakeLogs returns a CloudWatchLogsClient that serves `events` as a single page, -// then signals exhaustion by echoing the same nextForwardToken on the next call — -// exactly how GetLogEvents ends pagination. Records the tokens it was called with. -function fakeLogs(events: OutputLogEvent[]): CloudWatchLogsClient { - let served = false; - return { - send: async () => { - if (!served) { - served = true; - return { events, nextForwardToken: "t-end" }; - } - return { events: [], nextForwardToken: "t-end" }; // token unchanged → done - }, - } as unknown as CloudWatchLogsClient; -} - -// fakePagedLogs serves each element of `pages` on successive calls, advancing the -// forward token per page and repeating the last token once to end. Captures every -// nextToken the caller sent, so a test can assert the loop paged correctly. -function fakePagedLogs(pages: OutputLogEvent[][]): { - client: CloudWatchLogsClient; - tokens: (string | undefined)[]; -} { - const tokens: (string | undefined)[] = []; - let call = 0; - const client = { - send: async (command: { input: { nextToken?: string } }) => { - tokens.push(command.input.nextToken); - const i = call++; - if (i < pages.length) return { events: pages[i], nextForwardToken: `t-${i}` }; - return { events: [], nextForwardToken: `t-${pages.length - 1}` }; // repeat last → done - }, - } as unknown as CloudWatchLogsClient; - return { client, tokens }; +async function* events( + messages: ({ message?: string } | string)[], +): AsyncGenerator { + for (const item of messages) { + yield { + timestamp: new Date(0), + message: typeof item === "string" ? item : (item.message ?? ""), + }; + } } // A realistic stream shaped after the real `gen_ai.evaluation.result` records @@ -49,7 +23,7 @@ function fakePagedLogs(pages: OutputLogEvent[][]): { // attributes["aws.bedrock_agentcore.evaluation_level"] (Title-case), session.id // sits under attributes, and the trace id is the top-level camelCase `traceId`. // One SESSION-level and one TRACE-level record, plus a non-JSON control line. -const EVENTS: OutputLogEvent[] = [ +const EVENTS = [ { message: JSON.stringify({ attributes: { @@ -88,7 +62,7 @@ test("isTerminalStatus recognizes the terminal arm only", () => { }); test("readEvaluationResults keeps level + scope so sessions and traces are distinguishable", async () => { - const results = await readEvaluationResults(fakeLogs(EVENTS), "lg", "ls", createSilentLogger()); + const results = await readEvaluationResults(events(EVENTS), createSilentLogger()); // The non-JSON control line is skipped; the two evaluation records parse. expect(results).toHaveLength(2); @@ -109,35 +83,6 @@ test("readEvaluationResults keeps level + scope so sessions and traces are disti expect(results.map((r) => r.level)).toEqual(["Session", "Trace"]); }); -test("readEvaluationResults follows pagination until the forward token stops advancing", async () => { - const page = (name: string): OutputLogEvent => ({ - message: JSON.stringify({ - attributes: { - "gen_ai.evaluation.name": name, - "aws.bedrock_agentcore.evaluation_level": "Trace", - "session.id": "s1", - }, - }), - }); - const { client, tokens } = fakePagedLogs([ - [page("Builtin.Correctness")], - [page("Builtin.Helpfulness")], - [page("Builtin.Faithfulness")], - ]); - - const results = await readEvaluationResults(client, "lg", "ls", createSilentLogger()); - - // All three pages' records are collected. - expect(results.map((r) => r.evaluatorId)).toEqual([ - "Builtin.Correctness", - "Builtin.Helpfulness", - "Builtin.Faithfulness", - ]); - // First call has no token; later calls carry the prior page's forward token; a - // final call detects the repeated token and stops. - expect(tokens).toEqual([undefined, "t-0", "t-1", "t-2"]); -}); - // Real-log-shape validation lives in the fixture-backed command-flow test // (batch-evaluation.fixture.test.tsx), where RECORD=1 captures a live GetLogEvents // response and matchGolden pins the parsed output. This file stays a pure unit @@ -145,45 +90,16 @@ test("readEvaluationResults follows pagination until the forward token stops adv test("readEvaluationResults skips lines without an evaluation name", async () => { const results = await readEvaluationResults( - fakeLogs([ + events([ { message: JSON.stringify({ attributes: { "some.other.metric": 1 } }) }, { message: "" }, { message: undefined }, ]), - "lg", - "ls", createSilentLogger(), ); expect(results).toEqual([]); }); -test("readEvaluationResults throws (not silently truncates) when it hits the page cap", async () => { - // Token advances on every call, so the loop never detects exhaustion and runs - // into MAX_RESULT_PAGES. It must throw so the caller surfaces truncation, rather - // than returning the accumulated partial list as if it were complete. - let call = 0; - const everAdvancing = { - send: async () => ({ - events: [ - { - message: JSON.stringify({ - attributes: { "gen_ai.evaluation.name": "Builtin.Correctness" }, - }), - }, - ], - nextForwardToken: `t-${call++}`, // always changes → never exhausts - }), - } as unknown as CloudWatchLogsClient; - - const err = await readEvaluationResults(everAdvancing, "lg", "ls", createSilentLogger()).then( - () => undefined, - (e) => e as ResultTruncationError, - ); - expect(err).toBeInstanceOf(ResultTruncationError); - expect(err?.message).toMatch(/incomplete/); - expect(err?.source).toBe("internal"); // our page cap, not a user or service fault -}); - test("parseEvaluationLogEvent warns on and skips an unparseable line", () => { const warnings: string[] = []; const logger = createSilentLogger(); diff --git a/src/core/batchEvaluationResults.tsx b/src/core/batchEvaluationResults.tsx index 2c6d6131c..3cbbe7086 100644 --- a/src/core/batchEvaluationResults.tsx +++ b/src/core/batchEvaluationResults.tsx @@ -1,15 +1,11 @@ -import { GetLogEventsCommand, type CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; -import { ResultTruncationError } from "../errors"; +import type { CloudWatchLogEvent } from "./observability/index"; import type { BatchEvaluationResultEntry } from "../handlers/eval/types"; import type { Logger } from "../logging"; -// Per-session batch-evaluation result retrieval, mirroring -// core/onlineEvalExecutionRole.tsx's pattern: a self-contained module that takes -// an injected AWS client (here CloudWatchLogsClient) and owns one slice of Core's -// behavior. A completed batch evaluation writes each score as an OTel-shaped log -// record to a per-job CloudWatch stream; this module reads that stream and parses -// the records. EvalClient calls readEvaluationResults with the client from -// `this.clients.logs(...)` and the log group + stream from the job's outputConfig. +// A completed batch evaluation writes each score as an OTel-shaped log record to +// a per-job CloudWatch stream. CloudWatchClient owns retrieving and paginating +// that exact stream; this module owns only the eval-specific record parsing and +// output shape. // Terminal batch-evaluation statuses — after these, results are final and worth // retrieving. Mirrors the AgentCore BatchEvaluationStatus enum's terminal arm. @@ -19,65 +15,19 @@ export function isTerminalStatus(status?: string): boolean { return !!status && TERMINAL_STATUSES.has(status); } -// GetLogEvents returns at most 1 MB / 10,000 events per call, so a job with many -// results spans multiple pages. This caps the page loop as a safety valve against -// a non-advancing token (see below). At 10k events/page it allows ~1M results, but -// the 1 MB limit binds first — large explanations can cap a page well under 10k, so -// this is not a "far beyond any real job" ceiling. Hitting it means the results are -// truncated, which we surface as an error (see below) rather than silently -// returning a partial list as if complete. -const MAX_RESULT_PAGES = 100; - -// readEvaluationResults reads and parses the per-session/-trace/-tool scores from -// a completed batch evaluation's CloudWatch result stream, following pagination to -// completion. Throws if the stream exceeds MAX_RESULT_PAGES (results would be -// truncated) — see the throw site. The caller supplies the log group and stream -// name from the job's GetBatchEvaluation outputConfig (the service-selected values -// — we do not derive the stream name, since its format is not part of the SDK -// contract). +// readEvaluationResults parses the per-session/-trace/-tool scores from a +// completed batch evaluation's normalized CloudWatch events. export async function readEvaluationResults( - logs: CloudWatchLogsClient, - logGroupName: string, - logStreamName: string, + events: AsyncIterable, logger: Logger, ): Promise { const results: BatchEvaluationResultEntry[] = []; - - // Page forward from the head. GetLogEvents echoes the input token back as - // nextForwardToken once the stream is exhausted, so the loop ends when the - // token stops advancing. startFromHead is only honored on the first call (no - // token); subsequent calls are positioned by the token. - let token: string | undefined; - for (let page = 0; page < MAX_RESULT_PAGES; page++) { - const response = await logs.send( - new GetLogEventsCommand({ - logGroupName, - logStreamName, - startFromHead: true, - nextToken: token, - }), - ); - - for (const event of response.events ?? []) { - if (!event.message) continue; - const entry = parseEvaluationLogEvent(event.message, logger); - if (entry) results.push(entry); - } - - const next = response.nextForwardToken; - if (!next || next === token) return results; // exhausted: token stopped advancing - token = next; + for await (const event of events) { + if (!event.message) continue; + const entry = parseEvaluationLogEvent(event.message, logger); + if (entry) results.push(entry); } - - // Cap reached with the token still advancing: the stream has more pages than we - // read, so `results` is truncated. Throw rather than return the partial list — - // getBatchEvaluation catches this into `resultsError`, which the CLI surfaces as - // a stderr warning (stdout metadata stays clean), the same customer-visible path - // as any other CloudWatch read failure. A silent partial list would read as - // complete. - throw new ResultTruncationError( - `batch-evaluation results exceed ${MAX_RESULT_PAGES} CloudWatch pages; retrieved ${results.length} results are incomplete`, - ); + return results; } // parseEvaluationLogEvent turns one CloudWatch result-log message into a result diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 10d0796b2..34ad58774 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -116,6 +116,7 @@ import { sanitizeQueryValue, type InsightsRowLimit, } from "./observability"; +import { CloudWatchClient } from "./observability/index"; import type { BatchEvaluationDetail, CodeBasedUpdate, @@ -175,6 +176,7 @@ import { scopePolicyName, } from "./onlineEvalExecutionRole"; import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; +import { harnessRuntimeFromResponse } from "./harness"; const DEFAULT_INGESTION_WAIT_MS = 180_000; const DATASET_EXAMPLES_BATCH_LIMIT = 1000; @@ -212,6 +214,7 @@ const EVAL_INSIGHTS_ROW_LIMIT: InsightsRowLimit = { }; const DEFAULT_BATCH_INSIGHTS_PAGE_SIZE = 50; +const BATCH_EVALUATION_RESULT_MAX_PAGES = 100; // noopLogger is the default for the optional logger arg so callers that don't // need batch-evaluation result-log diagnostics (e.g. dataset-only tests) can @@ -235,6 +238,7 @@ export class EvalClient implements CoreEvalClient { private readonly logger: Logger = noopLogger, private readonly newSessionId: () => string = randomUUID, private readonly now: () => number = () => Date.now(), + private readonly cloudWatch: CloudWatchClient = new CloudWatchClient(clients), ) {} async createEvaluator( @@ -434,9 +438,16 @@ export class EvalClient implements CoreEvalClient { try { detail.results = await readEvaluationResults( - this.clients.logs({ region: options.region }), - cw.logGroupName, - cw.logStreamName, + this.cloudWatch.readLogStream( + { + logGroupName: cw.logGroupName, + logStreamName: cw.logStreamName, + }, + { + maxPages: BATCH_EVALUATION_RESULT_MAX_PAGES, + }, + options, + ), this.logger, ); return { detail }; @@ -1946,17 +1957,15 @@ async function resolveAgentToNameAndId( } const harness = await control.send(new GetHarnessCommand({ harnessId: agent })); - const environment = harness.harness?.environment; - const runtimeEnv = - environment && "agentCoreRuntimeEnvironment" in environment - ? environment.agentCoreRuntimeEnvironment - : undefined; - if (!runtimeEnv?.agentRuntimeId || !runtimeEnv?.agentRuntimeName) { + try { + return harnessRuntimeFromResponse(agent, harness); + } catch (error) { + if (!(error instanceof InputValidationError)) throw error; throw new InputValidationError(`"${agent}" does not exist as a runtime or a harness`, { + cause: error, meta: { agent }, }); } - return { runtimeId: runtimeEnv.agentRuntimeId, runtimeName: runtimeEnv.agentRuntimeName }; } // agentDataSource builds the CloudWatch data source for an agent id, resolving it diff --git a/src/core/harness.test.ts b/src/core/harness.test.ts new file mode 100644 index 000000000..da354171c --- /dev/null +++ b/src/core/harness.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from "bun:test"; +import { + GetHarnessCommand, + type GetHarnessResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { InputValidationError } from "../errors"; +import type { AwsClients, ClientConfig } from "./types"; +import { HarnessClient, harnessRuntimeFromResponse } from "./harness"; + +const RESPONSE = { + harness: { + environment: { + agentCoreRuntimeEnvironment: { + agentRuntimeId: "harness_MyHarness-AbC123", + agentRuntimeName: "harness_MyHarness", + }, + }, + }, +} as GetHarnessResponse; + +describe("harnessRuntimeFromResponse", () => { + test("rejects a harness without a resolvable Runtime environment", () => { + expect(() => + harnessRuntimeFromResponse("MyHarness-abc123", { + harness: { environment: { $unknown: ["futureProvider", {}] } }, + } as GetHarnessResponse), + ).toThrow(InputValidationError); + }); +}); + +describe("HarnessClient.resolveRuntime", () => { + test("gets the harness with the configured client and forwards cancellation", async () => { + const configs: ClientConfig[] = []; + const controller = new AbortController(); + const clients = { + control: (config: ClientConfig) => { + configs.push(config); + return { + send: async (command: unknown, options?: { abortSignal?: AbortSignal }) => { + expect(command).toBeInstanceOf(GetHarnessCommand); + expect((command as GetHarnessCommand).input).toEqual({ + harnessId: "MyHarness-abc123", + }); + expect(options?.abortSignal).toBe(controller.signal); + return RESPONSE; + }, + }; + }, + } as unknown as AwsClients; + const client = new HarnessClient(clients); + + await client.resolveRuntime( + "MyHarness-abc123", + { region: "us-west-2", endpointUrl: "https://control.test" }, + controller.signal, + ); + expect(configs).toEqual([{ region: "us-west-2", endpoint: "https://control.test" }]); + }); +}); diff --git a/src/core/harness.tsx b/src/core/harness.tsx index 7a5edeec9..0d88822ec 100644 --- a/src/core/harness.tsx +++ b/src/core/harness.tsx @@ -35,7 +35,12 @@ import { type InvokeHarnessRequest, type InvokeHarnessResponse, } from "@aws-sdk/client-bedrock-agentcore"; -import type { CoreHarnessClient, CreateHarnessInput } from "../handlers/harness/types"; +import type { + CoreHarnessClient, + CreateHarnessInput, + ResolvedHarnessRuntime, +} from "../handlers/harness/types"; +import { InputValidationError } from "../errors"; import type { AwsClients, CoreOptions } from "./types"; import { abortable } from "./abortable"; import { ensureDefaultExecutionRole } from "./executionRole"; @@ -53,6 +58,17 @@ export class HarnessClient implements CoreHarnessClient { .send(new GetHarnessCommand({ harnessId: id })); } + async resolveRuntime( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const response = await this.clients + .control(toClientConfig(options)) + .send(new GetHarnessCommand({ harnessId: id }), { abortSignal: signal }); + return harnessRuntimeFromResponse(id, response); + } + async getHarnessVersion( id: string, version: string, @@ -207,6 +223,27 @@ export class HarnessClient implements CoreHarnessClient { } } +export function harnessRuntimeFromResponse( + id: string, + response: GetHarnessResponse, +): ResolvedHarnessRuntime { + const environment = response.harness?.environment; + const runtime = + environment && "agentCoreRuntimeEnvironment" in environment + ? environment.agentCoreRuntimeEnvironment + : undefined; + if (!runtime?.agentRuntimeId || !runtime.agentRuntimeName) { + throw new InputValidationError( + `Harness "${id}" does not expose an AgentCore Runtime environment`, + { meta: { harnessId: id } }, + ); + } + return { + runtimeId: runtime.agentRuntimeId, + runtimeName: runtime.agentRuntimeName, + }; +} + // retryWhileRoleUnassumable retries `operation` while it fails with the // validation error AgentCore raises for an execution role it cannot yet assume // (fresh IAM roles propagate over several seconds). Any other failure — or diff --git a/src/core/index.tsx b/src/core/index.tsx index 057c3cd03..d6c8658b0 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -9,6 +9,7 @@ import { IdentityClient } from "./identity"; import { MemoryClient } from "./memory"; import { PolicyClient } from "./policy"; import { ObservabilityClient } from "./observability"; +import { CloudWatchClient } from "./observability/index"; import { RuntimeClient } from "./runtime"; import type { OpenRuntimeShell } from "./runtime"; import { FsReadWriteJson } from "../io"; @@ -90,6 +91,7 @@ export class CoreClient implements AwsClients { this.createLogsClient = config.createLogsClient; this.logger = config.logger; const fetch = config.fetch ?? globalThis.fetch; + const cloudWatch = new CloudWatchClient(this); this.fetch = fetch; this.runtime = new RuntimeClient( this, @@ -108,12 +110,13 @@ export class CoreClient implements AwsClients { this.logger.child({ module: "eval" }), config.newSessionId, config.now, + cloudWatch, ); // Observability resolves a project's deployed runtime from its stack // outputs, so it reads aws-targets.json through the same JSON layer the // project manager uses. - this.observability = new ObservabilityClient(this, { + this.observability = new ObservabilityClient(cloudWatch, { readJson: new FsReadWriteJson({ logger: this.logger.child({ module: "observability" }), }), diff --git a/src/core/observability.test.ts b/src/core/observability.test.ts index 30777dc93..80dc92d60 100644 --- a/src/core/observability.test.ts +++ b/src/core/observability.test.ts @@ -24,6 +24,7 @@ import { sanitizeQueryValue, type DescribeStackOutputs, } from "./observability"; +import { CloudWatchClient } from "./observability/index"; describe("runtimeLogGroup", () => { test("derives the fixed per-runtime path keyed by runtime id and endpoint", () => { @@ -141,7 +142,10 @@ function clientWith(logs: CloudWatchLogsClient, describeStackOutputs?: DescribeS throw new Error("not implemented"); }, } as ReadWriteJson; - return new ObservabilityClient(clients, { readJson, describeStackOutputs }); + return new ObservabilityClient(new CloudWatchClient(clients), { + readJson, + describeStackOutputs, + }); } function fakeProject(rootPath: string, name = "My_Project"): Project { diff --git a/src/core/observability.ts b/src/core/observability.ts index 97a38ec6c..f014ee400 100644 --- a/src/core/observability.ts +++ b/src/core/observability.ts @@ -23,7 +23,6 @@ import { ObservabilityClient as GenericObservabilityClient, } from "./observability/index"; import { isStackNotFound } from "./project/backends/cdk/environment"; -import type { AwsClients } from "./types"; // Shared CloudWatch observability helpers. AgentCore Runtimes write their logs // and OTel telemetry to per-runtime CloudWatch log groups; both the eval flows @@ -244,8 +243,8 @@ export class ObservabilityClient private readonly readJson: ReadWriteJson; private readonly describeStackOutputs: DescribeStackOutputs; - constructor(clients: AwsClients, deps: ObservabilityClientDeps) { - super(new CloudWatchClient(clients)); + constructor(cloudWatch: CloudWatchClient, deps: ObservabilityClientDeps) { + super(cloudWatch); this.readJson = deps.readJson; this.describeStackOutputs = deps.describeStackOutputs ?? describeStackOutputsWithSdk; } diff --git a/src/core/observability/cloudWatchClient.test.ts b/src/core/observability/cloudWatchClient.test.ts index 1832c4e41..350f69866 100644 --- a/src/core/observability/cloudWatchClient.test.ts +++ b/src/core/observability/cloudWatchClient.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { DescribeLogGroupsCommand, FilterLogEventsCommand, + GetLogEventsCommand, GetQueryResultsCommand, ResourceNotFoundException, StartLiveTailCommand, @@ -9,6 +10,7 @@ import { type CloudWatchLogsClient, type StartLiveTailResponseStream, } from "@aws-sdk/client-cloudwatch-logs"; +import { ResultTruncationError } from "../../errors"; import type { ClientConfig } from "../types"; import { CloudWatchClient } from "./cloudWatchClient"; import type { CloudWatchLogEvent } from "./types"; @@ -17,6 +19,10 @@ const SOURCE = { provider: "cloudwatch" as const, logGroupName: "/aws/bedrock-agentcore/runtimes/runtime-1-DEFAULT", }; +const STREAM_SOURCE = { + logGroupName: SOURCE.logGroupName, + logStreamName: "batch-evaluation/results", +}; const OPTIONS = { region: "us-west-2", endpointUrl: "https://logs.test", @@ -42,6 +48,84 @@ async function collect(records: AsyncIterable) { return result; } +describe("CloudWatchClient.readLogStream", () => { + test("reads an exact stream until the forward token stops advancing", async () => { + const inputs: unknown[] = []; + const { client, configs } = clientWith(async (command) => { + expect(command).toBeInstanceOf(GetLogEventsCommand); + const input = (command as GetLogEventsCommand).input; + inputs.push(input); + if (input.nextToken === "page-1") { + return { + events: [{ timestamp: 2, ingestionTime: 3, message: "two" }], + nextForwardToken: "page-1", + }; + } + return { + events: [{ timestamp: 1, message: "one" }], + nextForwardToken: "page-1", + }; + }); + + const records = await collect(client.readLogStream(STREAM_SOURCE, {}, OPTIONS)); + + expect(configs).toEqual([{ region: "us-west-2", endpoint: "https://logs.test" }]); + expect(inputs).toEqual([ + { + logGroupName: STREAM_SOURCE.logGroupName, + logStreamName: STREAM_SOURCE.logStreamName, + startFromHead: true, + }, + { + logGroupName: STREAM_SOURCE.logGroupName, + logStreamName: STREAM_SOURCE.logStreamName, + startFromHead: true, + nextToken: "page-1", + }, + ]); + expect(records).toEqual([ + { + timestamp: new Date(1), + message: "one", + logStreamName: STREAM_SOURCE.logStreamName, + }, + { + timestamp: new Date(2), + ingestionTime: new Date(3), + message: "two", + logStreamName: STREAM_SOURCE.logStreamName, + }, + ]); + }); + + test("throws rather than returning a partial stream at the page ceiling", async () => { + let call = 0; + const { client } = clientWith(async () => ({ + events: [{ timestamp: call, message: `event-${call}` }], + nextForwardToken: `page-${call++}`, + })); + + const error = await collect(client.readLogStream(STREAM_SOURCE, { maxPages: 2 }, OPTIONS)).then( + () => undefined, + (caught) => caught, + ); + + expect(error).toBeInstanceOf(ResultTruncationError); + expect((error as Error).message).toContain("retrieved events are incomplete"); + expect((error as ResultTruncationError).source).toBe("internal"); + }); + + test("translates a missing stream into customer guidance", async () => { + const { client } = clientWith(async () => { + throw new ResourceNotFoundException({ message: "missing", $metadata: {} }); + }); + + await expect(collect(client.readLogStream(STREAM_SOURCE, {}, OPTIONS))).rejects.toThrow( + `CloudWatch log stream ${STREAM_SOURCE.logStreamName} does not exist`, + ); + }); +}); + describe("CloudWatchClient.searchLogs", () => { test("paginates, preserves provider metadata, and uses the configured client", async () => { const inputs: unknown[] = []; diff --git a/src/core/observability/cloudWatchClient.ts b/src/core/observability/cloudWatchClient.ts index c668e9152..a2c0a423e 100644 --- a/src/core/observability/cloudWatchClient.ts +++ b/src/core/observability/cloudWatchClient.ts @@ -1,12 +1,14 @@ import { DescribeLogGroupsCommand, FilterLogEventsCommand, + GetLogEventsCommand, ResourceNotFoundException, StartLiveTailCommand, type FilteredLogEvent, type LiveTailSessionLogEvent, + type OutputLogEvent, } from "@aws-sdk/client-cloudwatch-logs"; -import { ResourceNotFoundError } from "../../errors"; +import { ResourceNotFoundError, ResultTruncationError } from "../../errors"; import type { AwsClients, CoreOptions } from "../types"; import { toClientConfig } from "../utils"; import { runInsightsQuery } from "./insights"; @@ -16,12 +18,69 @@ import type { InsightsQueryRow, LogSearchQuery, LogSource, + LogStreamQuery, + LogStreamSource, LogTailQuery, } from "./types"; export class CloudWatchClient { constructor(private readonly clients: Pick) {} + async *readLogStream( + source: LogStreamSource, + query: LogStreamQuery, + options: CoreOptions, + signal?: AbortSignal, + ): AsyncGenerator { + if (query.maxPages !== undefined && query.maxPages <= 0) return; + + const logs = this.clients.logs(toClientConfig(options)); + let nextToken: string | undefined; + let pages = 0; + + while (true) { + if (query.maxPages !== undefined && pages >= query.maxPages) { + throw new ResultTruncationError( + `CloudWatch log stream exceeded ${query.maxPages} pages; retrieved events are incomplete`, + { + meta: { + logGroupName: source.logGroupName, + logStreamName: source.logStreamName, + maxPages: query.maxPages, + }, + }, + ); + } + + const requestToken = nextToken; + let response; + try { + response = await logs.send( + new GetLogEventsCommand({ + logGroupName: source.logGroupName, + logStreamName: source.logStreamName, + startFromHead: true, + ...(requestToken ? { nextToken: requestToken } : {}), + }), + { abortSignal: signal }, + ); + } catch (error) { + if (error instanceof ResourceNotFoundException) { + throw missingLogStreamError(source, error); + } + throw error; + } + pages++; + + for (const event of response.events ?? []) { + yield toCloudWatchLogEvent(event, source.logStreamName); + } + + nextToken = response.nextForwardToken; + if (!nextToken || nextToken === requestToken) return; + } + } + async *searchLogs( source: LogSource, query: LogSearchQuery, @@ -162,13 +221,18 @@ export class CloudWatchClient { } function toCloudWatchLogEvent( - event: FilteredLogEvent | LiveTailSessionLogEvent, + event: FilteredLogEvent | LiveTailSessionLogEvent | OutputLogEvent, + logStreamName?: string, ): CloudWatchLogEvent { return { timestamp: new Date(event.timestamp ?? Date.now()), message: event.message ?? "", ...(event.ingestionTime !== undefined ? { ingestionTime: new Date(event.ingestionTime) } : {}), - ...(event.logStreamName ? { logStreamName: event.logStreamName } : {}), + ...("logStreamName" in event && event.logStreamName + ? { logStreamName: event.logStreamName } + : logStreamName + ? { logStreamName } + : {}), }; } @@ -179,3 +243,17 @@ function missingLogGroupError(source: LogSource, cause?: unknown): ResourceNotFo { cause, meta: { logGroupName: source.logGroupName } }, ); } + +function missingLogStreamError(source: LogStreamSource, cause?: unknown): ResourceNotFoundError { + return new ResourceNotFoundError( + `CloudWatch log stream ${source.logStreamName} does not exist in log group ` + + `${source.logGroupName}. Has the resource emitted results yet?`, + { + cause, + meta: { + logGroupName: source.logGroupName, + logStreamName: source.logStreamName, + }, + }, + ); +} diff --git a/src/core/observability/index.ts b/src/core/observability/index.ts index d8a755d02..46527288c 100644 --- a/src/core/observability/index.ts +++ b/src/core/observability/index.ts @@ -15,6 +15,8 @@ export type { ListTracesQuery, LogSearchQuery, LogSource, + LogStreamQuery, + LogStreamSource, LogTailQuery, TraceRecord, TraceSummary, diff --git a/src/core/observability/types.ts b/src/core/observability/types.ts index ffeb0c8c1..202bdee2e 100644 --- a/src/core/observability/types.ts +++ b/src/core/observability/types.ts @@ -5,6 +5,11 @@ export type LogSource = { logGroupName: string; }; +/** Exact CloudWatch log stream selected by a caller. */ +export type LogStreamSource = LogSource & { + logStreamName: string; +}; + /** CloudWatch log event normalized at the AWS client boundary. */ export type CloudWatchLogEvent = { timestamp: Date; @@ -13,6 +18,11 @@ export type CloudWatchLogEvent = { logStreamName?: string; }; +export type LogStreamQuery = { + /** Safety ceiling for callers that require a bounded stream read. */ + maxPages?: number; +}; + export type LogSearchQuery = { startTimeMs: number; endTimeMs: number; diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx index 034e3ac4d..b3b8a8f62 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx @@ -24,10 +24,11 @@ const FIXTURES = join(import.meta.dir, "__fixtures__"); // re-recording. // // This suite exercises the real seam end to end: parsing → handler → CoreClient → -// GetBatchEvaluation (data plane) → readEvaluationResults → GetLogEvents (the -// createLogsClient fixture seam). The TestCoreClient suite (batch-evaluation.test.tsx) -// covers the edges that can't be recorded on demand: a non-terminal job, a -// CloudWatch read failure, and list pagination. +// GetBatchEvaluation (data plane) → CloudWatchClient.readLogStream → +// readEvaluationResults → GetLogEvents (the createLogsClient fixture seam). The +// TestCoreClient suite (batch-evaluation.test.tsx) covers the edges that can't be +// recorded on demand: a non-terminal job, a CloudWatch read failure, and list +// pagination. const FIXTURE_JOB_ID = "GTProbe2_1786034545579-8ffefc851e"; // A well-formed but absent id, to reach the not-found path without a diff --git a/src/handlers/harness/harness.test.tsx b/src/handlers/harness/harness.test.tsx index 532d6e76e..88286eef9 100644 --- a/src/handlers/harness/harness.test.tsx +++ b/src/handlers/harness/harness.test.tsx @@ -29,16 +29,20 @@ const REGION = "us-west-2"; // run builds a fresh handler tree (CoreClient carries per-run caches, so this // keeps tests isolated) over an in-memory io, routes `args` beneath `agentcore`, // and returns whatever the command wrote to stdout. -async function run(args: string[]): Promise { +function createFixtureCore(): CoreClient { const { createControlClient, createDataClient, createIamClient, createLogsClient } = fixtureFactories(FIXTURES); - const core = new CoreClient({ + return new CoreClient({ createControlClient, createDataClient, createIamClient, createLogsClient, logger: createSilentLogger(), }); +} + +async function run(args: string[]): Promise { + const core = createFixtureCore(); const io = testIO(); const root = createRootHandler(core, { io: io.io, @@ -79,6 +83,19 @@ describe("harness get", () => { }); }); +describe("harness runtime resolution", () => { + test("resolves the underlying Runtime from a recorded harness response", async () => { + const core = createFixtureCore(); + + await expect( + core.harness.resolveRuntime("MyPDXHarness-rhkXkAE1IS", { region: REGION }), + ).resolves.toEqual({ + runtimeId: "harness_MyPDXHarness-8WBKGfHzjg", + runtimeName: "harness_MyPDXHarness", + }); + }); +}); + describe("harness endpoint list", () => { test("prints the harness's endpoints as JSON for a given id", async () => { const out = await run(["harness", "endpoint", "list", "--id", "MyPDXHarness-rhkXkAE1IS"]); diff --git a/src/handlers/harness/index.tsx b/src/handlers/harness/index.tsx index b0b74ef65..72a4a129c 100644 --- a/src/handlers/harness/index.tsx +++ b/src/handlers/harness/index.tsx @@ -12,6 +12,8 @@ import { createInvokeHarnessHandler } from "./invoke"; import { createExecHarnessHandler } from "./exec"; import { createEndpointHandler } from "./endpoint"; import { createVersionHandler } from "./version"; +import { createHarnessLogsHandler } from "./logs"; +import { createHarnessTracesHandler } from "./traces"; export function createHarnessHandler(core: Core, io: AppIO): Router { const harness = new Router("harness", "manage AgentCore harnesses"); @@ -29,6 +31,8 @@ export function createHarnessHandler(core: Core, io: AppIO): Router { harness.handler(createDeleteHarnessHandler(core)); harness.handler(createInvokeHarnessHandler(core, io)); harness.handler(createExecHarnessHandler(core, io)); + harness.handler(createHarnessLogsHandler(core, io)); + harness.handler(createHarnessTracesHandler(core, io)); // Endpoint and version commands live under their own sub-routers, e.g. // `agentcore harness endpoint create`. diff --git a/src/handlers/harness/logs/index.tsx b/src/handlers/harness/logs/index.tsx new file mode 100644 index 000000000..634244964 --- /dev/null +++ b/src/handlers/harness/logs/index.tsx @@ -0,0 +1,39 @@ +import z from "zod"; +import { DEFAULT_ENDPOINT_QUALIFIER, runtimeLogGroup } from "../../../core/observability"; +import type { AppIO } from "../../../io"; +import { flag } from "../../../router"; +import { createLogsHandler } from "../../observability/logs"; +import type { Core } from "../../types"; +import { coreOptsFromCtx } from "../../utils"; + +const harnessFlags = [ + flag("id", "the ID of the harness", z.string().min(1).max(48)), + flag("qualifier", "the harness endpoint qualifier", z.string().min(1).optional()), +] as const; + +export const createHarnessLogsHandler = (core: Core, io: AppIO) => + createLogsHandler(io, { + description: "stream or search a harness's logs", + flags: harnessFlags, + read: async (ctx, flags, request, signal) => { + const options = coreOptsFromCtx(ctx); + const runtime = await core.harness.resolveRuntime(flags.id, options, signal); + const source = { + logGroupName: runtimeLogGroup( + runtime.runtimeId, + flags.qualifier ?? DEFAULT_ENDPOINT_QUALIFIER, + ), + }; + + if (request.mode === "search") { + return { + events: core.observability.searchLogs(source, request.query, options, signal), + }; + } + + return { + events: core.observability.tailLogs(source, request.query, options, signal), + announcement: `Streaming logs for harness ${flags.id}... (Ctrl+C to stop)`, + }; + }, + }); diff --git a/src/handlers/harness/traces/get/index.tsx b/src/handlers/harness/traces/get/index.tsx new file mode 100644 index 000000000..daa1be085 --- /dev/null +++ b/src/handlers/harness/traces/get/index.tsx @@ -0,0 +1,31 @@ +import z from "zod"; +import { DEFAULT_ENDPOINT_QUALIFIER, runtimeLogGroup } from "../../../../core/observability"; +import type { AppIO } from "../../../../io"; +import { flag } from "../../../../router"; +import { resolveTraceOutputPath } from "../../../observability/traceOutputPath"; +import { createGetTraceHandler } from "../../../observability/traces"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +const harnessFlags = [ + flag("id", "the ID of the harness", z.string().min(1).max(48)), + flag("qualifier", "the harness endpoint qualifier", z.string().min(1).optional()), +] as const; + +export const createGetHarnessTraceHandler = (core: Core, io: AppIO) => + createGetTraceHandler(io, { + description: "download a harness trace's log records to a JSON file", + flags: harnessFlags, + read: async (ctx, flags, query, signal) => { + const options = coreOptsFromCtx(ctx); + const runtime = await core.harness.resolveRuntime(flags.id, options, signal); + const source = { + logGroupName: runtimeLogGroup( + runtime.runtimeId, + flags.qualifier ?? DEFAULT_ENDPOINT_QUALIFIER, + ), + }; + return core.observability.getTrace(source, query, options, signal); + }, + resolveOutputPath: (_ctx, _flags, request) => resolveTraceOutputPath(request), + }); diff --git a/src/handlers/harness/traces/index.tsx b/src/handlers/harness/traces/index.tsx new file mode 100644 index 000000000..525226907 --- /dev/null +++ b/src/handlers/harness/traces/index.tsx @@ -0,0 +1,12 @@ +import type { AppIO } from "../../../io"; +import { createTracesHandler } from "../../observability/traces"; +import type { Core } from "../../types"; +import { createGetHarnessTraceHandler } from "./get"; +import { createListHarnessTracesHandler } from "./list"; + +export const createHarnessTracesHandler = (core: Core, io: AppIO) => + createTracesHandler({ + description: "inspect a harness's traces", + list: createListHarnessTracesHandler(core, io), + get: createGetHarnessTraceHandler(core, io), + }); diff --git a/src/handlers/harness/traces/list/index.tsx b/src/handlers/harness/traces/list/index.tsx new file mode 100644 index 000000000..02dd56c44 --- /dev/null +++ b/src/handlers/harness/traces/list/index.tsx @@ -0,0 +1,29 @@ +import z from "zod"; +import { DEFAULT_ENDPOINT_QUALIFIER, runtimeLogGroup } from "../../../../core/observability"; +import type { AppIO } from "../../../../io"; +import { flag } from "../../../../router"; +import { createListTracesHandler } from "../../../observability/traces"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +const harnessFlags = [ + flag("id", "the ID of the harness", z.string().min(1).max(48)), + flag("qualifier", "the harness endpoint qualifier", z.string().min(1).optional()), +] as const; + +export const createListHarnessTracesHandler = (core: Core, io: AppIO) => + createListTracesHandler(io, { + description: "list a harness's recent traces", + flags: harnessFlags, + read: async (ctx, flags, query, signal) => { + const options = coreOptsFromCtx(ctx); + const runtime = await core.harness.resolveRuntime(flags.id, options, signal); + const source = { + logGroupName: runtimeLogGroup( + runtime.runtimeId, + flags.qualifier ?? DEFAULT_ENDPOINT_QUALIFIER, + ), + }; + return core.observability.listTraces(source, query, options, signal); + }, + }); diff --git a/src/handlers/harness/types.tsx b/src/handlers/harness/types.tsx index 6f8ad3a08..23f0df7be 100644 --- a/src/handlers/harness/types.tsx +++ b/src/handlers/harness/types.tsx @@ -32,6 +32,11 @@ export type CreateHarnessInput = Omit executionRoleArn?: string; }; +export type ResolvedHarnessRuntime = { + runtimeId: string; + runtimeName: string; +}; + export interface CoreHarnessClient { createHarness(input: CreateHarnessInput, options: CoreOptions): Promise; updateHarness( @@ -55,6 +60,11 @@ export interface CoreHarnessClient { options: CoreOptions, ): Promise; getHarness(id: string, options: CoreOptions): Promise; + resolveRuntime( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; getHarnessVersion(id: string, version: string, options: CoreOptions): Promise; getHarnessEndpoint( id: string, diff --git a/src/handlers/runtime/traces/outputPath.ts b/src/handlers/observability/traceOutputPath.ts similarity index 100% rename from src/handlers/runtime/traces/outputPath.ts rename to src/handlers/observability/traceOutputPath.ts diff --git a/src/handlers/observability/traces.ts b/src/handlers/observability/traces.ts index 9779b3db1..f2feb116a 100644 --- a/src/handlers/observability/traces.ts +++ b/src/handlers/observability/traces.ts @@ -133,7 +133,6 @@ export function createGetTraceHandler[] io: AppIO, config: { description: string; - outputDescription: string; flags: F; read( ctx: Context, @@ -150,7 +149,11 @@ export function createGetTraceHandler[] ): Handler { const getFlags = [ ...config.flags, - flag("output", config.outputDescription, outputSchema), + flag( + "output", + "the output file path (default: .json in the current directory)", + outputSchema, + ), ...traceWindowFlags, ] as const; diff --git a/src/handlers/runtime/logs/index.tsx b/src/handlers/runtime/logs/index.tsx index f06249b33..bea96272f 100644 --- a/src/handlers/runtime/logs/index.tsx +++ b/src/handlers/runtime/logs/index.tsx @@ -1,3 +1,4 @@ +import z from "zod"; import { DEFAULT_ENDPOINT_QUALIFIER, runtimeLogGroup } from "../../../core/observability"; import type { AppIO } from "../../../io"; import { flag } from "../../../router"; @@ -6,7 +7,10 @@ import type { Core } from "../../types"; import { coreOptsFromCtx } from "../../utils"; import { runtimeIdSchema } from "../invoke/request"; -const runtimeFlags = [flag("id", "the ID of the Runtime", runtimeIdSchema)] as const; +const runtimeFlags = [ + flag("id", "the ID of the Runtime", runtimeIdSchema), + flag("qualifier", "the Runtime endpoint qualifier", z.string().min(1).optional()), +] as const; /** * `runtime logs` follows a deployed runtime's CloudWatch log group live @@ -21,7 +25,7 @@ export const createRuntimeLogsHandler = (core: Core, io: AppIO) => read: (ctx, flags, request, signal) => { const options = coreOptsFromCtx(ctx); const source = { - logGroupName: runtimeLogGroup(flags.id, DEFAULT_ENDPOINT_QUALIFIER), + logGroupName: runtimeLogGroup(flags.id, flags.qualifier ?? DEFAULT_ENDPOINT_QUALIFIER), }; if (request.mode === "search") { diff --git a/src/handlers/runtime/logs/logs.test.tsx b/src/handlers/runtime/logs/logs.test.tsx index 91e41cbd4..51769edb8 100644 --- a/src/handlers/runtime/logs/logs.test.tsx +++ b/src/handlers/runtime/logs/logs.test.tsx @@ -57,6 +57,25 @@ describe("runtime logs", () => { expect(io.stdout()).toBe("2024-03-02T14:50:00.000Z tailed"); }); + test("uses the requested endpoint qualifier", async () => { + const { core, route } = testLogsCommand(); + + await route([ + "runtime", + "logs", + "--id", + "my_agent-AbC123XyZ9", + "--qualifier", + "live", + "--since", + "1h", + ]); + + expect(core.observability.calls[0]?.args[0]).toEqual({ + logGroupName: "/aws/bedrock-agentcore/runtimes/my_agent-AbC123XyZ9-live", + }); + }); + test("rejects --limit outside search mode", async () => { const { route } = testLogsCommand(); diff --git a/src/handlers/runtime/traces/get/index.tsx b/src/handlers/runtime/traces/get/index.tsx index 2c0ee8c2c..a94274719 100644 --- a/src/handlers/runtime/traces/get/index.tsx +++ b/src/handlers/runtime/traces/get/index.tsx @@ -1,22 +1,25 @@ +import z from "zod"; import { DEFAULT_ENDPOINT_QUALIFIER, runtimeLogGroup } from "../../../../core/observability"; import type { AppIO } from "../../../../io"; import { flag } from "../../../../router"; import { createGetTraceHandler } from "../../../observability/traces"; +import { resolveTraceOutputPath } from "../../../observability/traceOutputPath"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { runtimeIdSchema } from "../../invoke/request"; -import { resolveTraceOutputPath } from "../outputPath"; -const runtimeFlags = [flag("id", "the ID of the Runtime", runtimeIdSchema)] as const; +const runtimeFlags = [ + flag("id", "the ID of the Runtime", runtimeIdSchema), + flag("qualifier", "the Runtime endpoint qualifier", z.string().min(1).optional()), +] as const; export const createGetRuntimeTraceHandler = (core: Core, io: AppIO) => createGetTraceHandler(io, { description: "download a trace's log records to a JSON file", - outputDescription: "the output file path (default .json in the current directory)", flags: runtimeFlags, read: (ctx, flags, query, signal) => { const source = { - logGroupName: runtimeLogGroup(flags.id, DEFAULT_ENDPOINT_QUALIFIER), + logGroupName: runtimeLogGroup(flags.id, flags.qualifier ?? DEFAULT_ENDPOINT_QUALIFIER), }; return core.observability.getTrace(source, query, coreOptsFromCtx(ctx), signal); }, diff --git a/src/handlers/runtime/traces/list/index.tsx b/src/handlers/runtime/traces/list/index.tsx index 9a5901d8e..8f3a875bb 100644 --- a/src/handlers/runtime/traces/list/index.tsx +++ b/src/handlers/runtime/traces/list/index.tsx @@ -1,3 +1,4 @@ +import z from "zod"; import { DEFAULT_ENDPOINT_QUALIFIER, runtimeLogGroup } from "../../../../core/observability"; import type { AppIO } from "../../../../io"; import { flag } from "../../../../router"; @@ -6,7 +7,10 @@ import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { runtimeIdSchema } from "../../invoke/request"; -const runtimeFlags = [flag("id", "the ID of the Runtime", runtimeIdSchema)] as const; +const runtimeFlags = [ + flag("id", "the ID of the Runtime", runtimeIdSchema), + flag("qualifier", "the Runtime endpoint qualifier", z.string().min(1).optional()), +] as const; export const createListRuntimeTracesHandler = (core: Core, io: AppIO) => createListTracesHandler(io, { @@ -14,7 +18,7 @@ export const createListRuntimeTracesHandler = (core: Core, io: AppIO) => flags: runtimeFlags, read: (ctx, flags, query, signal) => { const source = { - logGroupName: runtimeLogGroup(flags.id, DEFAULT_ENDPOINT_QUALIFIER), + logGroupName: runtimeLogGroup(flags.id, flags.qualifier ?? DEFAULT_ENDPOINT_QUALIFIER), }; return core.observability.listTraces(source, query, coreOptsFromCtx(ctx), signal); }, diff --git a/src/handlers/runtime/traces/traces.test.tsx b/src/handlers/runtime/traces/traces.test.tsx index 118d64ab4..955b0562f 100644 --- a/src/handlers/runtime/traces/traces.test.tsx +++ b/src/handlers/runtime/traces/traces.test.tsx @@ -7,8 +7,8 @@ import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; import { TestGlobalConfigAccessor } from "../../../testing/globalConfig"; import { createRootHandler } from "../../index"; import type { GetTraceQuery, ListTracesQuery } from "../../../core/observability/index"; +import { resolveTraceOutputPath } from "../../observability/traceOutputPath"; import { formatTraceTable, formatTraceTimestamp } from "../../observability/traces"; -import { resolveTraceOutputPath } from "./outputPath"; const REGION = "us-west-2"; const SINCE_MS = 1_709_391_000_000; @@ -85,6 +85,16 @@ describe("runtime traces list", () => { expect((core.observability.calls[0]!.args[1] as ListTracesQuery).limit).toBe(20); }); + test("uses the requested endpoint qualifier", async () => { + const { core, route } = testTracesCommand(); + + await route(["runtime", "traces", "list", "--id", "rt-1", "--qualifier", "canary"]); + + expect(core.observability.calls[0]?.args[0]).toEqual({ + logGroupName: "/aws/bedrock-agentcore/runtimes/rt-1-canary", + }); + }); + test("--json renders a single JSON document", async () => { const { core, io, route } = testTracesCommand(); core.observability.traceSummaries = [{ traceId: "abc123", timestamp: "1709391000000" }]; diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index cca896254..cf3bdd18e 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -111,7 +111,11 @@ import type { StartRecommendationResponse, } from "@aws-sdk/client-bedrock-agentcore"; import type { Core } from "../handlers/types"; -import type { CoreHarnessClient, CreateHarnessInput } from "../handlers/harness/types"; +import type { + CoreHarnessClient, + CreateHarnessInput, + ResolvedHarnessRuntime, +} from "../handlers/harness/types"; import type { CoreGatewayClient, CreateGatewayInput, @@ -241,6 +245,10 @@ const DEFAULT_UPDATE_ENDPOINT_RESPONSE: UpdateHarnessEndpointResponse = {} as UpdateHarnessEndpointResponse; const DEFAULT_DELETE_ENDPOINT_RESPONSE: DeleteHarnessEndpointResponse = {} as DeleteHarnessEndpointResponse; +const DEFAULT_RESOLVED_HARNESS_RUNTIME: ResolvedHarnessRuntime = { + runtimeId: "harness_runtime-0000000000", + runtimeName: "harness_runtime", +}; const DEFAULT_CREATE_API_KEY_RESPONSE = {} as CreateApiKeyCredentialProviderResponse; const DEFAULT_GET_API_KEY_RESPONSE = {} as GetApiKeyCredentialProviderResponse; const DEFAULT_LIST_API_KEYS_RESPONSE: ListApiKeyCredentialProvidersResponse = { @@ -380,6 +388,7 @@ export class TestHarnessClient implements CoreHarnessClient { private createEndpointResponse: CreateHarnessEndpointResponse = DEFAULT_CREATE_ENDPOINT_RESPONSE; private updateEndpointResponse: UpdateHarnessEndpointResponse = DEFAULT_UPDATE_ENDPOINT_RESPONSE; private deleteEndpointResponse: DeleteHarnessEndpointResponse = DEFAULT_DELETE_ENDPOINT_RESPONSE; + private resolvedRuntime: ResolvedHarnessRuntime = DEFAULT_RESOLVED_HARNESS_RUNTIME; private error?: Error; // setListResponse sets what listHarnesses resolves to (when not erroring). @@ -396,6 +405,11 @@ export class TestHarnessClient implements CoreHarnessClient { return this; } + setResolvedRuntime(runtime: ResolvedHarnessRuntime): this { + this.resolvedRuntime = runtime; + return this; + } + // setGetVersionResponse sets what getHarnessVersion resolves to (when not // erroring). setGetVersionResponse(response: GetHarnessResponse): this { @@ -559,6 +573,16 @@ export class TestHarnessClient implements CoreHarnessClient { return this.getResponse; } + async resolveRuntime( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + this.calls.push({ method: "resolveRuntime", args: [id, options, signal] }); + if (this.error) throw this.error; + return this.resolvedRuntime; + } + async getHarnessVersion( id: string, version: string,