diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml new file mode 100644 index 000000000..72e8647da --- /dev/null +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1,83 @@ +# End-to-end integration suite. Given a git reference and a test-path pattern, +# builds the Node CLI at that reference and runs the matching e2e tests against a +# real AWS account. Runs on pushes to refactor and is manually dispatchable +# against an individual PR (or any ref) with a custom pattern; also reusable via +# workflow_call. Reuses the agentcore-devx-devtools collaborator gate so only +# authorized users can spend real AWS resources. Telemetry is disabled. +name: e2e-test +on: + push: + branches: [refactor] + workflow_dispatch: + inputs: + ref: + description: git reference (commit, branch, or tag) to build and test + type: string + default: refactor + test_path: + description: filepath pattern selecting which e2e tests to run + type: string + default: test/project + workflow_call: + inputs: + ref: + required: true + type: string + test_path: + type: string + default: test/project + +concurrency: + # A run deploys to real AWS; cancelling mid-deploy would orphan resources. + group: e2e-test-${{ inputs.ref || github.ref }} + cancel-in-progress: false + +env: + AGENTCORE_TELEMETRY_DISABLED: "1" + +jobs: + # Only collaborators with write access may trigger a run — on a personal fork + # that is the owner alone. Reuses the shared devx-devtools gate. + authorize: + if: github.repository_owner == 'Hweinstock' + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + is-authorized: ${{ steps.gate.outputs.is-authorized }} + steps: + - id: gate + uses: aws/agentcore-devx-devtools/.github/actions/check-collaborator@31aa3b031a86664e29861d68956e44b07cf21a74 + with: + subject: ${{ github.actor }} + required-permission: write + + e2e: + needs: authorize + if: needs.authorize.outputs.is-authorized == 'true' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + - uses: actions/setup-node@v4 + with: + node-version: "20" + - uses: astral-sh/setup-uv@v6 + - run: bun install --frozen-lockfile + - run: bun run build + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.E2E_ROLE_ARN }} + aws-region: ${{ vars.E2E_REGION || 'us-east-1' }} + - name: Run e2e suite + env: + AWS_REGION: ${{ vars.E2E_REGION || 'us-east-1' }} + E2E_TEST_PATH: ${{ inputs.test_path || 'test/project' }} + run: bun run test:e2e diff --git a/package.json b/package.json index 318b68886..e67a688fb 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "compile:windows-arm64": "bun scripts/build.ts compile bun-windows-arm64", "start": "bun run src/index.ts", "test": "bun test", + "test:e2e": "bun test --preload ./test/preRunCleanup.ts ${E2E_TEST_PATH:-test/project}", "typecheck": "tsc --noEmit", "lint": "oxlint --fix", "lint:check": "oxlint", diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..d13799a69 --- /dev/null +++ b/test/README.md @@ -0,0 +1,88 @@ +# End-to-end integration tests + +This directory contains the end-to-end (e2e) integration tests for the `agentcore` +CLI. Unlike the unit tests under `src/`, which exercise handlers and core logic in +process, these tests treat the CLI as a black box: they build the CLI, run it as a +real subprocess, and drive complete customer journeys — creating a project, adding +resources, deploying to a real AWS account, and invoking the deployed resources. +They assert only on what a customer can observe (process exit codes and printed +output) and never reach into the CLI's internals. + +## What these tests cover + +Each feature has its own file, and each file corresponds to exactly one project. +A single project deploys many resources together in one CloudFormation stack, so +the slow, expensive deploy happens once per file instead of once per resource. + +- **`project/runtime.test.ts`** — one project containing a runtime for every + template the CLI ships, across both CodeZip and Container builds: + `agent-python`, `agent-python-strands` (zip and container), + `agent-typescript-strands`, `mcp-python-fastmcp`, `a2a-python-strands`, and + `agui-python-strands`. The first runtime is scaffolded by `project create`; the + rest are added with `project add runtime`. HTTP runtimes are invoked with a + prompt; MCP, A2A, AGUI, and TypeScript runtimes (whose data plane is not a plain + prompt) are verified by confirming they appear in the deploy output. + +- **`project/memory.test.ts`** — one project containing a strands runtime for each + memory configuration (`none`, `shortTerm`, `longAndShortTerm`). Every runtime is + invoked; the two memory-backed runtimes must recall a fact stated in an earlier + turn of the same session. + +- **`project/harness.test.ts`** — one project containing several harness + configurations (a default harness plus tuned variants). Every harness is invoked + with a prompt. + +Every test is a `test.each` row over a plain data table. Adding coverage — a new +template, build type, memory option, or harness setting — means adding a row to the +table at the top of the file, not writing a new test. + +## How the tests run the CLI + +`helpers/run.ts` exposes a single `run(args, cwd?)` function that spawns the built +CLI (`node dist/index.js`) with the given arguments and returns its stdout, stderr, +and exit code. `helpers/project.ts` wraps a throwaway temp-directory project: it +scaffolds the project with `project create`, exposes a `run` bound to the project +directory, and tears the project down afterward. The tests therefore read as the +literal CLI commands a customer would type, for example: + +```ts +project.run(["project", "invoke", "runtime", "--name", "pyzip", "--payload", payload, "--json"]); +``` + +`helpers/retry.ts` retries an invoke a few times, because a freshly deployed runtime +can cold-start and reject the first request. + +## Running the tests + +The tests deploy to and invoke real AWS resources, so you need valid AWS +credentials for a test account before running them: + +```sh +ada credentials update --account --role Admin --once +export AWS_REGION=us-east-1 # defaults to us-east-1 if unset + +bun run test:e2e +``` + +To run a subset, point `E2E_TEST_PATH` at a file or directory: + +```sh +E2E_TEST_PATH=test/project/runtime.test.ts bun run test:e2e +``` + +`bun run test:e2e` loads `test/preRunCleanup.ts` as a preload. That preload runs +once before any test file and deletes stale `AgentCore-e2e` CloudFormation stacks +left behind by earlier runs that crashed before their teardown. Each test also +tears its own project down (`project remove all` followed by `project deploy`) in +an `afterAll` hook, so a normal run leaves nothing behind; the stale-stack sweep is +the backstop for abnormal exits. + +## Continuous integration + +`.github/workflows/e2e-test.yml` runs this suite. Given a git reference and a +filepath pattern, it builds the CLI at that reference and runs the matching tests. +It triggers on pushes to `refactor`, can be dispatched manually against any +reference (for example an individual PR) with a custom `test_path`, and is reusable +from other workflows via `workflow_call`. It authorizes runs through the shared +`agentcore-devx-devtools` collaborator check and assumes an AWS role via OIDC from +the repository variable `E2E_ROLE_ARN`. diff --git a/test/helpers/project.ts b/test/helpers/project.ts new file mode 100644 index 000000000..848e2d2de --- /dev/null +++ b/test/helpers/project.ts @@ -0,0 +1,47 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { run, type RunResult } from "./run"; + +export const uniqueName = (prefix: string): string => + `${prefix}${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}` + .replace(/[^a-z0-9]/gi, "") + .slice(0, 42); + +export function expectOk(result: RunResult): RunResult { + if (result.exitCode !== 0) { + throw new Error( + `exited ${result.exitCode}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, + ); + } + return result; +} + +export class Project { + private constructor( + readonly name: string, + readonly root: string, + readonly dir: string, + ) {} + + static async create(name: string, createArgs: string[]): Promise { + const root = await mkdtemp(join(tmpdir(), "agentcore-e2e-")); + expectOk(await run(["project", "create", "--name", name, "--skip-git", ...createArgs], root)); + return new Project(name, root, join(root, name)); + } + + run(args: string[]): Promise { + return run(args, this.dir); + } + + async teardown(): Promise { + try { + await this.run(["project", "remove", "all", "--yes"]); + await this.run(["project", "deploy", "--yes", "--json"]); + } catch { + // Best-effort; the pre-run stale-stack sweep is the backstop. + } finally { + await rm(this.root, { recursive: true, force: true }); + } + } +} diff --git a/test/helpers/retry.ts b/test/helpers/retry.ts new file mode 100644 index 000000000..f54a9f482 --- /dev/null +++ b/test/helpers/retry.ts @@ -0,0 +1,12 @@ +export async function retry(fn: () => Promise, attempts = 3, delayMs = 15_000): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error; + if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + throw lastError; +} diff --git a/test/helpers/run.ts b/test/helpers/run.ts new file mode 100644 index 000000000..f4133c5cb --- /dev/null +++ b/test/helpers/run.ts @@ -0,0 +1,22 @@ +import { spawn } from "node:child_process"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const cli = join(resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."), "dist", "index.js"); + +export type RunResult = { stdout: string; stderr: string; exitCode: number }; + +export function run(args: string[], cwd?: string): Promise { + return new Promise((resolvePromise, reject) => { + const child = spawn("node", [cli, ...args], { + cwd, + env: { ...process.env, AGENTCORE_TELEMETRY_DISABLED: "1", FORCE_COLOR: "0" }, + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (exitCode) => resolvePromise({ stdout, stderr, exitCode: exitCode ?? -1 })); + }); +} diff --git a/test/preRunCleanup.ts b/test/preRunCleanup.ts new file mode 100644 index 000000000..7666fae0b --- /dev/null +++ b/test/preRunCleanup.ts @@ -0,0 +1,31 @@ +import { + CloudFormationClient, + DeleteStackCommand, + ListStacksCommand, +} from "@aws-sdk/client-cloudformation"; + +const region = process.env.AWS_REGION ?? "us-east-1"; +const stackPrefix = "AgentCore-e2e"; +const staleAgeMs = 2 * 60 * 60 * 1000; + +export async function cleanupStaleStacks(cfn: CloudFormationClient): Promise { + let nextToken: string | undefined; + do { + const page = await cfn.send(new ListStacksCommand({ NextToken: nextToken })); + nextToken = page.NextToken; + for (const stack of page.StackSummaries ?? []) { + const status = stack.StackStatus ?? ""; + const age = Date.now() - (stack.CreationTime?.getTime() ?? Date.now()); + if (stack.ParentId || !stack.StackName?.startsWith(stackPrefix)) continue; + if (status === "DELETE_COMPLETE" || status.endsWith("_IN_PROGRESS") || age < staleAgeMs) + continue; + try { + await cfn.send(new DeleteStackCommand({ StackName: stack.StackName })); + } catch { + // Leave it for the next run rather than failing this one. + } + } + } while (nextToken); +} + +await cleanupStaleStacks(new CloudFormationClient({ region })); diff --git a/test/project/harness.test.ts b/test/project/harness.test.ts new file mode 100644 index 000000000..0594650a2 --- /dev/null +++ b/test/project/harness.test.ts @@ -0,0 +1,45 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { expectOk, Project, uniqueName } from "../helpers/project"; +import { retry } from "../helpers/retry"; + +const PROMPT = "Reply with a short greeting."; +const DEPLOY_TIMEOUT_MS = 40 * 60 * 1000; +const INVOKE_TIMEOUT_MS = 15 * 60 * 1000; + +const projectName = uniqueName("e2ehn"); + +const ADDED: [name: string, flags: string[]][] = [ + ["added", []], + ["tuned", ["--max-iterations", "5"]], +]; + +const HARNESSES = [projectName, ...ADDED.map(([name]) => name)]; + +describe("e2e: project harness configurations", () => { + let project: Project; + + beforeAll(async () => { + project = await Project.create(projectName, []); + for (const [name, flags] of ADDED) { + expectOk(await project.run(["project", "add", "harness", "--name", name, ...flags])); + } + expectOk(await project.run(["project", "deploy", "--yes", "--json"])); + }, DEPLOY_TIMEOUT_MS); + + afterAll(async () => { + await project?.teardown(); + }, DEPLOY_TIMEOUT_MS); + + test.each(HARNESSES)( + "%s", + async (name) => { + const result = await retry(() => + project + .run(["project", "invoke", "harness", "--name", name, "--prompt", PROMPT, "--json"]) + .then(expectOk), + ); + expect(result.stdout.length).toBeGreaterThan(0); + }, + INVOKE_TIMEOUT_MS, + ); +}); diff --git a/test/project/memory.test.ts b/test/project/memory.test.ts new file mode 100644 index 000000000..17ada7507 --- /dev/null +++ b/test/project/memory.test.ts @@ -0,0 +1,79 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { expectOk, Project, uniqueName } from "../helpers/project"; +import { retry } from "../helpers/retry"; + +const REMEMBER = JSON.stringify({ prompt: "Remember that my favorite color is teal." }); +const RECALL = JSON.stringify({ prompt: "What is my favorite color? Answer with one word." }); +const DEPLOY_TIMEOUT_MS = 40 * 60 * 1000; +const INVOKE_TIMEOUT_MS = 15 * 60 * 1000; + +const MEMORIES: [name: string, template: string, recalls: boolean][] = [ + ["agent_python_minimal", "agent-python-minimal", false], + ["agent_python_strands", "agent-python-strands", true], +]; + +const session = (name: string): string => + `e2ememory${name}${Date.now().toString(36)}`.padEnd(40, "x").slice(0, 60); + +describe("e2e: project runtime memory configurations", () => { + let project: Project; + + beforeAll(async () => { + const [, template] = MEMORIES[0]!; + project = await Project.create(uniqueName("e2emem"), ["--template", template]); + for (const [name, tmpl] of MEMORIES.slice(1)) { + expectOk( + await project.run(["project", "add", "runtime", "--name", name, "--template", tmpl]), + ); + } + expectOk(await project.run(["project", "deploy", "--yes", "--json"])); + }, DEPLOY_TIMEOUT_MS); + + afterAll(async () => { + await project?.teardown(); + }, DEPLOY_TIMEOUT_MS); + + test.each(MEMORIES)( + "%s", + async (name, _template, recalls) => { + const sessionId = session(name); + const remembered = await retry(() => + project + .run([ + "project", + "invoke", + "runtime", + "--name", + name, + "--session-id", + sessionId, + "--payload", + REMEMBER, + "--json", + ]) + .then(expectOk), + ); + expect(remembered.stdout.length).toBeGreaterThan(0); + if (recalls) { + const recalled = await retry(() => + project + .run([ + "project", + "invoke", + "runtime", + "--name", + name, + "--session-id", + sessionId, + "--payload", + RECALL, + "--json", + ]) + .then(expectOk), + ); + expect(recalled.stdout.toLowerCase()).toContain("teal"); + } + }, + 2 * INVOKE_TIMEOUT_MS, + ); +}); diff --git a/test/project/runtime.test.ts b/test/project/runtime.test.ts new file mode 100644 index 000000000..4d8379d50 --- /dev/null +++ b/test/project/runtime.test.ts @@ -0,0 +1,62 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { expectOk, Project, uniqueName } from "../helpers/project"; +import { retry } from "../helpers/retry"; +import type { RunResult } from "../helpers/run"; + +const PROMPT = JSON.stringify({ prompt: "Reply with a short greeting." }); +const DEPLOY_TIMEOUT_MS = 40 * 60 * 1000; +const INVOKE_TIMEOUT_MS = 15 * 60 * 1000; + +// The first runtime is created by `project create`, which names it after the +// template (agent_python_minimal). The rest are added with short names: memory +// resources embed `_` in strategy names capped at 48 chars, +// and a name must not equal a template dependency (e.g. mcp, langchain) or uv +// treats the project as depending on itself. +const RUNTIMES: [name: string, template: string, check: "invoke" | "deployed"][] = [ + ["agent_python_minimal", "agent-python-minimal", "invoke"], + ["strandsa", "agent-python-strands", "invoke"], + ["strandsc", "agent-python-strands-container", "invoke"], + ["lcagent", "agent-python-langchain", "invoke"], + ["tsstrands", "agent-typescript-strands", "deployed"], + ["vercel", "agent-typescript-vercel", "deployed"], + ["mcpfast", "mcp-python-fastmcp", "deployed"], + ["a2aagent", "a2a-python-strands", "deployed"], + ["aguiagent", "agui-python-strands", "deployed"], +]; + +describe("e2e: project runtime configurations", () => { + let project: Project; + let deployment: RunResult; + + beforeAll(async () => { + const [, template] = RUNTIMES[0]!; + project = await Project.create(uniqueName("e2ert"), ["--template", template]); + for (const [name, tmpl] of RUNTIMES.slice(1)) { + expectOk( + await project.run(["project", "add", "runtime", "--name", name, "--template", tmpl]), + ); + } + deployment = expectOk(await project.run(["project", "deploy", "--yes", "--json"])); + }, DEPLOY_TIMEOUT_MS); + + afterAll(async () => { + await project?.teardown(); + }, DEPLOY_TIMEOUT_MS); + + test.each(RUNTIMES)( + "%s", + async (name, _template, check) => { + if (check === "invoke") { + const result = await retry(() => + project + .run(["project", "invoke", "runtime", "--name", name, "--payload", PROMPT, "--json"]) + .then(expectOk), + ); + expect(result.stdout.length).toBeGreaterThan(0); + } else { + expect(deployment.stdout.toLowerCase()).toContain(name.toLowerCase()); + } + }, + INVOKE_TIMEOUT_MS, + ); +});