test(e2e): add end-to-end integration suite and e2e-tests workflow

.github/workflows/e2e-test.yml

@@ -0,0 +1,83 @@
+# End-to-end integration suite. Given a git reference and a test-path pattern,
+# builds the Node CLI at that reference and runs the matching e2e tests against a
+# real AWS account. Runs on pushes to refactor and is manually dispatchable
+# against an individual PR (or any ref) with a custom pattern; also reusable via
+# workflow_call. Reuses the agentcore-devx-devtools collaborator gate so only
+# authorized users can spend real AWS resources. Telemetry is disabled.
+name: e2e-test
+on:
+  push:
+    branches: [refactor]
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: git reference (commit, branch, or tag) to build and test
+        type: string
+        default: refactor
+      test_path:
+        description: filepath pattern selecting which e2e tests to run
+        type: string
+        default: test/project
+  workflow_call:
+    inputs:
+      ref:
+        required: true
+        type: string
+      test_path:
+        type: string
+        default: test/project
+
+concurrency:
+  # A run deploys to real AWS; cancelling mid-deploy would orphan resources.
+  group: e2e-test-${{ inputs.ref || github.ref }}
+  cancel-in-progress: false
+
+env:
+  AGENTCORE_TELEMETRY_DISABLED: "1"
+
+jobs:
+  # Only collaborators with write access may trigger a run — on a personal fork
+  # that is the owner alone. Reuses the shared devx-devtools gate.
+  authorize:
+    if: github.repository_owner == 'Hweinstock'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      is-authorized: ${{ steps.gate.outputs.is-authorized }}
+    steps:
+      - id: gate
+        uses: aws/agentcore-devx-devtools/.github/actions/check-collaborator@31aa3b031a86664e29861d68956e44b07cf21a74
+        with:
+          subject: ${{ github.actor }}
+          required-permission: write
+
+  e2e:
+    needs: authorize
+    if: needs.authorize.outputs.is-authorized == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          persist-credentials: false
+      - uses: oven-sh/setup-bun@v2
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - uses: astral-sh/setup-uv@v6
+      - run: bun install --frozen-lockfile
+      - run: bun run build
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.E2E_ROLE_ARN }}
+          aws-region: ${{ vars.E2E_REGION || 'us-east-1' }}
+      - name: Run e2e suite
+        env:
+          AWS_REGION: ${{ vars.E2E_REGION || 'us-east-1' }}
+          E2E_TEST_PATH: ${{ inputs.test_path || 'test/project' }}
+        run: bun run test:e2e

package.json

@@ -29,6 +29,7 @@
     "compile:windows-arm64": "bun scripts/build.ts compile bun-windows-arm64",
     "start": "bun run src/index.ts",
     "test": "bun test",
+    "test:e2e": "bun test --preload ./test/preRunCleanup.ts ${E2E_TEST_PATH:-test/project}",
     "typecheck": "tsc --noEmit",
     "lint": "oxlint --fix",
     "lint:check": "oxlint",

test/README.md

@@ -0,0 +1,88 @@
+# End-to-end integration tests
+
+This directory contains the end-to-end (e2e) integration tests for the `agentcore`
+CLI. Unlike the unit tests under `src/`, which exercise handlers and core logic in
+process, these tests treat the CLI as a black box: they build the CLI, run it as a
+real subprocess, and drive complete customer journeys — creating a project, adding
+resources, deploying to a real AWS account, and invoking the deployed resources.
+They assert only on what a customer can observe (process exit codes and printed
+output) and never reach into the CLI's internals.
+
+## What these tests cover
+
+Each feature has its own file, and each file corresponds to exactly one project.
+A single project deploys many resources together in one CloudFormation stack, so
+the slow, expensive deploy happens once per file instead of once per resource.
+
+- **`project/runtime.test.ts`** — one project containing a runtime for every
+  template the CLI ships, across both CodeZip and Container builds:
+  `agent-python`, `agent-python-strands` (zip and container),
+  `agent-typescript-strands`, `mcp-python-fastmcp`, `a2a-python-strands`, and
+  `agui-python-strands`. The first runtime is scaffolded by `project create`; the
+  rest are added with `project add runtime`. HTTP runtimes are invoked with a
+  prompt; MCP, A2A, AGUI, and TypeScript runtimes (whose data plane is not a plain
+  prompt) are verified by confirming they appear in the deploy output.
+
+- **`project/memory.test.ts`** — one project containing a strands runtime for each
+  memory configuration (`none`, `shortTerm`, `longAndShortTerm`). Every runtime is
+  invoked; the two memory-backed runtimes must recall a fact stated in an earlier
+  turn of the same session.
+
+- **`project/harness.test.ts`** — one project containing several harness
+  configurations (a default harness plus tuned variants). Every harness is invoked
+  with a prompt.
+
+Every test is a `test.each` row over a plain data table. Adding coverage — a new
+template, build type, memory option, or harness setting — means adding a row to the
+table at the top of the file, not writing a new test.
+
+## How the tests run the CLI
+
+`helpers/run.ts` exposes a single `run(args, cwd?)` function that spawns the built
+CLI (`node dist/index.js`) with the given arguments and returns its stdout, stderr,
+and exit code. `helpers/project.ts` wraps a throwaway temp-directory project: it
+scaffolds the project with `project create`, exposes a `run` bound to the project
+directory, and tears the project down afterward. The tests therefore read as the
+literal CLI commands a customer would type, for example:
+
+```ts
+project.run(["project", "invoke", "runtime", "--name", "pyzip", "--payload", payload, "--json"]);
+```
+
+`helpers/retry.ts` retries an invoke a few times, because a freshly deployed runtime
+can cold-start and reject the first request.
+
+## Running the tests
+
+The tests deploy to and invoke real AWS resources, so you need valid AWS
+credentials for a test account before running them:
+
+```sh
+ada credentials update --account <dev-account> --role Admin --once
+export AWS_REGION=us-east-1   # defaults to us-east-1 if unset
+
+bun run test:e2e
+```
+
+To run a subset, point `E2E_TEST_PATH` at a file or directory:
+
+```sh
+E2E_TEST_PATH=test/project/runtime.test.ts bun run test:e2e
+```
+
+`bun run test:e2e` loads `test/preRunCleanup.ts` as a preload. That preload runs
+once before any test file and deletes stale `AgentCore-e2e` CloudFormation stacks
+left behind by earlier runs that crashed before their teardown. Each test also
+tears its own project down (`project remove all` followed by `project deploy`) in
+an `afterAll` hook, so a normal run leaves nothing behind; the stale-stack sweep is
+the backstop for abnormal exits.
+
+## Continuous integration
+
+`.github/workflows/e2e-test.yml` runs this suite. Given a git reference and a
+filepath pattern, it builds the CLI at that reference and runs the matching tests.
+It triggers on pushes to `refactor`, can be dispatched manually against any
+reference (for example an individual PR) with a custom `test_path`, and is reusable
+from other workflows via `workflow_call`. It authorizes runs through the shared
+`agentcore-devx-devtools` collaborator check and assumes an AWS role via OIDC from
+the repository variable `E2E_ROLE_ARN`.

test/helpers/project.ts

@@ -0,0 +1,47 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { run, type RunResult } from "./run";
+
+export const uniqueName = (prefix: string): string =>
+  `${prefix}${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`
+    .replace(/[^a-z0-9]/gi, "")
+    .slice(0, 42);
+
+export function expectOk(result: RunResult): RunResult {
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `exited ${result.exitCode}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`,
+    );
+  }
+  return result;
+}
+
+export class Project {
+  private constructor(
+    readonly name: string,
+    readonly root: string,
+    readonly dir: string,
+  ) {}
+
+  static async create(name: string, createArgs: string[]): Promise<Project> {
+    const root = await mkdtemp(join(tmpdir(), "agentcore-e2e-"));
+    expectOk(await run(["project", "create", "--name", name, "--skip-git", ...createArgs], root));
+    return new Project(name, root, join(root, name));
+  }
+
+  run(args: string[]): Promise<RunResult> {
+    return run(args, this.dir);
+  }
+
+  async teardown(): Promise<void> {
+    try {
+      await this.run(["project", "remove", "all", "--yes"]);
+      await this.run(["project", "deploy", "--yes", "--json"]);
+    } catch {
+      // Best-effort; the pre-run stale-stack sweep is the backstop.
+    } finally {
+      await rm(this.root, { recursive: true, force: true });
+    }
+  }
+}

test/helpers/retry.ts

@@ -0,0 +1,12 @@
+export async function retry<T>(fn: () => Promise<T>, attempts = 3, delayMs = 15_000): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+  }
+  throw lastError;
+}

test/helpers/run.ts

@@ -0,0 +1,22 @@
+import { spawn } from "node:child_process";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const cli = join(resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."), "dist", "index.js");
+
+export type RunResult = { stdout: string; stderr: string; exitCode: number };
+
+export function run(args: string[], cwd?: string): Promise<RunResult> {
+  return new Promise((resolvePromise, reject) => {
+    const child = spawn("node", [cli, ...args], {
+      cwd,
+      env: { ...process.env, AGENTCORE_TELEMETRY_DISABLED: "1", FORCE_COLOR: "0" },
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => (stdout += chunk));
+    child.stderr.on("data", (chunk) => (stderr += chunk));
+    child.on("error", reject);
+    child.on("close", (exitCode) => resolvePromise({ stdout, stderr, exitCode: exitCode ?? -1 }));
+  });
+}

test/preRunCleanup.ts

@@ -0,0 +1,31 @@
+import {
+  CloudFormationClient,
+  DeleteStackCommand,
+  ListStacksCommand,
+} from "@aws-sdk/client-cloudformation";
+
+const region = process.env.AWS_REGION ?? "us-east-1";
+const stackPrefix = "AgentCore-e2e";
+const staleAgeMs = 2 * 60 * 60 * 1000;
+
+export async function cleanupStaleStacks(cfn: CloudFormationClient): Promise<void> {
+  let nextToken: string | undefined;
+  do {
+    const page = await cfn.send(new ListStacksCommand({ NextToken: nextToken }));
+    nextToken = page.NextToken;
+    for (const stack of page.StackSummaries ?? []) {
+      const status = stack.StackStatus ?? "";
+      const age = Date.now() - (stack.CreationTime?.getTime() ?? Date.now());
+      if (stack.ParentId || !stack.StackName?.startsWith(stackPrefix)) continue;
+      if (status === "DELETE_COMPLETE" || status.endsWith("_IN_PROGRESS") || age < staleAgeMs)
+        continue;
+      try {
+        await cfn.send(new DeleteStackCommand({ StackName: stack.StackName }));
+      } catch {
+        // Leave it for the next run rather than failing this one.
+      }
+    }
+  } while (nextToken);
+}
+
+await cleanupStaleStacks(new CloudFormationClient({ region }));

test/project/harness.test.ts

@@ -0,0 +1,45 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { expectOk, Project, uniqueName } from "../helpers/project";
+import { retry } from "../helpers/retry";
+
+const PROMPT = "Reply with a short greeting.";
+const DEPLOY_TIMEOUT_MS = 40 * 60 * 1000;
+const INVOKE_TIMEOUT_MS = 15 * 60 * 1000;
+
+const projectName = uniqueName("e2ehn");
+
+const ADDED: [name: string, flags: string[]][] = [
+  ["added", []],
+  ["tuned", ["--max-iterations", "5"]],
+];
+
+const HARNESSES = [projectName, ...ADDED.map(([name]) => name)];
+
+describe("e2e: project harness configurations", () => {
+  let project: Project;
+
+  beforeAll(async () => {
+    project = await Project.create(projectName, []);
+    for (const [name, flags] of ADDED) {
+      expectOk(await project.run(["project", "add", "harness", "--name", name, ...flags]));
+    }
+    expectOk(await project.run(["project", "deploy", "--yes", "--json"]));
+  }, DEPLOY_TIMEOUT_MS);
+
+  afterAll(async () => {
+    await project?.teardown();
+  }, DEPLOY_TIMEOUT_MS);
+
+  test.each(HARNESSES)(
+    "%s",
+    async (name) => {
+      const result = await retry(() =>
+        project
+          .run(["project", "invoke", "harness", "--name", name, "--prompt", PROMPT, "--json"])
+          .then(expectOk),
+      );
+      expect(result.stdout.length).toBeGreaterThan(0);
+    },
+    INVOKE_TIMEOUT_MS,
+  );
+});