diff --git a/.changeset/local-artifacts-binding.md b/.changeset/local-artifacts-binding.md new file mode 100644 index 00000000000..546c6af0120 --- /dev/null +++ b/.changeset/local-artifacts-binding.md @@ -0,0 +1,9 @@ +--- +"miniflare": minor +"wrangler": minor +"@cloudflare/workers-utils": minor +--- + +Add a local Artifacts binding simulator backed by native Git for Worker development + +Artifacts bindings now use local, persistent repositories by default in development. Set `remote: true` on the binding to opt into the remote resource. The simulator supports the binding RPC methods and authenticated Git smart HTTP without requiring Cloudflare credentials for local use. diff --git a/packages/miniflare/scripts/build.mjs b/packages/miniflare/scripts/build.mjs index 56ba5b35265..516f094f304 100644 --- a/packages/miniflare/scripts/build.mjs +++ b/packages/miniflare/scripts/build.mjs @@ -446,7 +446,13 @@ async function buildPackage() { logLevel: watch ? "info" : "warning", outdir: outPath, outbase: pkgRoot, - entryPoints: [indexPath, ...fixtureBuilds], + entryPoints: [ + indexPath, + // A separate thread keeps native Git available during synchronous Node + // binding-proxy calls, which block Miniflare's main event loop. + path.join(pkgRoot, "src/plugins/artifacts/git-sidecar.ts"), + ...fixtureBuilds, + ], }; if (watch) { diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 1284d267928..ef661dfc487 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -77,6 +77,7 @@ import { WORKER_BINDING_SERVICE_LOOPBACK, WORKFLOWS_PLUGIN_NAME, } from "./plugins"; +import { ArtifactsController } from "./plugins/artifacts/controller"; import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants"; import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version"; import { closeBrowserProcess } from "./plugins/browser-rendering/process"; @@ -888,6 +889,7 @@ export class Miniflare { #maybeInspectorProxyController?: InspectorProxyController; #previousRuntimeInspectorPort?: number; + #artifactsController = new ArtifactsController(); #containerPrivilegesCache = new ContainerPrivilegesCache(); #hyperdriveProxyController: HyperdriveProxyController = new HyperdriveProxyController(); @@ -2326,6 +2328,7 @@ export class Miniflare { queueConsumers, containerPrivilegesCache: this.#containerPrivilegesCache, hyperdriveProxyController: this.#hyperdriveProxyController, + artifactsController: this.#artifactsController, }; for (const [key, plugin] of this.#mergedPluginEntries) { const pluginServicesExtensions = await plugin.getServices({ @@ -2341,6 +2344,11 @@ export class Miniflare { } else { pluginServices = pluginServicesExtensions.services; extensions.push(...pluginServicesExtensions.extensions); + for (const socket of pluginServicesExtensions.sockets ?? []) { + if (!sockets.some((existing) => existing.name === socket.name)) { + sockets.push(socket); + } + } } for (const service of pluginServices) { @@ -2672,11 +2680,13 @@ export class Miniflare { async #assembleAndUpdateConfig(reusePorts = false) { this.#hyperdriveProxyController.beginUpdate(); + this.#artifactsController.beginUpdate(); try { await this.#assembleAndUpdateConfigInternal(reusePorts); } finally { - // If assembly or runtime startup failed, keep the old listeners and - // close only the new candidates. A successful update already committed. + // A failed update retains listeners used by the previous runtime and + // closes only candidates started during this attempt. + await this.#artifactsController.abortUpdate(); this.#hyperdriveProxyController.abortUpdate(); } } @@ -2800,6 +2810,7 @@ export class Miniflare { } } this.#hyperdriveProxyController.commitUpdate(activeExternalAddresses); + await this.#artifactsController.commitUpdate(); // Note: `updateConfig()` doesn't resolve until ports for all required // sockets have been recorded. At this point, `maybeSocketPorts` contains // all of `requiredSockets` as keys. @@ -3915,6 +3926,14 @@ export class Miniflare { } const runtimeCleanupOutcome = await runtimeDisposeOutcome; + try { + await this.#artifactsController.dispose(); + } catch (error) { + if (!independentCleanupFailed) { + independentCleanupFailed = true; + independentCleanupError = error; + } + } this.#devRegistry.unregisterWorkers(); try { await Promise.all( diff --git a/packages/miniflare/src/plugins/artifacts/LOCAL_DEVELOPMENT.md b/packages/miniflare/src/plugins/artifacts/LOCAL_DEVELOPMENT.md new file mode 100644 index 00000000000..9e244f75d6d --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/LOCAL_DEVELOPMENT.md @@ -0,0 +1,75 @@ +# Develop with local Artifacts + +## Prerequisite: Git + +This first version of the local Artifacts emulator requires Git installed globally on your development machine, with `git` available on the PATH of the process starting Wrangler, Vite, or Workers Vitest. Run `git --version` in that environment before starting the dev server or tests. If Git is missing or cannot run, local Artifacts fails at startup with installation instructions instead of waiting for a repository operation. Restart your dev tool after installing Git so it picks up the updated PATH. + +A binding with `"remote": true` does not start the local emulator and does not need local Git. Production Artifacts also does not require Git on your machine. We plan to open-source a JavaScript Git engine in the future; it is not part of this initial local release, which uses native Git. + +## Configure a binding + +For a Wrangler project, add an Artifacts binding to `wrangler.jsonc`: + +```jsonc +{ + "name": "example-worker", + "main": "src/index.ts", + "compatibility_date": "2026-09-03", + "artifacts": [{ "binding": "REPOS", "namespace": "examples" }], +} +``` + +To try this unreleased feature from the Workers SDK checkout, install its dependencies and build Miniflare and Wrangler, then start the checkout's CLI from the app directory: + +```sh +# From the workers-sdk root: +pnpm install --frozen-lockfile +pnpm --filter miniflare build +pnpm --filter wrangler build + +# From your app directory, using the absolute path to this checkout: +node /path/to/workers-sdk/packages/wrangler/bin/wrangler.js dev --port=0 --inspector-port=0 +``` + +Open the URL on Wrangler's `Ready on` line. With `@cloudflare/vite-plugin`, use the same Wrangler config with `cloudflare()` in your Vite config and start the Vite dev server. In Workers Vitest, point `cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" } })` at this config. These paths share the Miniflare implementation. To force local behavior explicitly, add `"remote": false` to the binding. The absence of `remote` is also local; this differs from older development behavior that implicitly used a remote Artifacts resource. Do not rely on production repositories being available locally. + +A Worker can create and inspect a local repository: + +```ts +const created = await env.REPOS.create("demo"); +const repo = await env.REPOS.get("demo"); +const listing = await env.REPOS.list(); +const info = await repo.info(); +// created.remote is a loopback HTTP Git URL; created.token is a secret. +``` + +To use Git smart HTTP, use the returned `remote` URL and a repository token. If `git clone` prompts for a username and password, enter any username (for example, `git`) and use `created.token` as the password. This is an Artifacts repository token, **not** a GitHub or Cloudflare account password. A configured Git credential helper may remember a password entered at the prompt. To avoid that, supply a bearer token for one Git command without storing it in source or in a Git remote URL: + +```sh +read -s REPO_TOKEN # Paste the token returned by create(); input is hidden. +printf '\n' +# Replace the URL with the `remote` value returned by create() or info(). +GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader \ + GIT_CONFIG_VALUE_0="Authorization: Bearer $REPO_TOKEN" \ + git clone 'http://127.0.0.1:/git/examples/demo.git' +unset REPO_TOKEN +``` + +After pushing `hello.txt` to `main`, read it in the Worker with `await repo.readFile({ ref: "main", path: "hello.txt" })` (a `Blob`, or `null` when absent). The initial create token permits writes. Its plaintext is returned only once; if you lose it, use `repo.createToken("read")` for cloning or `repo.createToken("write")` for pushing and save the returned `plaintext` securely. Revoke tokens that are no longer needed. Avoid printing tokens or committing them to config. A plain request without a token cannot clone a private local repository. Git's URL uses an ephemeral loopback port; obtain the current URL after restarting the dev server instead of hardcoding it. + +## Storage and network boundaries + +Local repositories and metadata persist under the dev tool's Miniflare resource persistence path (for example, Wrangler's `.wrangler/state/` when persistence is enabled). Use the same project root and persistence settings to keep local data across reloads and restarts. Disabling persistence (for example, Vite's `persistState: false`) makes the data temporary. Local and deployed Artifacts are separate: local create, push, and delete operations do not update remote resources. To opt in to a remote binding, set `"remote": true` on that Artifacts binding and configure the credentials required by your development tool. Remote mode uses a Cloudflare proxy, not the local Git store; do not assume local data migrates to it. + +To reset local state, stop the dev server first, back up any repositories you need, then delete the `artifacts/` directory inside that dev tool's resource persistence path. Older draft persisted Artifacts layouts are not automatically migrated and may require this backup-and-reset procedure. Do not delete unrelated binding data in the same persistence directory. + +A local binding does not contact Cloudflare for repository operations. **Explicit imports from HTTPS Git sources are different:** they make outbound network requests from your development machine to the supplied host, including private-network hosts. Import does not follow HTTP redirects; supply the final HTTPS Git URL so credentials cannot be redirected to another destination. Imports can require authentication and may fail due to TLS certificates, connectivity, or upstream Git permissions. For a private Git CA, set `GIT_SSL_CAINFO` to the trusted CA file in the dev tool's host environment before starting the dev server. Do not disable certificate verification. After a successful import, URL credentials are omitted from the local repository metadata and Git config. In-flight process-crash recovery is not yet validated; treat partial local storage after an interrupted import as sensitive if the URL contained credentials. Treat the import URL as a secret while calling `import()` and avoid logging it. Avoid importing untrusted or private URLs in examples or tests. No remote resources are contacted by the local-only examples above. + +## Troubleshooting and limits + +- If startup says local Artifacts requires Git, install Git globally, check `git --version` in the dev tool's environment, and restart the tool with `git` on PATH. +- If startup rejects an older storage layout, back it up and reset only the Artifacts persistence directory; a restart alone will not convert it. +- If Git asks for a username and password, use any username and the repository token as the password. If Git reports unauthorized, check the repository, scope, expiry, and revocation; do not use a Cloudflare or GitHub account password. +- If startup reports `Address already in use`, the dev or inspector port may be occupied. Pass `--port=0 --inspector-port=0` and use the printed `Ready on` URL. +- If a Git URL stops responding after restart, get the new `remote` from the binding; the loopback listener is tied to the dev process. +- Local development is not a replica of production placement or infrastructure. The local Git endpoint is HTTP on loopback only, and local state and tokens do not transfer to a remote namespace. Miniflare Artifacts, Wrangler dev, and Vite local fixtures have passed Windows CI; Workers Vitest's Artifacts fixture has not been verified on Windows. diff --git a/packages/miniflare/src/plugins/artifacts/README.md b/packages/miniflare/src/plugins/artifacts/README.md new file mode 100644 index 00000000000..c31c600cacc --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/README.md @@ -0,0 +1,148 @@ +# Artifacts in Miniflare + +For setup, persistence, Git authentication, and local/remote behavior, see +[Local development](./LOCAL_DEVELOPMENT.md). + +The Artifacts plugin implements the local Artifacts binding in workerd. This +first version requires Git installed globally on the host and available on +PATH; Miniflare checks at local startup and gives installation instructions if +Git is missing. A Node worker thread runs the native Git backend. Unless +`dev.remote` is explicitly `true`, an Artifacts binding uses local storage +rather than contacting Cloudflare. We plan to open-source a JavaScript Git +engine in the future; it is not included in this version. + +## Local startup + +```mermaid +sequenceDiagram + participant Dev as Developer tool + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Controller as ArtifactsController + participant Sidecar as Node Git sidecar + participant Workerd as workerd + + Dev->>MF: Configure Artifacts binding + MF->>Plugin: getBindings() + Plugin->>Plugin: Select local and validate namespace + Plugin-->>MF: Bind to local namespace entrypoint + MF->>Plugin: getServices() + Plugin->>Plugin: Verify host Git is available on PATH + Plugin->>Controller: get(Git storage path) + Controller->>Sidecar: Start worker thread and private HTTP listener + Sidecar-->>Controller: Address and secret + Controller-->>Plugin: Sidecar endpoint and Git socket port + Plugin-->>MF: Binding Worker, DO storage, backend service, Git socket + MF->>Workerd: Start configured services and socket +``` + +`getBindings()` routes each local binding to a namespace-scoped service. +`getServices()` creates the binding Worker, namespace Durable Object, persistent +metadata disk, internal Git backend service, and Git socket. The controller +reuses a sidecar per Git storage path and closes it when Miniflare is disposed. +The socket used by Git clients and the sidecar's private HTTP listener are +**different endpoints**. + +## Local binding RPC + +For example, `env.REPOS.create("demo")`: + +```mermaid +sequenceDiagram + participant App as User Worker + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Disk as Local disk + + App->>Entry: env.REPOS.create("demo") + Entry->>DO: RPC create("demo") + DO->>DO: Validate and serialize mutation + DO->>Backend: POST internal create request + Backend->>Sidecar: Inject private backend header + Sidecar->>Disk: git init --bare + Sidecar-->>DO: Git state (via backend service) + DO->>Disk: Persist metadata and token hash + DO-->>Entry: Repository info and initial token + Entry-->>App: Create result +``` + +The Durable Object stores repository metadata and token hashes on local disk; +the sidecar stores the bare Git repositories separately. `git-sidecar.ts` +handles HTTP routing and repository locks, while `git-client.ts` owns +repository-scoped Git commands and result parsing. Read operations such as +`readFile()` also use the binding Worker and native Git backend. Node callers +reach the same binding through Miniflare's Node binding proxy. Namespace and +repository names are hashed into short, namespace-scoped on-disk paths; this +leaves room for Git packfiles and workerd SQLite files on platforms with path +length limits. Older draft storage layouts are not migrated automatically: +startup rejects a detected old layout rather than silently creating empty +repositories. Back up any local repositories before resetting the Artifacts +persistence directory. Do not delete the persistence directory without first +checking whether it holds work you need. + +## Git clone and push + +```mermaid +sequenceDiagram + participant Git as Git client + participant Socket as Git socket + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Native as Native Git + + Git->>Socket: Git smart HTTP at /git/:namespace/:repo.git + Socket->>Entry: Route request + Entry->>DO: fetch(request) + DO->>DO: Check endpoint, repository, token, and scope + DO->>Backend: Forward request without Authorization header + Backend->>Sidecar: Inject private backend header + Sidecar->>Native: Stream via git http-backend + Native-->>Sidecar: Discovery or pack response + Sidecar-->>DO: Git response (via backend service) + opt Successful receive-pack push + DO->>Backend: Read updated refs + Backend->>Sidecar: Internal refs request + Sidecar-->>DO: Refs (via backend service) + DO->>DO: Persist updated push metadata + end + DO-->>Entry: Git response + Entry-->>Socket: Git response + Socket-->>Git: Git response +``` + +The binding Worker checks the supported Git endpoints and authenticates +repository tokens before forwarding. The internal service injects the sidecar +secret; the client's `Authorization` header is removed before that hop. For a +successful push, the Durable Object refreshes repository metadata before +returning the response. The sidecar uses native `git http-backend` for Git smart +HTTP rather than reproducing Git's pack protocol. + +## Explicit remote binding + +When `dev.remote` is `true`, this binding uses the remote-bindings proxy instead +of creating local Artifacts storage or a local Git sidecar for that namespace. + +```mermaid +sequenceDiagram + participant App as User Worker + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Proxy as Remote-bindings proxy + participant CF as Cloudflare Artifacts + + MF->>Plugin: getBindings() with remote: true + Plugin-->>MF: Bind to remote proxy service + MF->>Proxy: Start remote proxy Worker + App->>Proxy: env.REPOS method call + Proxy->>CF: Remote binding request + CF-->>Proxy: Result + Proxy-->>App: Result +``` + +A local binding does not automatically synchronize with production Artifacts. +An explicit repository import may contact its specified HTTPS Git remote; that +is distinct from enabling a remote Artifacts binding. diff --git a/packages/miniflare/src/plugins/artifacts/controller.ts b/packages/miniflare/src/plugins/artifacts/controller.ts new file mode 100644 index 00000000000..1a69edefc9e --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/controller.ts @@ -0,0 +1,150 @@ +import { once } from "node:events"; +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import getPort from "get-port"; +import type { GitSidecar } from "./git-sidecar"; + +export interface LocalArtifactsBackend { + sidecar: GitSidecar; + port: number; +} + +// Node binding proxies make synchronous calls. The Git backend must run on +// another thread, otherwise those calls prevent it from answering workerd. +export class ArtifactsController { + #backends = new Map>(); + // Keep the old roots alive until workerd accepts the replacement config. + #activeRoots = new Set(); + #pendingRoots?: Set; + + beginUpdate(): void { + if (this.#pendingRoots) { + throw new Error("Artifacts runtime update already in progress"); + } + this.#pendingRoots = new Set(); + } + + async commitUpdate(): Promise { + const roots = this.#pendingRoots; + if (!roots) { + throw new Error("No Artifacts runtime update to commit"); + } + this.#pendingRoots = undefined; + this.#activeRoots = roots; + await this.#closeExcept(roots); + } + + async abortUpdate(): Promise { + if (!this.#pendingRoots) { + return; + } + this.#pendingRoots = undefined; + await this.#closeExcept(this.#activeRoots); + } + + get(root: string): Promise { + if (!this.#pendingRoots) { + throw new Error("Artifacts backend requested outside runtime update"); + } + this.#pendingRoots.add(root); + let backend = this.#backends.get(root); + if (!backend) { + backend = this.#start(root); + this.#backends.set(root, backend); + void backend.catch(() => { + if (this.#backends.get(root) === backend) { + this.#backends.delete(root); + } + }); + } + return backend; + } + + async #start(root: string): Promise { + const port = await getPort({ host: "127.0.0.1" }); + // __dirname is dist/src in the bundled Miniflare entry point. + const worker = new Worker( + path.join(__dirname, "plugins/artifacts/git-sidecar.js"), + { workerData: { root } } + ); + try { + const endpoint = await waitForSidecarReady(worker); + worker.on("error", (error) => { + process.emitWarning( + `Local Artifacts Git backend failed: ${error.message}` + ); + }); + return { + port, + sidecar: { + ...endpoint, + async close() { + if (worker.threadId === -1) { + return; + } + const exited = once(worker, "exit"); + worker.postMessage("close"); + await exited; + }, + }, + }; + } catch (error) { + await worker.terminate(); + throw error; + } + } + + async #closeExcept(roots: Set): Promise { + const retired = [...this.#backends].filter(([root]) => !roots.has(root)); + for (const [root] of retired) { + this.#backends.delete(root); + } + const results = await Promise.allSettled( + retired.map(([, backend]) => + backend.then( + ({ sidecar }) => sidecar.close(), + () => undefined + ) + ) + ); + for (const result of results) { + if (result.status === "rejected") { + process.emitWarning("Unable to stop a retired Artifacts Git backend"); + } + } + } + + async dispose(): Promise { + this.#pendingRoots = undefined; + this.#activeRoots.clear(); + await this.#closeExcept(this.#activeRoots); + } +} + +function waitForSidecarReady( + worker: Worker +): Promise> { + return new Promise((resolve, reject) => { + function cleanup() { + worker.off("message", ready); + worker.off("error", failed); + worker.off("exit", exited); + } + function ready(value: Pick) { + cleanup(); + resolve(value); + } + function failed(error: Error) { + cleanup(); + reject(error); + } + function exited(code: number) { + failed( + new Error(`Local Artifacts Git backend exited during startup (${code})`) + ); + } + worker.on("message", ready); + worker.on("error", failed); + worker.on("exit", exited); + }); +} diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts new file mode 100644 index 00000000000..f707285854b --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -0,0 +1,430 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { devNull, tmpdir } from "node:os"; +import path from "node:path"; + +const decoder = new TextDecoder(); +type GitOptions = { allowFailure?: boolean; timeout?: number }; +type GitResult = { stdout: Buffer; stderr: Buffer; status: number }; + +// Git runs on the host, never in the user's Worker. Do not inherit Cloudflare +// tokens, Git config overrides, or credential helpers from the invoking shell. +export function gitEnvironment( + source: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv { + const keep = [ + "PATH", + "SystemRoot", + "WINDIR", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + "LANG", + // Explicit host CA bundles may be used to trust a private HTTPS Git origin. + // Do not inherit other Git TLS overrides (especially GIT_SSL_NO_VERIFY), + // credential helpers, or HTTP headers from the host environment. + "GIT_SSL_CAINFO", + ]; + const env: NodeJS.ProcessEnv = {}; + for (const key of keep) { + const value = source[key]; + if (value !== undefined) { + env[key] = value; + } + } + return { + ...env, + GIT_CONFIG_NOSYSTEM: "1", + // Git for Windows cannot read null-device paths. A unique, nonexistent + // config path prevents it from falling back to the host's Git settings. + GIT_CONFIG_GLOBAL: + process.platform === "win32" + ? path.join(tmpdir(), `miniflare-artifacts-empty-${randomUUID()}`) + : devNull, + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "credential.helper", + GIT_CONFIG_VALUE_0: "", + GIT_TERMINAL_PROMPT: "0", + }; +} + +/** Verify host Git at startup and report how to install it if unavailable. */ +export async function assertGitAvailable(): Promise { + try { + await runGit(["--version"], { timeout: 30_000 }); + } catch (error) { + throw new Error( + "Local Artifacts requires Git installed on the host and available on PATH. Install Git, verify that `git --version` works, then restart your dev server or test runner.", + { cause: error } + ); + } +} + +export async function runGit( + args: string[], + options: GitOptions = {} +): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", args, { + env: gitEnvironment(), + stdio: ["pipe", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + const timeout = setTimeout(() => child.kill(), options.timeout ?? 300_000); + timeout.unref(); + child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.once("close", (status) => { + clearTimeout(timeout); + const result = { + stdout: Buffer.concat(stdout), + stderr: Buffer.concat(stderr), + status: status ?? 1, + }; + if (result.status === 0 || options.allowFailure) { + resolve(result); + } else { + reject( + new Error( + decoder.decode(result.stderr).trim() || + `git ${args[0]} exited with status ${result.status}` + ) + ); + } + }); + child.stdin.end(); + }); +} + +export interface RepositoryState { + defaultBranch: string; + refs: Record; +} + +export interface CommitMetadata { + hash: string; + treeHash: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + parents: string[]; + authoredAt: number; + committedAt: number; +} + +/** Repository-scoped Git operations; clone/init are supported before it exists. */ +export class GitClient { + constructor(readonly repository: string) {} + + private git(args: string[], options?: GitOptions): Promise { + return runGit(["-C", this.repository, ...args], options); + } + + async init(defaultBranch: string): Promise { + await runGit([ + "init", + "--bare", + `--initial-branch=${defaultBranch}`, + this.repository, + ]); + } + + async forkFrom( + source: GitClient, + branch: string, + defaultBranchOnly: boolean + ): Promise { + const sourceBranch = `refs/heads/${branch}`; + const sourceHasBranch = await source.git( + ["rev-parse", "--verify", "--quiet", sourceBranch], + { allowFailure: true } + ); + if (defaultBranchOnly && sourceHasBranch.status === 0) { + await runGit([ + "clone", + "--bare", + "--local", + "--single-branch", + "--branch", + branch, + source.repository, + this.repository, + ]); + } else if (defaultBranchOnly) { + // Git cannot clone a named branch that has no commits yet. + await this.init(branch); + } else { + await runGit([ + "clone", + "--bare", + "--local", + source.repository, + this.repository, + ]); + } + } + + async importFrom( + url: string, + branch?: string, + depth?: number + ): Promise { + // An import may carry credentials. Never let Git redirect them to a + // different destination (or silently expand the requested network access). + const args = ["-c", "http.followRedirects=false", "clone", "--bare"]; + if (branch) { + args.push("--branch", branch, "--single-branch"); + } + if (depth) { + args.push("--depth", String(depth)); + } + args.push(url, this.repository); + await runGit(args); + // A bare clone records remote.origin.url in its local Git config. It may + // include Basic credentials or query tokens used only for the import. + // Local repositories do not fetch from the upstream after creation. + await this.git(["remote", "remove", "origin"]); + } + + async configure(generation: string): Promise { + await this.git(["config", "http.receivepack", "true"]); + await this.git(["config", "local-artifacts.generation", generation]); + } + + async generation(): Promise { + const result = await this.git( + ["config", "--get", "local-artifacts.generation"], + { allowFailure: true } + ); + return result.status === 0 ? decoder.decode(result.stdout).trim() : null; + } + + async state(): Promise { + const refsOutput = await this.git([ + "for-each-ref", + "--format=%(refname)%00%(objectname)", + ]); + const refs: Record = {}; + for (const line of decoder.decode(refsOutput.stdout).trim().split("\n")) { + if (!line) { + continue; + } + const [name, oid] = line.split("\0"); + if (name && oid) { + refs[name] = oid; + } + } + const head = await this.git(["symbolic-ref", "--short", "HEAD"], { + allowFailure: true, + }); + const defaultBranch = + decoder + .decode(head.stdout) + .trim() + .replace(/^refs\/heads\//, "") || "main"; + return { defaultBranch, refs }; + } + + async readBlob(hash: string): Promise<{ data: string } | null> { + if ((await this.objectType(hash)) !== "blob") { + return null; + } + const object = await this.git(["cat-file", "blob", hash]); + return { data: Buffer.from(object.stdout).toString("base64") }; + } + + async readTree(hash: string): Promise | null> { + if ((await this.objectType(hash)) !== "tree") { + return null; + } + const tree = await this.git(["ls-tree", "-z", hash]); + return decoder + .decode(tree.stdout) + .split("\0") + .filter(Boolean) + .map((line) => { + const match = /^(\d+) \w+ ([0-9a-f]{40})\t([\s\S]+)$/.exec(line); + if (!match?.[1] || !match[2] || match[3] === undefined) { + throw new Error("Invalid git tree output"); + } + const mode = match[1] === "040000" ? "40000" : match[1]; + return { + name: match[3], + mode, + hash: match[2], + type: treeEntryType(mode), + }; + }); + } + + async readCommit(hash: string): Promise { + if ((await this.objectType(hash)) !== "commit") { + return null; + } + const object = await this.git(["cat-file", "commit", hash]); + return parseCommit(hash, decoder.decode(object.stdout)); + } + + async readFile(ref: string, path: string): Promise<{ data: string } | null> { + const commit = await this.resolveCommit(ref); + if (!commit) { + return null; + } + const entry = await this.git(["ls-tree", "-z", commit, "--", path], { + allowFailure: true, + }); + if (entry.status !== 0 || entry.stdout.length === 0) { + return null; + } + const line = decoder.decode(entry.stdout).split("\0", 1)[0] ?? ""; + const match = /^(\d+) \w+ [0-9a-f]{40}\t([\s\S]+)$/.exec(line); + if (!match?.[1] || match[2] !== path || match[1] === "040000") { + return null; + } + const object = await this.git(["cat-file", "blob", `${commit}:${path}`], { + allowFailure: true, + }); + return object.status === 0 + ? { data: Buffer.from(object.stdout).toString("base64") } + : null; + } + + async log(ref?: string, limit = 50, offset = 0): Promise { + const commit = await this.resolveCommit(ref ?? "HEAD", ref === undefined); + if (!commit) { + return []; + } + const effectiveLimit = Math.min(limit, 1000); + const revs = await this.git( + [ + "rev-list", + "--first-parent", + `--max-count=${effectiveLimit}`, + `--skip=${offset}`, + commit, + ], + { allowFailure: true } + ); + if (revs.status !== 0) { + return []; + } + const hashes = decoder + .decode(revs.stdout) + .trim() + .split("\n") + .filter(Boolean); + return Promise.all( + hashes.map(async (hash) => { + const commitMetadata = await this.readCommit(hash); + if (!commitMetadata) { + throw new Error(`Commit disappeared: ${hash}`); + } + return commitMetadata; + }) + ); + } + + private async objectType(hash: string): Promise { + const result = await this.git(["cat-file", "-t", hash], { + allowFailure: true, + }); + return result.status === 0 ? decoder.decode(result.stdout).trim() : null; + } + + private async resolveCommit( + ref: string, + allowHead = false + ): Promise { + const candidates = allowHead + ? ["HEAD"] + : /^[0-9a-f]{40}$/.test(ref) + ? [ref] + : [`refs/heads/${ref}`, `refs/tags/${ref}`]; + for (const candidate of candidates) { + const result = await this.git( + ["rev-parse", "--verify", "--quiet", `${candidate}^{commit}`], + { allowFailure: true } + ); + if (result.status === 0) { + return decoder.decode(result.stdout).trim(); + } + } + return null; + } +} + +function commitHeader(headers: string[], name: string): string | undefined { + return headers + .find((line) => line.startsWith(`${name} `)) + ?.slice(name.length + 1); +} + +function parseCommitIdentity( + headers: string[], + name: string +): { + person: { name: string; email: string }; + time: number; +} { + const match = /^(.*) <([^<>]*)> (-?\d+) [+-]\d{4}$/.exec( + commitHeader(headers, name) ?? "" + ); + if (!match?.[1] || match[2] === undefined || !match[3]) { + throw new Error(`Invalid commit ${name}`); + } + return { + person: { name: match[1], email: match[2] }, + time: Number(match[3]), + }; +} + +function parseCommit(hash: string, value: string): CommitMetadata { + const split = value.indexOf("\n\n"); + const headers = value.slice(0, split < 0 ? value.length : split).split("\n"); + const author = parseCommitIdentity(headers, "author"); + const committer = parseCommitIdentity(headers, "committer"); + const treeHash = commitHeader(headers, "tree"); + if (!treeHash) { + throw new Error("Invalid commit tree"); + } + return { + hash, + treeHash, + message: (split < 0 ? "" : value.slice(split + 2)).replace(/\n$/, ""), + author: author.person, + committer: committer.person, + parents: headers + .filter((line) => line.startsWith("parent ")) + .map((line) => line.slice(7)), + authoredAt: author.time, + committedAt: committer.time, + }; +} + +function treeEntryType(mode: string): string { + if (mode === "40000") { + return "tree"; + } + if (mode === "120000") { + return "symlink"; + } + if (mode === "160000") { + return "gitlink"; + } + if (mode === "100755") { + return "exec"; + } + return "blob"; +} diff --git a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts new file mode 100644 index 00000000000..3e003b66789 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts @@ -0,0 +1,548 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { mkdir, readdir, rename, stat } from "node:fs/promises"; +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import path from "node:path"; +import { parentPort, workerData } from "node:worker_threads"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { assertGitAvailable, GitClient, gitEnvironment } from "./git-client"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "./storage"; +import type { RepositoryState } from "./git-client"; + +const adminPath = "/__local_artifacts__"; +const componentPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +type RepositoryLocks = Map>; +const decoder = new TextDecoder(); + +if (parentPort) { + const port = parentPort; + void startGitSidecar(workerData.root).then((sidecar) => { + port.postMessage({ address: sidecar.address, secret: sidecar.secret }); + port.once("message", async () => { + await sidecar.close(); + port.close(); + }); + }); +} + +export interface GitSidecar { + address: string; + secret: string; + close(): Promise; +} + +interface AdminRequest { + action: + | "create" + | "delete" + | "fork" + | "import" + | "refs" + | "readBlob" + | "readTree" + | "readCommit" + | "file" + | "log"; + namespace: string; + name: string; + defaultBranch?: string; + defaultBranchOnly?: boolean; + sourceName?: string; + sourceBranch?: string; + sourceUrl?: string; + branch?: string; + depth?: number; + hash?: string; + ref?: string; + path?: string; + limit?: number; + offset?: number; + generation?: string; +} + +function createGitSecret(): string { + return randomBytes(32).toString("hex"); +} + +export async function startGitSidecar(root: string): Promise { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + await assertGitAvailable(); + await cleanupRetiredRepositories(root); + const secret = createGitSecret(); + const repositoryLocks: RepositoryLocks = new Map(); + const server = createServer((request, response) => { + if (request.headers["x-local-artifacts-backend"] !== secret) { + response.writeHead(403).end("Forbidden"); + return; + } + void handleRequest(root, repositoryLocks, request, response).catch( + (error: unknown) => { + if (response.headersSent) { + response.destroy(error instanceof Error ? error : undefined); + } else { + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end( + error instanceof Error ? error.message : "Native Git backend failed" + ); + } + } + ); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + // The listener outlives this function; ArtifactsController closes it on disposal. + // `using` here would close it before workerd could make any requests. + server.unref(); + const addressInfo = server.address(); + if (!addressInfo || typeof addressInfo === "string") { + throw new Error("Unable to start native Git backend"); + } + const address = `127.0.0.1:${addressInfo.port}`; + return { + address, + secret, + close() { + server.closeAllConnections(); + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +function parseGitPath(pathname: string): { + namespace: string; + repository: string; + suffix: string; +} | null { + const match = pathname.match(/^\/git\/([^/]+)\/([^/]+\.git)(\/.*)?$/); + if (!match?.[1] || !match[2]) { + return null; + } + const namespace = decodeURIComponent(match[1]); + const repository = decodeURIComponent(match[2].slice(0, -4)); + validateComponent(namespace, "namespace"); + validateComponent(repository, "repository"); + return { namespace, repository, suffix: match[3] ?? "" }; +} + +// Native Git's CGI backend needs streaming Node requests and responses; keep +// packfiles on node:http rather than buffering them through a Worker router. +async function handleRequest( + root: string, + locks: RepositoryLocks, + request: IncomingMessage, + response: ServerResponse +): Promise { + const url = new URL(request.url ?? "/", "http://local-artifacts.invalid"); + if (url.pathname === adminPath) { + await handleAdmin(root, locks, request, response); + return; + } + const gitPath = parseGitPath(url.pathname); + if (!gitPath) { + response.writeHead(404).end("Not found"); + return; + } + const { namespace, repository, suffix } = gitPath; + const contentLength = Number(request.headers["content-length"] ?? 0); + if (!Number.isFinite(contentLength) || contentLength > 256 * 1024 * 1024) { + response.writeHead(413).end("Request body too large"); + return; + } + const path = repositoryPath(root, namespace, repository); + await withRepositoryLocks(locks, [path], async () => { + const generation = request.headers["x-local-artifacts-generation"]; + const configured = await new GitClient(path).generation(); + if (typeof generation !== "string" || generation !== configured) { + response.writeHead(409).end("Repository generation changed"); + return; + } + await runGitHttpBackend( + root, + request, + response, + `/${repositoryDirectory(namespace, repository)}${suffix}`, + url.search.slice(1) + ); + }); +} + +async function handleAdmin( + root: string, + locks: RepositoryLocks, + request: IncomingMessage, + response: ServerResponse +): Promise { + if (request.method !== "POST") { + response.writeHead(405).end("Method not allowed"); + return; + } + const body = JSON.parse( + decoder.decode(await readRequest(request)) + ) as AdminRequest; + validateComponent(body.namespace, "namespace"); + validateComponent(body.name, "repository"); + const repository = repositoryPath(root, body.namespace, body.name); + const lockPaths = [repository]; + if (body.action === "fork" && body.sourceName) { + validateComponent(body.sourceName, "repository"); + lockPaths.push(repositoryPath(root, body.namespace, body.sourceName)); + } + await withRepositoryLocks(locks, lockPaths, async () => { + const result = await handleAction(root, body, new GitClient(repository)); + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(result)); + }); +} + +async function handleAction( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + switch (body.action) { + case "create": + return handleCreate(root, body, git); + case "delete": + await retireRepository(git.repository); + return true; + case "fork": + return handleFork(root, body, git); + case "import": + return handleImport(root, body, git); + case "refs": + return git.state(); + case "readBlob": + return git.readBlob(required(body.hash, "hash")); + case "readTree": + return git.readTree(required(body.hash, "hash")); + case "readCommit": + return git.readCommit(required(body.hash, "hash")); + case "file": + return git.readFile( + required(body.ref, "ref"), + required(body.path, "path") + ); + case "log": + return git.log(body.ref, body.limit, body.offset); + } +} + +function handleCreate( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + return withNewRepository(root, body, git, () => + git.init(body.defaultBranch ?? "main") + ); +} + +function handleFork( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + const { sourceName, sourceBranch } = body; + if (!sourceName) { + throw new Error("Fork source is required"); + } + if (!sourceBranch) { + throw new Error("Fork source branch is required"); + } + const source = new GitClient( + repositoryPath(root, body.namespace, sourceName) + ); + return withNewRepository(root, body, git, () => + git.forkFrom(source, sourceBranch, body.defaultBranchOnly ?? true) + ); +} + +function handleImport( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + const { sourceUrl } = body; + if (!sourceUrl) { + throw new Error("Import source is required"); + } + return withNewRepository(root, body, git, () => + git.importFrom(sourceUrl, body.branch, body.depth) + ); +} + +async function withNewRepository( + root: string, + body: AdminRequest, + git: GitClient, + initialize: () => Promise +): Promise { + await mkdir(root, { recursive: true }); + if (await exists(git.repository)) { + // The namespace has no record for this name. A prior deletion may have + // committed metadata before the process stopped removing Git data. + // Never remove a directory that was not created by this simulator. + if ((await git.generation()) === null) { + throw new Error(`Repository "${body.name}" already exists on disk`); + } + await retireRepository(git.repository); + } + try { + await initialize(); + await git.configure(required(body.generation, "generation")); + return await git.state(); + } catch (error) { + await removeDir(git.repository); + throw error; + } +} + +// Moving the directory out of its active path is atomic on the same volume. +// A crash while removing its contents leaves only an inaccessible trash path; +// the next sidecar startup retries cleanup without blocking name reuse. +async function retireRepository(repository: string): Promise { + const retired = `${repository}.deleting-${randomBytes(16).toString("hex")}`; + try { + await rename(repository, retired); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + await removeDir(retired); +} + +async function cleanupRetiredRepositories(root: string): Promise { + for (const entry of await readdir(root, { withFileTypes: true })) { + if ( + entry.isDirectory() && + /^[0-9a-f]{32}\.git\.deleting-[0-9a-f]{32}$/.test(entry.name) + ) { + await removeDir(path.join(root, entry.name)); + } + } +} + +function required(value: string | undefined, name: string): string { + if (!value) { + throw new Error(`${name} is required`); + } + return value; +} + +function parseGitBackendHeaders(headerBytes: Buffer): { + status: number; + headers: Record; +} { + const lines = decoder.decode(headerBytes).split(/\r?\n/); + let status = 200; + const headers: Record = {}; + for (const line of lines) { + const separator = line.indexOf(":"); + if (separator < 0) { + continue; + } + const name = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + if (name.toLowerCase() === "status") { + status = Number(value.slice(0, 3)); + } else { + headers[name] = value; + } + } + return { status, headers }; +} + +async function runGitHttpBackend( + root: string, + request: IncomingMessage, + response: ServerResponse, + pathInfo: string, + query: string +): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", ["http-backend"], { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: root, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: pathInfo, + QUERY_STRING: query, + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + REMOTE_USER: "local-artifacts", + REMOTE_ADDR: request.socket.remoteAddress ?? "127.0.0.1", + }, + stdio: ["pipe", "pipe", "pipe"], + }); + let header = Buffer.alloc(0); + let headersSent = false; + let settled = false; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill(); + }, 300_000); + timeout.unref(); + const stderr: Buffer[] = []; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + request.off("aborted", abort); + response.off("close", abort); + if (error) { + reject(error); + } else { + resolve(); + } + }; + const abort = () => { + if (!response.writableEnded) { + child.kill(); + } + }; + const write = (chunk: Buffer) => { + if (!response.write(chunk)) { + child.stdout.pause(); + response.once("drain", () => child.stdout.resume()); + } + }; + request.once("aborted", abort); + response.once("close", abort); + child.stdin.on("error", (error) => { + if (!request.destroyed) { + finish(error); + } + }); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.stdout.on("data", (chunk: Buffer) => { + if (headersSent) { + write(chunk); + return; + } + header = Buffer.concat([header, chunk]); + const crlf = header.indexOf("\r\n\r\n"); + const lf = header.indexOf("\n\n"); + const index = crlf >= 0 ? crlf : lf; + if (index < 0) { + if (header.length > 64 * 1024) { + child.kill(); + } + return; + } + const separatorLength = crlf >= 0 ? 4 : 2; + const { status, headers } = parseGitBackendHeaders( + header.subarray(0, index) + ); + response.writeHead(status, headers); + headersSent = true; + const remaining = header.subarray(index + separatorLength); + if (remaining.length) { + write(remaining); + } + header = Buffer.alloc(0); + }); + child.stdout.on("end", () => { + if (headersSent) { + response.end(); + } + }); + child.once("error", finish); + child.once("close", (status) => { + if (timedOut) { + finish(new Error("git http-backend timed out")); + return; + } + if (!headersSent) { + finish( + new Error( + decoder.decode(Buffer.concat(stderr)).trim() || + `git http-backend exited with status ${status}` + ) + ); + } else { + finish(); + } + }); + request.pipe(child.stdin); + }); +} + +function readRequest(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => resolve(Buffer.concat(chunks))); + request.once("error", reject); + }); +} + +function validateComponent(value: string, type: string): void { + if (!componentPattern.test(value)) { + throw new Error(`Invalid ${type}`); + } +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +} + +async function withRepositoryLocks( + repositoryLocks: RepositoryLocks, + paths: string[], + action: () => Promise +): Promise { + const keys = [...new Set(paths)].sort(); + const previous = keys.map( + (key) => repositoryLocks.get(key) ?? Promise.resolve() + ); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + for (const key of keys) { + repositoryLocks.set(key, current); + } + await Promise.all(previous); + try { + return await action(); + } finally { + release(); + for (const key of keys) { + if (repositoryLocks.get(key) === current) { + repositoryLocks.delete(key); + } + } + } +} diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 48242920859..c79d6b87093 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -1,31 +1,87 @@ +import { createHash } from "node:crypto"; +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import SCRIPT from "worker:artifacts/binding"; import { buildRemoteProxyProps, getEnvBindingsOfType, + getPersistPath, getRemoteProxyConnectionString, ProxyNodeBinding, remoteProxyClientWorker, } from "../shared"; -import type { Plugin } from "../shared"; +import { assertGitAvailable } from "./git-client"; +import { assertSupportedGitLayout } from "./storage"; +import type { Service, Socket } from "../../runtime"; +import type { MiniflareBinding, Plugin } from "../shared"; +import type { ArtifactsController } from "./controller"; + +export { ArtifactsController, type LocalArtifactsBackend } from "./controller"; +export type { GitSidecar } from "./git-sidecar"; export const ARTIFACTS_PLUGIN_NAME = "artifacts"; -// One shared remote-proxy service for every artifacts binding; per-binding -// config travels via props. const ARTIFACTS_REMOTE_SERVICE_NAME = `${ARTIFACTS_PLUGIN_NAME}:remote`; +const ENTRYPOINT = "LocalArtifactsNamespace"; +const OBJECT_CLASS = "LocalArtifactsNamespaceObject"; +const LOCAL_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +type ArtifactsBinding = Extract; +type ArtifactsEntry = [name: string, binding: ArtifactsBinding]; + +function isLocalArtifactsBinding(binding: ArtifactsBinding): boolean { + return binding.dev?.remote !== true; +} + +function getLocalNamespaceNames(bindings: ArtifactsEntry[]): Set { + return new Set( + bindings + .filter(([, binding]) => isLocalArtifactsBinding(binding)) + .map(([, binding]) => binding.namespace) + ); +} + +function validateLocalNamespace(namespace: string): void { + if (!LOCAL_NAMESPACE_PATTERN.test(namespace)) { + throw new Error(`Invalid local Artifacts namespace: ${namespace}`); + } +} + +function namespaceId(namespace: string): string { + // 128 bits keeps storage paths short enough for workerd's SQLite files on Windows. + return createHash("sha256").update(namespace).digest("hex").slice(0, 32); +} + +function localServiceName(namespace: string): string { + return `${ARTIFACTS_PLUGIN_NAME}-${namespaceId(namespace)}`; +} export const ARTIFACTS_PLUGIN: Plugin = { bindingTypeDescription: "Artifacts", - async getBindings(options) { + getBindings(options) { return getEnvBindingsOfType(options.config, "artifacts").map( - ([name, binding]) => ({ - name, - service: { - name: ARTIFACTS_REMOTE_SERVICE_NAME, - props: buildRemoteProxyProps( - getRemoteProxyConnectionString(binding, options.dev), - name - ), - }, - }) + ([name, binding]) => { + // Offline by default; remote proxying requires explicit opt-in. + if (isLocalArtifactsBinding(binding)) { + validateLocalNamespace(binding.namespace); + return { + name, + service: { + name: localServiceName(binding.namespace), + entrypoint: ENTRYPOINT, + }, + }; + } + return { + name, + service: { + name: ARTIFACTS_REMOTE_SERVICE_NAME, + props: buildRemoteProxyProps( + getRemoteProxyConnectionString(binding, options.dev), + name + ), + }, + }; + } ); }, getNodeBindings(options) { @@ -36,16 +92,106 @@ export const ARTIFACTS_PLUGIN: Plugin = { ]) ); }, - async getServices({ options }) { - if (getEnvBindingsOfType(options.config, "artifacts").length === 0) { - return []; - } - - return [ - { + async getServices({ options, sharedOptions, tmpPath, artifactsController }) { + const bindings = getEnvBindingsOfType(options.config, "artifacts"); + const services: Service[] = []; + const sockets: Socket[] = []; + if (bindings.some(([, binding]) => !isLocalArtifactsBinding(binding))) { + services.push({ name: ARTIFACTS_REMOTE_SERVICE_NAME, worker: remoteProxyClientWorker(), - }, - ]; + }); + } + const namespaces = getLocalNamespaceNames(bindings); + if (namespaces.size > 0) { + await assertGitAvailable(); + } + const root = getPersistPath( + ARTIFACTS_PLUGIN_NAME, + tmpPath, + sharedOptions.resourcePersistencePath + ); + + for (const namespace of namespaces) { + const local = await createLocalNamespaceRuntime( + root, + namespace, + artifactsController + ); + sockets.push(local.socket); + services.push(...local.services); + } + return { services, extensions: [], sockets }; }, }; + +async function createLocalNamespaceRuntime( + root: string, + namespace: string, + artifactsController: ArtifactsController +): Promise<{ services: Service[]; socket: Socket }> { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + const persistPath = path.join(root, namespaceId(namespace)); + const metadataPath = path.join(persistPath, "metadata").replaceAll("\\", "/"); + await mkdir(metadataPath, { recursive: true }); + await assertSupportedGitLayout(metadataPath); + const { sidecar, port } = await artifactsController.get( + path.join(persistPath, "git") + ); + const serviceName = localServiceName(namespace); + const storageName = `${serviceName}-storage`; + const backendName = `${serviceName}-backend`; + const socket: Socket = { + name: `${serviceName}-git`, + address: `127.0.0.1:${port}`, + http: {}, + service: { name: serviceName, entrypoint: ENTRYPOINT }, + }; + const storageService: Service = { + name: storageName, + disk: { path: metadataPath, writable: true }, + }; + const gitBackendService: Service = { + name: backendName, + external: { + address: sidecar.address, + http: { + injectRequestHeaders: [ + { name: "X-Local-Artifacts-Backend", value: sidecar.secret }, + ], + }, + }, + }; + const bindingService: Service = { + name: serviceName, + worker: { + compatibilityDate: "2026-09-03", + modules: [{ name: "binding.worker.js", esModule: SCRIPT() }], + bindings: [ + { + name: "config", + json: JSON.stringify({ + namespace, + origin: `http://127.0.0.1:${port}`, + }), + }, + { + name: "localArtifactsNamespace", + durableObjectNamespace: { className: OBJECT_CLASS }, + }, + { name: "gitBackend", service: { name: backendName } }, + ], + // Keys must be unique across namespaces in one workerd, but should + // not repeat the long service prefix in SQLite's directory name. + durableObjectNamespaces: [ + { className: OBJECT_CLASS, uniqueKey: namespaceId(namespace) }, + ], + durableObjectStorage: { localDisk: storageName }, + }, + }; + return { + services: [storageService, gitBackendService, bindingService], + socket, + }; +} diff --git a/packages/miniflare/src/plugins/artifacts/storage.ts b/packages/miniflare/src/plugins/artifacts/storage.ts new file mode 100644 index 00000000000..ac693dc1bcc --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/storage.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { readdir } from "node:fs/promises"; +import path from "node:path"; + +/** + * Keep repository paths short: native Git adds long pack and lock filenames. + * Hash the tuple rather than nesting namespace and repository hashes. Each + * identifier retains 128 bits and no user input becomes a path component. + */ +export function repositoryDirectory(namespace: string, name: string): string { + const id = createHash("sha256") + .update(JSON.stringify([namespace, name])) + .digest("hex") + .slice(0, 32); + return `${id}.git`; +} + +export function repositoryPath( + root: string, + namespace: string, + name: string +): string { + return path.join(root, repositoryDirectory(namespace, name)); +} + +/** Do not quietly abandon repositories written by earlier unreleased layouts. */ +export async function assertSupportedGitLayout(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + if ( + entries.some( + (entry) => + entry.isDirectory() && + (/^[0-9a-f]{64}$/.test(entry.name) || /^artifacts[-:]/.test(entry.name)) + ) + ) { + throw new Error( + "Local Artifacts storage uses an older draft layout. Back up your local repositories, then reset the Artifacts persistence directory before restarting." + ); + } +} diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 9c0658a0c3b..578f1a1deca 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -12,6 +12,7 @@ import type { Extension, Service, ServiceDesignator, + Socket, Worker_Binding, Worker_Module, } from "../../runtime"; @@ -21,6 +22,7 @@ import type { QueueConsumerSchema, QueueProducerSchema, } from "../../workers"; +import type { ArtifactsController } from "../artifacts/controller"; import type { ContainerPrivilegesCache } from "../core/container"; import type { DOContainerOptions } from "../do"; import type { HyperdriveProxyController } from "../hyperdrive/hyperdrive-proxy"; @@ -88,11 +90,14 @@ export interface PluginServicesOptions { devRegistryEnabled: boolean; containerPrivilegesCache: ContainerPrivilegesCache; hyperdriveProxyController: HyperdriveProxyController; + artifactsController: ArtifactsController; } export interface ServicesExtensions { services: Service[]; extensions: Extension[]; + // Internal plugin listeners (e.g. Git smart HTTP), deduplicated by name. + sockets?: Socket[]; } /** diff --git a/packages/miniflare/src/workers/artifacts/binding.worker.ts b/packages/miniflare/src/workers/artifacts/binding.worker.ts new file mode 100644 index 00000000000..1ad28e69706 --- /dev/null +++ b/packages/miniflare/src/workers/artifacts/binding.worker.ts @@ -0,0 +1,1255 @@ +import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; + +const repoName = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +type TokenScope = "read" | "write"; + +type RepositorySource = + | { type: "fork"; repository: string } + | { type: "remote"; url: string }; + +interface RepositoryOptions { + description?: string | undefined; + readOnly?: boolean; + setDefaultBranch?: string; +} + +interface ForkOptions { + description?: string; + readOnly?: boolean; + defaultBranchOnly?: boolean; +} + +interface ImportRepositoryOptions { + source: { url: string; branch?: string; depth?: number }; + target: { name: string; opts?: { description?: string; readOnly?: boolean } }; +} + +interface RepositoryToken { + id: string; + tokenHash: string; + scope: TokenScope; + createdAt: string; + expiresAt: string; + revokedAt?: string; +} + +interface CreatedToken { + record: RepositoryToken; + plaintext: string; +} + +interface Repository { + id: string; + name: string; + namespace: string; + description?: string | undefined; + readOnly: boolean; + defaultBranch: string; + remote: string; + source?: RepositorySource | undefined; + createdAt: string; + updatedAt: string; + lastPushAt?: string | undefined; + tokens: RepositoryToken[]; +} + +interface RepositoryMetadata { + id: string; + name: string; + remote: string; + defaultBranch: string; + description: string | null; + source: string | null; + readOnly: boolean; + createdAt: string; + updatedAt: string; + lastPushAt: string | null; +} + +interface CreateRepositoryResult { + id: string; + name: string; + description: string | null; + defaultBranch: string; + remote: string; + token: string; +} + +interface CreateTokenResult { + id: string; + plaintext: string; + scope: TokenScope; + expiresAt: string; +} +type TokenInfo = Omit & { + state: "active" | "expired" | "revoked"; +}; + +interface RepositoryListOptions { + cursor?: string; + limit?: number; +} + +type RepositoryListResult = { + repos: Array< + RepositoryMetadata & { status: "ready"; jurisdiction: "unrestricted" } + >; + total: number; + cursor?: string; +}; + +interface RepositoryLogOptions { + ref?: string; + limit?: number; + offset?: number; +} + +interface BackendState { + defaultBranch: string; + refs: Record; +} + +type BackendRequest = + | { action: "create"; defaultBranch: string } + | { action: "delete" } + | { + action: "fork"; + sourceName: string; + sourceBranch: string; + defaultBranchOnly?: boolean; + } + | { action: "import"; sourceUrl: string; branch?: string; depth?: number } + | { action: "refs" } + | { action: "readBlob"; hash: string } + | { action: "readTree"; hash: string } + | { action: "readCommit"; hash: string } + | { action: "file"; ref: string; path: string } + | { action: "log"; ref?: string; limit?: number; offset?: number }; + +interface TreeEntry { + name: string; + mode: string; + hash: string; + type: "tree" | "blob" | "symlink" | "gitlink" | "exec"; +} + +interface CommitMetadata { + hash: string; + treeHash: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + parents: string[]; + authoredAt: number; + committedAt: number; +} + +interface LocalArtifactsRepo { + info(): Promise; + createToken(scope?: TokenScope, ttl?: number): Promise; + listTokens(): Promise<{ total: number; tokens: TokenInfo[] }>; + revokeToken(tokenOrId: string): Promise; + readBlob(hash: string): Promise; + readTree(hash: string): Promise; + readCommit(hash: string): Promise; + readFile(args: { ref: string; path: string }): Promise; + log(options?: RepositoryLogOptions): Promise; + fork(name: string, options?: ForkOptions): Promise; +} + +interface BackendBlob { + data: string; +} + +type ArtifactsErrorCode = + | "ALREADY_EXISTS" + | "NOT_FOUND" + | "INVALID_INPUT" + | "INVALID_REPO_NAME" + | "INVALID_TTL" + | "INVALID_URL" + | "REMOTE_AUTH_REQUIRED" + | "MEMORY_LIMIT" + | "UPSTREAM_UNAVAILABLE" + | "INTERNAL_ERROR"; + +const numericCodes: Record = { + INVALID_INPUT: 10100, + INVALID_REPO_NAME: 10101, + INVALID_TTL: 10103, + INVALID_URL: 10104, + REMOTE_AUTH_REQUIRED: 10106, + NOT_FOUND: 10200, + ALREADY_EXISTS: 10201, + INTERNAL_ERROR: 10400, + UPSTREAM_UNAVAILABLE: 10401, + MEMORY_LIMIT: 10402, +}; + +class ArtifactsError extends Error { + readonly code: ArtifactsErrorCode; + readonly numericCode: number; + + constructor(code: ArtifactsErrorCode, message: string) { + super(message); + this.name = "ArtifactsError"; + this.code = code; + this.numericCode = numericCodes[code]; + } +} + +interface LocalArtifactsEnvironment { + config: { namespace: string; origin: string }; + localArtifactsNamespace: DurableObjectNamespace; + gitBackend: Fetcher; +} + +type RepositoryMap = Record; + +export class LocalArtifactsNamespace extends WorkerEntrypoint { + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return publicCall(() => this.#namespace().create(name, options)); + } + + async get(name: string): Promise { + const repository = await publicCall(() => this.#namespace().get(name)); + return repository as unknown as LocalArtifactsRepo; + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return publicCall(() => this.#namespace().import(options)); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + return publicCall(() => this.#namespace().list(options)); + } + + async delete(name: string): Promise { + return publicCall(() => this.#namespace().delete(name)); + } + + override async fetch(request: Request): Promise { + return this.#namespace().fetch(request); + } + + #namespace(): DurableObjectStub { + const { localArtifactsNamespace, config } = this.env; + return localArtifactsNamespace.get( + localArtifactsNamespace.idFromName(config.namespace) + ); + } +} + +export class LocalArtifactsNamespaceObject extends DurableObject { + // The resolved tail lets the first mutation run immediately. exclusive() + // serializes subsequent metadata and backend mutations across awaits. + private operations = Promise.resolve(); + + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + + const repository = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + try { + const state = await this.backend(repository, { + action: "create", + defaultBranch: repository.defaultBranch, + }); + applyBackendState(repository, state); + repositories[name.toLowerCase()] = repository; + const token = await createToken(repository, "write"); + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist Artifacts repository" + ); + } + return createRepoResult(repository, token); + } catch (error) { + if (error instanceof ArtifactsError) { + throw error; + } + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to create Artifacts repository" + ); + } + }); + } + + async get(name: string): Promise { + validateName(name); + const repository = await this.getRepo(name); + return createRepoHandle(this, repository.name, repository.id); + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return this.exclusive(async () => { + const { source, target, sourceUrl } = parseImportRequest(options); + const repositories = await this.repos(); + if (hasRepository(repositories, target.name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${target.name}` + ); + } + validateRepositoryOptions(target.opts ?? {}); + const repository = makeRepo( + target.name, + this.env.config.namespace, + this.env.config.origin, + target.opts ?? {} + ); + // The clone URL may contain one-time credentials. Keep them out of + // repository metadata returned by info() and persisted in storage. + const publicSourceUrl = new URL(sourceUrl); + publicSourceUrl.username = ""; + publicSourceUrl.password = ""; + publicSourceUrl.search = ""; + publicSourceUrl.hash = ""; + repository.source = { type: "remote", url: publicSourceUrl.href }; + const state = await this.backend(repository, { + action: "import", + sourceUrl: sourceUrl.href, + ...(source.branch === undefined ? {} : { branch: source.branch }), + ...(source.depth === undefined ? {} : { depth: source.depth }), + }); + applyBackendState(repository, state); + repository.updatedAt = new Date().toISOString(); + const token = await createToken(repository, "write"); + repositories[repository.name.toLowerCase()] = repository; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist imported Artifacts repository" + ); + } + return createRepoResult(repository, token); + }); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + const limit = options.limit ?? 50; + if (!Number.isInteger(limit) || limit < 1 || limit > 200) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 200." + ); + } + const offset = decodeCursor(options.cursor); + const repositories = Object.values(await this.repos()).sort((left, right) => + right.createdAt.localeCompare(left.createdAt) + ); + const page = repositories + .slice(offset, offset + limit) + .map((repository) => ({ + ...repoMetadata(repository), + status: "ready" as const, + // Local storage has no placement jurisdiction; this is the public default. + jurisdiction: "unrestricted" as const, + })); + const cursor = + offset + page.length < repositories.length + ? encodeCursor(offset + page.length) + : undefined; + return { + repos: page, + total: repositories.length, + ...(cursor === undefined ? {} : { cursor }), + }; + } + + async delete(name: string): Promise { + return this.exclusive(async () => { + validateName(name); + const repositories = await this.repos(); + if (!hasRepository(repositories, name)) { + return false; + } + const repository = getRepo(repositories, name); + // Persist removal first. If storage fails, the Git contents and + // metadata must both remain usable. Git cleanup first retires the + // directory atomically; startup or a later creation cleans orphans. + delete repositories[name.toLowerCase()]; + await this.saveRepos(repositories); + await this.backend(repository, { action: "delete" }); + return true; + }); + } + + async createToken( + name: string, + scope: TokenScope = "write", + ttl?: number, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const token = await createToken(repository, scope, ttl); + await this.saveRepos(repositories); + return tokenResult(token); + }); + } + + async listTokens( + name: string, + expectedId?: string + ): Promise<{ total: number; tokens: TokenInfo[] }> { + const repository = await this.getRepo(name, expectedId); + const active = repository.tokens.filter( + (token) => !token.revokedAt && isActive(token) + ); + const tokens = active + .toReversed() + .slice(0, 30) + .map(({ tokenHash: _tokenHash, revokedAt: _revokedAt, ...token }) => ({ + ...token, + state: "active" as const, + })); + return { total: active.length, tokens }; + } + + async revokeToken( + name: string, + tokenOrId: string, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + if (!isTokenId(tokenOrId) && !isArtifactToken(tokenOrId)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid tokenOrId: must be a token id or artifact token." + ); + } + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const tokenHash = isArtifactToken(tokenOrId) + ? await hashToken(tokenOrId) + : undefined; + const token = repository.tokens.find( + (candidate) => + candidate.id === tokenOrId || + (tokenHash !== undefined && + constantTimeEqual(candidate.tokenHash, tokenHash)) + ); + if (!token || token.revokedAt) { + return false; + } + token.revokedAt = new Date().toISOString(); + await this.saveRepos(repositories); + return true; + }); + } + + async fork( + sourceName: string, + name: string, + options: ForkOptions = {}, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + const source = getRepo(repositories, sourceName, expectedId); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + const fork = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + let state: BackendState; + try { + state = await this.backend(fork, { + action: "fork", + sourceName: source.name, + sourceBranch: source.defaultBranch, + ...(options.defaultBranchOnly === undefined + ? {} + : { defaultBranchOnly: options.defaultBranchOnly }), + }); + } catch { + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to fork Artifacts repository" + ); + } + applyBackendState(fork, state); + fork.source = { type: "fork", repository: source.name }; + const token = await createToken(fork, "write"); + repositories[name.toLowerCase()] = fork; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(fork); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist forked Artifacts repository" + ); + } + return createRepoResult(fork, token); + }); + } + + override async fetch(request: Request): Promise { + return handleGitRequest(this, this.env.config.namespace, request); + } + + async repos(): Promise { + const repositories = + (await this.ctx.storage.get("repos")) ?? + (Object.create(null) as RepositoryMap); + // The local Git listener can move after a process restart. + for (const repository of Object.values(repositories)) { + repository.remote = `${this.env.config.origin}/git/${repository.namespace}/${repository.name}.git`; + } + return repositories; + } + + saveRepos(repositories: RepositoryMap): Promise { + return this.ctx.storage.put("repos", repositories); + } + + async getRepo(name: string, expectedId?: string): Promise { + return getRepo(await this.repos(), name, expectedId); + } + + async backend( + repository: Repository, + request: BackendRequest + ): Promise { + const response = await this.env.gitBackend.fetch( + "http://local-artifacts-backend/__local_artifacts__", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ...request, + namespace: repository.namespace, + name: repository.name, + generation: repository.id, + }), + } + ); + if (!response.ok) { + const message = await response.text(); + if (request.action === "import") { + throw classifyImportError(message); + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } + return response.json(); + } + + private async deleteBackend(repository: Repository): Promise { + try { + await this.backend(repository, { action: "delete" }); + } catch { + // Preserve the original persistence error; native delete is idempotent on retry. + } + } + + gitFetch(repository: Repository, request: Request): Promise { + const headers = new Headers(request.headers); + headers.delete("Authorization"); + headers.set("X-Local-Artifacts-Generation", repository.id); + const url = new URL(request.url); + url.pathname = url.pathname.replace( + /^\/git\/[^/]+\/[^/]+\.git/, + `/git/${repository.namespace}/${repository.name}.git` + ); + return this.env.gitBackend.fetch( + new Request(url, new Request(request, { headers })) + ); + } + + async syncAfterPush( + name: string, + previousRefs: Record + ): Promise { + await this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name); + const state = await this.backend(repository, { + action: "refs", + }); + if (JSON.stringify(state.refs) !== JSON.stringify(previousRefs)) { + repository.updatedAt = new Date().toISOString(); + repository.lastPushAt = repository.updatedAt; + } + await this.saveRepos(repositories); + }); + } + + private async exclusive(action: () => Promise): Promise { + const previous = this.operations; + let release!: () => void; + this.operations = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await action(); + } finally { + release(); + } + } +} + +function createRepoHandle( + namespace: LocalArtifactsNamespaceObject, + name: string, + id: string +): LocalArtifactsRepo { + const read = (request: BackendRequest): Promise => + publicCall(async () => + namespace.backend(await namespace.getRepo(name, id), request) + ); + const readFile = async (args: { + ref: string; + path: string; + }): Promise => { + if (!args?.ref) { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: ref required."); + } + if (!args?.path) { + throw new ArtifactsError("INVALID_INPUT", "Invalid path: path required."); + } + const path = normalizePath(args.path); + if (path === null) { + return null; + } + const result = await read({ + action: "file", + ref: args.ref, + path, + }); + if (result === null) { + return null; + } + const bytes = decodeBase64(result.data); + return new Blob([bytes], { type: contentType(args.path, bytes) }); + }; + + return { + info: () => + publicCall(async () => repoMetadata(await namespace.getRepo(name, id))), + createToken: (scope: TokenScope = "write", ttl?: number) => + publicCall(() => namespace.createToken(name, scope, ttl, id)), + listTokens: () => publicCall(() => namespace.listTokens(name, id)), + revokeToken: (tokenOrId: string) => + publicCall(() => namespace.revokeToken(name, tokenOrId, id)), + readBlob: async (hash: string) => { + validateHash(hash); + const result = await read({ + action: "readBlob", + hash, + }); + return result === null ? null : new Blob([decodeBase64(result.data)]); + }, + readTree: async (hash: string) => { + validateHash(hash); + return await read({ action: "readTree", hash }); + }, + readCommit: async (hash: string) => { + validateHash(hash); + return await read({ action: "readCommit", hash }); + }, + readFile, + log: (options: RepositoryLogOptions = {}) => { + validateLogOptions(options); + return read({ + action: "log", + ...(options.ref === undefined ? {} : { ref: options.ref }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.offset === undefined ? {} : { offset: options.offset }), + }); + }, + fork: (targetName: string, options: ForkOptions = {}) => + publicCall(() => namespace.fork(name, targetName, options, id)), + }; +} + +function parseGitRoute( + request: Request, + namespace: string +): { name: string; write: boolean; push: boolean } | null { + const url = new URL(request.url); + const match = url.pathname.match(/^\/git\/([^/]+)\/([^/]+)\.git(?:\/(.*))?$/); + if (!match || match[1] !== namespace || !match[2]) { + return null; + } + const path = match[3] ?? ""; + const service = url.searchParams.get("service"); + const discovery = + request.method === "GET" && + path === "info/refs" && + (service === "git-upload-pack" || service === "git-receive-pack"); + const pack = + request.method === "POST" && + (path === "git-upload-pack" || path === "git-receive-pack"); + if (!discovery && !pack) { + return null; + } + return { + name: match[2], + write: path === "git-receive-pack" || service === "git-receive-pack", + push: request.method === "POST" && path === "git-receive-pack", + }; +} + +async function handleGitRequest( + state: LocalArtifactsNamespaceObject, + namespace: string, + request: Request +): Promise { + const route = parseGitRoute(request, namespace); + if (!route) { + return new Response("Not found", { status: 404 }); + } + let repository: Repository; + try { + repository = await state.getRepo(route.name); + } catch (error) { + if (error instanceof ArtifactsError && error.code === "NOT_FOUND") { + return new Response("Not found", { status: 404 }); + } + throw error; + } + const authorizationError = await authorize( + repository, + request, + route.write ? "write" : "read" + ); + if (authorizationError) { + return authorizationError; + } + if (route.write && repository.readOnly) { + return new Response("Artifacts repository is read-only", { status: 403 }); + } + + const previousRefs = route.push + ? (await state.backend(repository, { action: "refs" })).refs + : undefined; + const response = await state.gitFetch(repository, request); + if (!previousRefs) { + return response; + } + + const body = await response.arrayBuffer(); + if (response.ok) { + try { + await state.syncAfterPush(repository.name, previousRefs); + } catch { + // Git has already accepted the push; metadata bookkeeping must not fail the client request. + } + } + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function authorize( + repository: Repository, + request: Request, + scope: TokenScope +): Promise { + const header = request.headers.get("Authorization") ?? ""; + let value = ""; + if (header.startsWith("Bearer ")) { + value = header.slice(7); + } else if (header.startsWith("Basic ")) { + try { + value = atob(header.slice(6)).split(":").slice(1).join(":"); + } catch { + value = ""; + } + } + const valueHash = await hashToken(value); + const token = repository.tokens.find( + (candidate) => + !candidate.revokedAt && + constantTimeEqual(candidate.tokenHash, valueHash) && + isActive(candidate) + ); + if (!token) { + return new Response("Invalid Artifacts token", { + status: 401, + headers: { "WWW-Authenticate": 'Basic realm="Artifacts"' }, + }); + } + if (scope === "write" && token.scope !== "write") { + return new Response("Artifacts token requires write scope", { + status: 403, + }); + } + return undefined; +} + +function applyBackendState(repository: Repository, state: BackendState): void { + repository.defaultBranch = state.defaultBranch; +} + +function makeRepo( + name: string, + namespace: string, + origin: string, + options: RepositoryOptions +): Repository { + const time = new Date().toISOString(); + return { + id: randomBase36(16), + name, + namespace, + description: options.description, + readOnly: Boolean(options.readOnly), + defaultBranch: options.setDefaultBranch ?? "main", + remote: `${origin}/git/${namespace}/${name}.git`, + createdAt: time, + updatedAt: time, + lastPushAt: undefined, + tokens: [], + }; +} + +function repoMetadata(repository: Repository): RepositoryMetadata { + return { + id: repository.id, + name: repository.name, + remote: repository.remote, + defaultBranch: repository.defaultBranch, + description: repository.description ?? null, + source: + repository.source === undefined + ? null + : repository.source.type === "fork" + ? `artifacts:${repository.namespace}/${repository.source.repository}` + : `git:${repository.source.url}`, + readOnly: repository.readOnly, + createdAt: repository.createdAt, + updatedAt: repository.updatedAt, + lastPushAt: repository.lastPushAt ?? null, + }; +} + +function createRepoResult( + repository: Repository, + token: CreatedToken +): CreateRepositoryResult { + return { + id: repository.id, + name: repository.name, + description: repository.description ?? null, + defaultBranch: repository.defaultBranch, + remote: repository.remote, + token: token.plaintext, + }; +} + +function tokenResult(token: CreatedToken): CreateTokenResult { + return { + id: token.record.id, + plaintext: token.plaintext, + scope: token.record.scope, + expiresAt: token.record.expiresAt, + }; +} + +function getRepo( + repositories: RepositoryMap, + name: string, + expectedId?: string +): Repository { + const repository = repositories[name.toLowerCase()]; + if ( + !repository || + (expectedId !== undefined && repository.id !== expectedId) + ) { + throw new ArtifactsError("NOT_FOUND", `Repository not found: ${name}.`); + } + return repository; +} + +function hasRepository(repositories: RepositoryMap, name: string): boolean { + return Object.hasOwn(repositories, name.toLowerCase()); +} + +function parseImportRequest(options: ImportRepositoryOptions): { + source: ImportRepositoryOptions["source"]; + target: ImportRepositoryOptions["target"]; + sourceUrl: URL; +} { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts import options are required" + ); + } + const { source, target } = options; + validateName(target?.name); + let sourceUrl: URL; + try { + sourceUrl = new URL(source?.url); + } catch { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be a valid URL." + ); + } + if (sourceUrl.protocol !== "https:") { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be an HTTPS URL." + ); + } + return { source, target, sourceUrl }; +} + +function validateName(name: unknown): asserts name is string { + if (typeof name !== "string" || !repoName.test(name)) { + throw new ArtifactsError("INVALID_REPO_NAME", "Invalid repo name."); + } +} + +function validateRepositoryOptions( + options: RepositoryOptions | ForkOptions +): void { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository options must be an object" + ); + } + if ( + options.description !== undefined && + typeof options.description !== "string" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository description must be a string" + ); + } + if (options.readOnly !== undefined && typeof options.readOnly !== "boolean") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository readOnly must be a boolean" + ); + } + if ("setDefaultBranch" in options && options.setDefaultBranch !== undefined) { + validateBranch(options.setDefaultBranch); + } + if ( + "defaultBranchOnly" in options && + options.defaultBranchOnly !== undefined && + typeof options.defaultBranchOnly !== "boolean" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts defaultBranchOnly must be a boolean" + ); + } +} + +function validateBranch(branch: unknown): asserts branch is string { + if ( + typeof branch !== "string" || + branch.length < 1 || + branch.length > 256 || + branch.startsWith("-") || + branch.startsWith(".") || + branch.endsWith("/") || + branch.endsWith(".") || + branch.endsWith(".lock") || + branch === "@" || + branch.includes("..") || + branch.includes("//") || + branch.includes("@{") || + branch.split("/").some((segment) => segment.startsWith(".")) || + [...branch].some((character) => character.charCodeAt(0) <= 32) || + /[~^:?*[\\]/.test(branch) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts default branch is invalid" + ); + } +} + +function validateLogOptions( + options: unknown +): asserts options is RepositoryLogOptions { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts log options must be an object" + ); + } + const { ref, limit, offset } = options as RepositoryLogOptions; + if (ref !== undefined && typeof ref !== "string") { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: must be a string."); + } + if ( + limit !== undefined && + (!Number.isInteger(limit) || limit < 1 || limit > 1000) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 1000." + ); + } + if (offset !== undefined && (!Number.isInteger(offset) || offset < 0)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid offset: must be a non-negative integer." + ); + } +} + +async function createToken( + repository: Repository, + scope: TokenScope = "write", + ttl = 86_400 +): Promise { + if (scope !== "read" && scope !== "write") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts token scope must be read or write" + ); + } + if (!Number.isInteger(ttl) || ttl < 60 || ttl > 31_536_000) { + throw new ArtifactsError( + "INVALID_TTL", + `Invalid TTL ${ttl}: must be between 60 and 31536000 seconds.` + ); + } + const createdAt = new Date(); + const expiresAt = new Date(createdAt.getTime() + ttl * 1000); + const plaintext = `art_v2_x_${randomHex(20)}?expires=${Math.floor(expiresAt.getTime() / 1000)}`; + const record = { + id: randomBase36(16), + tokenHash: await hashToken(plaintext), + scope, + createdAt: createdAt.toISOString(), + expiresAt: expiresAt.toISOString(), + }; + repository.tokens.push(record); + return { record, plaintext }; +} + +function isActive(token: Pick): boolean { + return Date.parse(token.expiresAt) > Date.now(); +} + +function randomHex(bytes: number): string { + return [...crypto.getRandomValues(new Uint8Array(bytes))] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +async function hashToken(value: string): Promise { + const secret = value.split("?", 1)[0] ?? ""; + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(secret) + ); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function constantTimeEqual(left: string, right: string): boolean { + if (left.length !== right.length) { + return false; + } + let difference = 0; + for (let index = 0; index < left.length; index++) { + difference |= left.charCodeAt(index) ^ right.charCodeAt(index); + } + return difference === 0; +} + +function randomBase36(length: number): string { + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"; + return [...crypto.getRandomValues(new Uint8Array(length))] + .map((byte) => alphabet[byte % alphabet.length]) + .join(""); +} + +function isTokenId(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-z]{16}$/.test(value); +} + +function isArtifactToken(value: unknown): value is string { + return ( + typeof value === "string" && + /^art_v(?:1_[0-9a-f]{40}|2_[xeuf]_[0-9a-f]{40})(?:\?expires=\d+)?$/.test( + value + ) + ); +} + +function validateHash(hash: unknown): asserts hash is string { + if (!hash) { + throw new ArtifactsError("INVALID_INPUT", "Invalid hash: hash required."); + } + if (typeof hash !== "string" || !/^[0-9a-f]{40}$/.test(hash)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid hash: expected a 40-character SHA-1 hex hash." + ); + } +} + +function normalizePath(path: string): string | null { + const segments = path + .split("/") + .filter((segment) => segment !== "" && segment !== "."); + if (segments.includes("..") || segments.length === 0) { + return null; + } + return segments.join("/"); +} + +function contentType(path: string, bytes: Uint8Array): string { + const extension = path.toLowerCase().match(/\.[^./]+$/)?.[0]; + const known: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".ico": "image/x-icon", + ".pdf": "application/pdf", + }; + if (extension && known[extension]) { + return known[extension]; + } + return bytes.subarray(0, 8192).includes(0) + ? "application/octet-stream" + : "text/plain;charset=utf-8"; +} + +function decodeBase64(value: string): Uint8Array { + return Uint8Array.from(atob(value), (character) => character.charCodeAt(0)); +} + +function encodeCursor(offset: number): string { + return btoa(JSON.stringify({ offset })); +} + +function decodeCursor(cursor: string | undefined): number { + if (cursor === undefined) { + return 0; + } + try { + const value = JSON.parse(atob(cursor)) as { offset?: unknown }; + return typeof value.offset === "number" && + Number.isInteger(value.offset) && + value.offset >= 0 + ? value.offset + : 0; + } catch { + return 0; + } +} + +async function publicCall(action: () => Promise): Promise { + try { + return await action(); + } catch (error) { + if (isArtifactsError(error)) { + throw error; + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } +} + +function isArtifactsError(error: unknown): error is ArtifactsError { + return ( + error instanceof Error && + error.name === "ArtifactsError" && + "code" in error && + typeof error.code === "string" && + "numericCode" in error && + typeof error.numericCode === "number" + ); +} + +function classifyImportError(message: string): ArtifactsError { + const normalized = message.toLowerCase(); + if ( + normalized.includes("authentication failed") || + normalized.includes("could not read username") || + normalized.includes("http 401") + ) { + return new ArtifactsError( + "REMOTE_AUTH_REQUIRED", + "Authentication is required to access the remote repository." + ); + } + if ( + normalized.includes("remote branch") && + normalized.includes("not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote branch not found."); + } + if ( + normalized.includes("not found") || + normalized.includes("repository not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote repository not found."); + } + if (normalized.includes("does not appear to be a git repository")) { + return new ArtifactsError( + "INVALID_URL", + "The URL does not point to a git repository." + ); + } + return new ArtifactsError( + "UPSTREAM_UNAVAILABLE", + "The upstream service is unavailable. Please retry." + ); +} diff --git a/packages/miniflare/test/config/v4-convert.spec.ts b/packages/miniflare/test/config/v4-convert.spec.ts index bd6ca18210c..3e156aad756 100644 --- a/packages/miniflare/test/config/v4-convert.spec.ts +++ b/packages/miniflare/test/config/v4-convert.spec.ts @@ -4,6 +4,27 @@ import { convertV4MiniflareOptions } from "../../src/config/v4-convert"; import type { RemoteProxyConnectionString } from "../../src/plugins/shared"; describe("convertV4MiniflareOptions", () => { + test("Artifacts defaults to local and only an explicit connection is remote", ({ + expect, + }) => { + const remoteProxyConnectionString = new URL( + "http://127.0.0.1:8787" + ) as RemoteProxyConnectionString; + const converted = convertV4MiniflareOptions({ + name: "worker", + script: "export default {};", + artifacts: { + LOCAL: { namespace: "local" }, + REMOTE: { namespace: "remote", remoteProxyConnectionString }, + }, + }); + expect(converted.workers[0].config.env?.LOCAL).toMatchObject({ + dev: { remote: false }, + }); + expect(converted.workers[0].config.env?.REMOTE).toMatchObject({ + dev: { remote: true }, + }); + }); test("preserves opaque K2 stream IDs through v4 parsing and conversion", ({ expect, }) => { diff --git a/packages/miniflare/test/plugins/artifacts/contract.spec.ts b/packages/miniflare/test/plugins/artifacts/contract.spec.ts new file mode 100644 index 00000000000..35891f176b1 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract.spec.ts @@ -0,0 +1,263 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { singleModuleManifest, useDispose } from "../../test-shared"; +import { compare, normalize } from "./contract/compare"; +import { runApprovedLiveComparison } from "./contract/live"; +import { runScenarios } from "./contract/scenarios"; +import type { + Created, + ArtifactsBinding, + Observation, +} from "./contract/scenarios"; + +const exec = promisify(execFile); +const SCRIPT = `export default { fetch() { return new Response("contract"); } };`; + +async function prepareLocalFixture(created: Created): Promise { + const source = await mkdtemp(path.join(tmpdir(), "artifacts-contract-")); + const environment = { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + GIT_AUTHOR_DATE: "2020-01-02T03:04:05Z", + GIT_COMMITTER_DATE: "2020-01-02T03:04:05Z", + }; + const git = async (...args: string[]) => + exec("git", args, { cwd: source, env: environment }); + try { + await git("init", "--initial-branch=main"); + await mkdir(path.join(source, "folder")); + await writeFile(path.join(source, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(source, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await writeFile(path.join(source, "folder", "nested.txt"), "nested\n"); + await git("add", "."); + await git("commit", "-m", "first"); + await writeFile(path.join(source, "second.txt"), "second\n"); + await git("add", "."); + await git("commit", "-m", "second"); + // Keep the token out of argv, stderr, and test output. The child alone sees the header. + const pushEnv = { + ...environment, + GIT_CONFIG_COUNT: "2", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `Authorization: Bearer ${created.token}`, + }; + await exec("git", ["push", created.remote, "main"], { + cwd: source, + env: pushEnv, + }); + await git("checkout", "-b", "feature"); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git("add", "."); + await git("commit", "-m", "feature"); + await exec("git", ["push", created.remote, "feature"], { + cwd: source, + env: pushEnv, + }); + } finally { + await removeDir(source); + } +} + +function local(): Miniflare { + return new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts", namespace: "contract-local" } }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); +} + +test("shared Artifacts contract runs against the local RPC binding and cleans up", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const observations = await runScenarios( + REPOS, + "contract-test-", + prepareLocalFixture + ); + const find = (step: string) => + observations.find((item) => item.step === step); + expect(observations.length).toBeGreaterThan(50); + expect(find("file binary")).toMatchObject({ + outcome: "value", + value: { + blob: { type: "application/octet-stream", bytes: [0, 255, 128, 1] }, + }, + }); + expect(find("file text")).toMatchObject({ + outcome: "value", + value: { blob: { type: "text/plain;charset=utf-8" } }, + }); + expect(find("duplicate case")).toMatchObject({ outcome: "error" }); + expect(find("retained after delete")).toMatchObject({ outcome: "error" }); + for (const kind of ["blob", "tree", "commit"]) { + expect(find(`${kind} present`)).toMatchObject({ outcome: "value" }); + expect(find(`${kind} missing`)).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find(`${kind} wrong type`)).toMatchObject({ + outcome: "value", + value: null, + }); + } + expect(find("file missing ref")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("file missing path")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("list next page")).toMatchObject({ outcome: "value" }); + expect(find("token revoke wrong repo")).toMatchObject({ + outcome: "value", + value: false, + }); + expect(find("fork all feature")).toMatchObject({ + outcome: "value", + value: { blob: { bytes: [...new TextEncoder().encode("feature\n")] } }, + }); + expect(find("fork main feature")).toMatchObject({ + outcome: "value", + value: null, + }); + expect((await REPOS.list()).total).toBe(0); +}); + +test("comparator preserves exact errors, Blob data, paths and non-loopback URLs", ({ + expect, +}) => { + const base: Observation[] = [ + { + step: "file", + outcome: "value", + value: { blob: { bytes: [0, 255], type: "text/plain" } }, + }, + { + step: "error", + outcome: "error", + value: { + name: "ArtifactsError", + code: "NOT_FOUND", + numericCode: 10200, + message: "not found", + }, + }, + { + step: "repo", + outcome: "value", + value: { + id: "first", + createdAt: "2020-01-01T00:00:00Z", + remote: "http://127.0.0.1:1234/git/repo.git", + name: "repo", + }, + }, + ]; + const variable = structuredClone(base); + (variable[2].value as Record).id = "second"; + (variable[2].value as Record).createdAt = + "2026-01-01T00:00:00Z"; + (variable[2].value as Record).remote = + "http://127.0.0.1:5678/git/repo.git"; + expect(compare(base, variable)).toEqual([]); + const different = structuredClone(variable); + (different[0].value as { blob: { bytes: number[] } }).blob.bytes[1] = 254; + (different[1].value as Record).numericCode = 10201; + (different[2].value as Record).remote = + "https://example.test/git/repo.git"; + expect( + compare(base, different).map(({ step, path, kind }) => [step, path, kind]) + ).toEqual([ + ["file", "$.blob.bytes[1]", "value"], + ["error", "$.numericCode", "error"], + ["repo", "$.remote", "value"], + ]); + expect( + normalize([ + { + step: "err", + outcome: "error", + value: { message: "id first at 2020-01-01T00:00:00Z" }, + }, + ])[0].value + ).toEqual({ message: "id first at 2020-01-01T00:00:00Z" }); + expect(compare(base, base.slice(1))[0].kind).toBe("missing-step"); + expect(compare(base, [...base, base[0]])[0]).toMatchObject({ + kind: "missing-step", + path: "$[3]", + }); + const reordered = structuredClone(base); + reordered.reverse(); + expect(compare(base, reordered)[0].kind).toBe("missing-step"); + const changed = structuredClone(base); + (changed[0].value as { blob: { type: string } }).blob.type = + "application/octet-stream"; + expect(compare(base, changed)).toMatchObject([ + { path: "$.blob.type", kind: "value" }, + ]); +}); + +test("live comparison refuses unapproved or reused fixtures without calling either binding", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const options = { + namespace: "contract-local", + approvedDisposableNamespace: "not-approved", + fixturePrefix: "contract-test-", + approvedDisposableFixturePrefix: "contract-test-", + confirmation: "I_APPROVE_DISPOSABLE_ARTIFACTS_COMPARISON" as const, + local: REPOS, + live: REPOS, + prepareLocalFixture, + prepareLiveFixture: prepareLocalFixture, + }; + await expect(runApprovedLiveComparison(options)).rejects.toThrow(/approval/); + await expect( + runApprovedLiveComparison({ + ...options, + approvedDisposableNamespace: "contract-local", + }) + ).rejects.toThrow(/distinct/); + expect((await REPOS.list()).total).toBe(0); + await REPOS.create("contract-test-main"); + await expect( + runScenarios(REPOS, "contract-test-", prepareLocalFixture) + ).rejects.toThrow(/not empty/); + expect((await REPOS.list()).total).toBe(1); + await REPOS.delete("contract-test-main"); + await expect( + runApprovedLiveComparison({ + ...options, + approvedDisposableNamespace: "contract-local", + approvedDisposableFixturePrefix: "other-prefix-", + }) + ).rejects.toThrow(/approval/); +}); diff --git a/packages/miniflare/test/plugins/artifacts/contract/README.md b/packages/miniflare/test/plugins/artifacts/contract/README.md new file mode 100644 index 00000000000..54f4a81ecff --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/README.md @@ -0,0 +1,23 @@ +# Artifacts observable contract (draft) + +`../contract.spec.ts` runs `runScenarios()` against Miniflare's real RPC binding, with a deterministic, locally pushed Git fixture. The shared scenarios capture returned objects, errors (`name`, `message`, `code`, `numericCode`), Blob MIME types and bytes, and missing/null outcomes. They cover namespace create/get/list/delete (including case and pagination boundaries), retained handles after delete/recreate, tokens and wrong-repository revocation, Git objects/files/refs, and empty/branch-selective/read-only forks. Git HTTP authorization is covered separately in `../index.spec.ts`; this suite does not expose a public RPC HTTP endpoint. + +**Do not treat local observations as an approved remote specification.** The comparator reports field-level differences without updating either side. It normalizes only identity fields (`id`, `token`, `plaintext`), ISO timestamp fields, and the port of a loopback `remote` URL. In particular, error strings/codes, Blob bytes/types, Git hashes, cursor behavior, object fields, and non-loopback remote URLs remain exact. A non-loopback live remote compared to a loopback local remote will be reported, not silently accepted. Traces include credentials before comparison: do not print, persist, or upload raw observations. The comparison API returns normalized mismatches only. + +## Optional manual comparison (not run by CI) + +There is **no** standalone CLI, deploy script, automatic provisioning, authentication helper, or remote endpoint in this directory. After the namespace owner explicitly approves a **disposable** namespace and a unique fixture prefix, a trusted private test environment may obtain two real binding objects (local and live) and call `runApprovedLiveComparison` from `live.ts` in-process. Pass: + +- `namespace` and the identical `approvedDisposableNamespace` confirmed by the owner (the harness cannot verify binding-to-namespace mapping); +- `fixturePrefix` and the identical `approvedDisposableFixturePrefix` confirmed by the owner: a fresh lowercase prefix ending in `-` (for example, `contract-sample-`), reserved solely for this run. The preflight refuses a namespace containing _any_ repositories, because list counts and cursors are global; +- the literal `confirmation: "I_APPROVE_DISPOSABLE_ARTIFACTS_COMPARISON"`; +- `local`, `live`, and `prepareLocalFixture` / `prepareLiveFixture` callbacks. Each callback receives a fresh repo's `remote` and a short-lived write `token` and must push the same two main commits and one feature commit as the local fixture, using a protected credential channel (not argv, logs, or committed files). Supply bindings through an authorized Worker/test context, **not** an unrestricted public RPC fetch route. + +Never pass a production namespace, existing repository, shared prefix, or resource credential through a command line. The harness deletes only repository names whose create/fork RPC returned success in this run; it never deletes a namespace or provisions/deploys a Worker. A failed RPC may still have created a resource; inspect partial failures manually with owner approval. Concurrent writers to the same namespace are not safe: reserve the namespace for the duration of the run. Cleanup failures throw; inspect and remove only the named fixtures with owner approval. Do not run a live comparison without explicit authorization. The local spec is the only automatically executed path. + +## Unresolved contract decisions + +- Remote ownership/authorization, namespace-to-binding mapping, exclusive namespace reservation and disposal approval require an actual owner; none is implied by this test harness. +- Exact production behavior for missing/wrong-type Git objects, commit-hash refs, path normalization, malformed cursors, token visibility/revocation, and retained handles requires an approved remote run. Current local handle errors can differ between direct binding proxy and an in-Worker call; do not rewrite error observations to hide this. +- A returned production Git remote will not be normalized to the local loopback URL. Its difference needs an explicit owner decision, not a comparator exception. +- Local-only regression assertions are not evidence of live platform parity. No production calls have been made by this workstream. diff --git a/packages/miniflare/test/plugins/artifacts/contract/compare.ts b/packages/miniflare/test/plugins/artifacts/contract/compare.ts new file mode 100644 index 00000000000..71eabe1bf21 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/compare.ts @@ -0,0 +1,141 @@ +import type { Observation } from "./scenarios"; + +export interface Mismatch { + step: string; + path: string; + kind: "missing-step" | "outcome" | "error" | "shape" | "value"; + local: unknown; + live: unknown; +} + +/** Normalize only identity fields, timestamps and loopback origins; never object contents or error text. */ +export function normalize(trace: Observation[]): Observation[] { + const ids = new Map(); + function normalizeValue(value: unknown, key = ""): unknown { + if (Array.isArray(value)) { + return value.map((item) => normalizeValue(item)); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([field, item]) => [ + field, + normalizeValue(item, field), + ]) + ); + } + if (typeof value !== "string") { + return value; + } + if (["id", "token", "plaintext"].includes(key)) { + if (!ids.has(value)) { + ids.set(value, `<${key}:${ids.size + 1}>`); + } + return ids.get(value); + } + if ( + ["createdAt", "updatedAt", "lastPushAt", "expiresAt"].includes(key) && + /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z$/.test(value) + ) { + return "