From 22b8e10e83b83961b24fb6a685e6cbb1740b371d Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Tue, 6 Oct 2026 15:23:13 +0100 Subject: [PATCH 01/21] [miniflare] Add local Artifacts binding simulator --- .changeset/local-artifacts-binding.md | 10 + packages/miniflare/scripts/build.mjs | 8 +- packages/miniflare/src/index.ts | 16 + .../miniflare/src/plugins/artifacts/README.md | 133 ++ .../src/plugins/artifacts/controller.ts | 101 ++ .../src/plugins/artifacts/git-client.ts | 408 ++++++ .../src/plugins/artifacts/git-sidecar.ts | 583 ++++++++ .../miniflare/src/plugins/artifacts/index.ts | 181 ++- .../miniflare/src/plugins/shared/index.ts | 5 + .../src/workers/artifacts/binding.worker.ts | 1229 +++++++++++++++++ .../miniflare/test/config/v4-convert.spec.ts | 21 + .../test/plugins/artifacts/index.spec.ts | 811 +++++++++++ .../src/maybe-start-or-update-session.test.ts | 14 + .../src/config/binding-local-support.ts | 2 +- .../config/binding-local-support.test.ts | 6 + .../src/__tests__/dev/remote-bindings.test.ts | 32 + packages/wrangler/src/dev/miniflare/index.ts | 4 +- 17 files changed, 3538 insertions(+), 26 deletions(-) create mode 100644 .changeset/local-artifacts-binding.md create mode 100644 packages/miniflare/src/plugins/artifacts/README.md create mode 100644 packages/miniflare/src/plugins/artifacts/controller.ts create mode 100644 packages/miniflare/src/plugins/artifacts/git-client.ts create mode 100644 packages/miniflare/src/plugins/artifacts/git-sidecar.ts create mode 100644 packages/miniflare/src/workers/artifacts/binding.worker.ts create mode 100644 packages/miniflare/test/plugins/artifacts/index.spec.ts diff --git a/.changeset/local-artifacts-binding.md b/.changeset/local-artifacts-binding.md new file mode 100644 index 00000000000..e402cd20fa2 --- /dev/null +++ b/.changeset/local-artifacts-binding.md @@ -0,0 +1,10 @@ +--- +"miniflare": minor +"wrangler": minor +"@cloudflare/workers-utils": minor +"@cloudflare/remote-bindings": minor +--- + +Add a local Artifacts binding simulator backed by native Git for Worker development + +Artifacts bindings now use local, persistent repositories by default in development. Set `remote: true` on the binding to opt into the remote resource. The simulator supports the binding RPC methods and authenticated Git smart HTTP without requiring Cloudflare credentials for local use. diff --git a/packages/miniflare/scripts/build.mjs b/packages/miniflare/scripts/build.mjs index 56ba5b35265..516f094f304 100644 --- a/packages/miniflare/scripts/build.mjs +++ b/packages/miniflare/scripts/build.mjs @@ -446,7 +446,13 @@ async function buildPackage() { logLevel: watch ? "info" : "warning", outdir: outPath, outbase: pkgRoot, - entryPoints: [indexPath, ...fixtureBuilds], + entryPoints: [ + indexPath, + // A separate thread keeps native Git available during synchronous Node + // binding-proxy calls, which block Miniflare's main event loop. + path.join(pkgRoot, "src/plugins/artifacts/git-sidecar.ts"), + ...fixtureBuilds, + ], }; if (watch) { diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 1284d267928..b6451bd87a1 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -77,6 +77,7 @@ import { WORKER_BINDING_SERVICE_LOOPBACK, WORKFLOWS_PLUGIN_NAME, } from "./plugins"; +import { ArtifactsController } from "./plugins/artifacts/controller"; import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants"; import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version"; import { closeBrowserProcess } from "./plugins/browser-rendering/process"; @@ -888,6 +889,7 @@ export class Miniflare { #maybeInspectorProxyController?: InspectorProxyController; #previousRuntimeInspectorPort?: number; + #artifactsController = new ArtifactsController(); #containerPrivilegesCache = new ContainerPrivilegesCache(); #hyperdriveProxyController: HyperdriveProxyController = new HyperdriveProxyController(); @@ -2326,6 +2328,7 @@ export class Miniflare { queueConsumers, containerPrivilegesCache: this.#containerPrivilegesCache, hyperdriveProxyController: this.#hyperdriveProxyController, + artifactsController: this.#artifactsController, }; for (const [key, plugin] of this.#mergedPluginEntries) { const pluginServicesExtensions = await plugin.getServices({ @@ -2341,6 +2344,11 @@ export class Miniflare { } else { pluginServices = pluginServicesExtensions.services; extensions.push(...pluginServicesExtensions.extensions); + for (const socket of pluginServicesExtensions.sockets ?? []) { + if (!sockets.some((existing) => existing.name === socket.name)) { + sockets.push(socket); + } + } } for (const service of pluginServices) { @@ -3915,6 +3923,14 @@ export class Miniflare { } const runtimeCleanupOutcome = await runtimeDisposeOutcome; + try { + await this.#artifactsController.dispose(); + } catch (error) { + if (!independentCleanupFailed) { + independentCleanupFailed = true; + independentCleanupError = error; + } + } this.#devRegistry.unregisterWorkers(); try { await Promise.all( diff --git a/packages/miniflare/src/plugins/artifacts/README.md b/packages/miniflare/src/plugins/artifacts/README.md new file mode 100644 index 00000000000..52f8f0cf077 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/README.md @@ -0,0 +1,133 @@ +# Artifacts in Miniflare + +The Artifacts plugin implements the local Artifacts binding in workerd. A Node +worker thread runs the native Git backend. Unless `dev.remote` is explicitly +`true`, an Artifacts binding uses local storage rather than contacting Cloudflare. + +## Local startup + +```mermaid +sequenceDiagram + participant Dev as Developer tool + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Controller as ArtifactsController + participant Sidecar as Node Git sidecar + participant Workerd as workerd + + Dev->>MF: Configure Artifacts binding + MF->>Plugin: getBindings() + Plugin->>Plugin: Select local; validate namespace + Plugin-->>MF: Bind to local namespace entrypoint + MF->>Plugin: getServices() + Plugin->>Controller: get(Git storage path) + Controller->>Sidecar: Start worker thread and private HTTP listener + Sidecar-->>Controller: Address and secret + Controller-->>Plugin: Sidecar endpoint and Git socket port + Plugin-->>MF: Binding Worker, DO storage, backend service, Git socket + MF->>Workerd: Start configured services and socket +``` + +`getBindings()` routes each local binding to a namespace-scoped service. +`getServices()` creates the binding Worker, namespace Durable Object, persistent +metadata disk, internal Git backend service, and Git socket. The controller +reuses a sidecar per Git storage path and closes it when Miniflare is disposed. +The socket used by Git clients and the sidecar's private HTTP listener are +**different endpoints**. + +## Local binding RPC + +For example, `env.REPOS.create("demo")`: + +```mermaid +sequenceDiagram + participant App as User Worker + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Disk as Local disk + + App->>Entry: env.REPOS.create("demo") + Entry->>DO: RPC create("demo") + DO->>DO: Validate and serialize mutation + DO->>Backend: POST internal create request + Backend->>Sidecar: Inject private backend header + Sidecar->>Disk: git init --bare + Sidecar-->>DO: Git state (via backend service) + DO->>Disk: Persist metadata and token hash + DO-->>Entry: Repository info and initial token + Entry-->>App: Create result +``` + +The Durable Object stores repository metadata and token hashes on local disk; +the sidecar stores the bare Git repositories separately. `git-sidecar.ts` +handles HTTP routing and repository locks, while `git-client.ts` owns +repository-scoped Git commands and result parsing. Read operations such as +`readFile()` also use the binding Worker and native Git backend. Node callers +reach the same binding through Miniflare's Node binding proxy. + +## Git clone and push + +```mermaid +sequenceDiagram + participant Git as Git client + participant Socket as Git socket + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Native as Native Git + + Git->>Socket: Git smart HTTP at /git/:namespace/:repo.git + Socket->>Entry: Route request + Entry->>DO: fetch(request) + DO->>DO: Check endpoint, repository, token, and scope + DO->>Backend: Forward request without Authorization header + Backend->>Sidecar: Inject private backend header + Sidecar->>Native: Stream via git http-backend + Native-->>Sidecar: Discovery or pack response + Sidecar-->>DO: Git response (via backend service) + opt Successful receive-pack push + DO->>Backend: Read updated refs + Backend->>Sidecar: Internal refs request + Sidecar-->>DO: Refs (via backend service) + DO->>DO: Persist updated push metadata + end + DO-->>Entry: Git response + Entry-->>Socket: Git response + Socket-->>Git: Git response +``` + +The binding Worker checks the supported Git endpoints and authenticates +repository tokens before forwarding. The internal service injects the sidecar +secret; the client's `Authorization` header is removed before that hop. For a +successful push, the Durable Object refreshes repository metadata before +returning the response. The sidecar uses native `git http-backend` for Git smart +HTTP rather than reproducing Git's pack protocol. + +## Explicit remote binding + +When `dev.remote` is `true`, this binding uses the remote-bindings proxy instead +of creating local Artifacts storage or a local Git sidecar for that namespace. + +```mermaid +sequenceDiagram + participant App as User Worker + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Proxy as Remote-bindings proxy + participant CF as Cloudflare Artifacts + + MF->>Plugin: getBindings() with remote: true + Plugin-->>MF: Bind to remote proxy service + MF->>Proxy: Start remote proxy Worker + App->>Proxy: env.REPOS method call + Proxy->>CF: Remote binding request + CF-->>Proxy: Result + Proxy-->>App: Result +``` + +A local binding does not automatically synchronize with production Artifacts. +An explicit repository import may contact its specified HTTPS Git remote; that +is distinct from enabling a remote Artifacts binding. diff --git a/packages/miniflare/src/plugins/artifacts/controller.ts b/packages/miniflare/src/plugins/artifacts/controller.ts new file mode 100644 index 00000000000..5225c1107f8 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/controller.ts @@ -0,0 +1,101 @@ +import { once } from "node:events"; +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import getPort from "get-port"; +import type { GitSidecar } from "./git-sidecar"; + +export interface LocalArtifactsBackend { + sidecar: GitSidecar; + port: number; +} + +// Node binding proxies make synchronous calls. The Git backend must run on +// another thread, otherwise those calls prevent it from answering workerd. +export class ArtifactsController { + #backends = new Map>(); + + get(root: string): Promise { + let backend = this.#backends.get(root); + if (!backend) { + backend = this.#start(root); + this.#backends.set(root, backend); + void backend.catch(() => this.#backends.delete(root)); + } + return backend; + } + + async #start(root: string): Promise { + const port = await getPort({ host: "127.0.0.1" }); + // __dirname is dist/src in the bundled Miniflare entry point. + const worker = new Worker( + path.join(__dirname, "plugins/artifacts/git-sidecar.js"), + { workerData: { root } } + ); + try { + const endpoint = await waitForSidecarReady(worker); + worker.on("error", (error) => { + process.emitWarning( + `Local Artifacts Git backend failed: ${error.message}` + ); + }); + return { + port, + sidecar: { + ...endpoint, + async close() { + if (worker.threadId === -1) { + return; + } + const exited = once(worker, "exit"); + worker.postMessage("close"); + await exited; + }, + }, + }; + } catch (error) { + await worker.terminate(); + throw error; + } + } + + async dispose(): Promise { + const backends = [...this.#backends.values()]; + this.#backends.clear(); + await Promise.all( + backends.map((backend) => + backend.then( + ({ sidecar }) => sidecar.close(), + () => undefined + ) + ) + ); + } +} + +function waitForSidecarReady( + worker: Worker +): Promise> { + return new Promise((resolve, reject) => { + function cleanup() { + worker.off("message", ready); + worker.off("error", failed); + worker.off("exit", exited); + } + function ready(value: Pick) { + cleanup(); + resolve(value); + } + function failed(error: Error) { + cleanup(); + reject(error); + } + function exited(code: number) { + failed( + new Error(`Local Artifacts Git backend exited during startup (${code})`) + ); + } + worker.on("message", ready); + worker.on("error", failed); + worker.on("exit", exited); + }); +} diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts new file mode 100644 index 00000000000..1bd680a818c --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -0,0 +1,408 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { devNull } from "node:os"; + +const decoder = new TextDecoder(); +type GitOptions = { allowFailure?: boolean; timeout?: number }; +type GitResult = { stdout: Buffer; stderr: Buffer; status: number }; + +// Git runs on the host, never in the user's Worker. Do not inherit Cloudflare +// tokens, Git config overrides, or credential helpers from the invoking shell. +export function gitEnvironment( + source: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv { + const keep = [ + "PATH", + "SystemRoot", + "WINDIR", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + "LANG", + ]; + const env: NodeJS.ProcessEnv = {}; + for (const key of keep) { + const value = source[key]; + if (value !== undefined) { + env[key] = value; + } + } + return { + ...env, + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: devNull, + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "credential.helper", + GIT_CONFIG_VALUE_0: "", + GIT_TERMINAL_PROMPT: "0", + }; +} + +export async function runGit( + args: string[], + options: GitOptions = {} +): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", args, { + env: gitEnvironment(), + stdio: ["pipe", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + const timeout = setTimeout(() => child.kill(), options.timeout ?? 300_000); + timeout.unref(); + child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.once("close", (status) => { + clearTimeout(timeout); + const result = { + stdout: Buffer.concat(stdout), + stderr: Buffer.concat(stderr), + status: status ?? 1, + }; + if (result.status === 0 || options.allowFailure) { + resolve(result); + } else { + reject( + new Error( + decoder.decode(result.stderr).trim() || + `git ${args[0]} exited with status ${result.status}` + ) + ); + } + }); + child.stdin.end(); + }); +} + +export interface RepositoryState { + defaultBranch: string; + refs: Record; +} + +export interface CommitMetadata { + hash: string; + treeHash: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + parents: string[]; + authoredAt: number; + committedAt: number; +} + +/** Repository-scoped Git operations; clone/init are supported before it exists. */ +export class GitClient { + constructor(readonly repository: string) {} + + private git(args: string[], options?: GitOptions): Promise { + return runGit(["-C", this.repository, ...args], options); + } + + async init(defaultBranch: string): Promise { + await runGit([ + "init", + "--bare", + `--initial-branch=${defaultBranch}`, + this.repository, + ]); + } + + async forkFrom( + source: GitClient, + branch: string, + defaultBranchOnly: boolean + ): Promise { + const sourceBranch = `refs/heads/${branch}`; + const sourceHasBranch = await source.git( + ["rev-parse", "--verify", "--quiet", sourceBranch], + { allowFailure: true } + ); + if (defaultBranchOnly && sourceHasBranch.status === 0) { + await runGit([ + "clone", + "--bare", + "--local", + "--single-branch", + "--branch", + branch, + source.repository, + this.repository, + ]); + } else if (defaultBranchOnly) { + // Git cannot clone a named branch that has no commits yet. + await this.init(branch); + } else { + await runGit([ + "clone", + "--bare", + "--local", + source.repository, + this.repository, + ]); + } + } + + async importFrom( + url: string, + branch?: string, + depth?: number + ): Promise { + const args = ["clone", "--bare"]; + if (branch) { + args.push("--branch", branch, "--single-branch"); + } + if (depth) { + args.push("--depth", String(depth)); + } + args.push(url, this.repository); + await runGit(args); + } + + async configure(generation: string): Promise { + await this.git(["config", "http.receivepack", "true"]); + await this.git(["config", "local-artifacts.generation", generation]); + } + + async generation(): Promise { + const result = await this.git( + ["config", "--get", "local-artifacts.generation"], + { allowFailure: true } + ); + return result.status === 0 ? decoder.decode(result.stdout).trim() : null; + } + + async state(): Promise { + const refsOutput = await this.git([ + "for-each-ref", + "--format=%(refname)%00%(objectname)", + ]); + const refs: Record = {}; + for (const line of decoder.decode(refsOutput.stdout).trim().split("\n")) { + if (!line) { + continue; + } + const [name, oid] = line.split("\0"); + if (name && oid) { + refs[name] = oid; + } + } + const head = await this.git(["symbolic-ref", "--short", "HEAD"], { + allowFailure: true, + }); + const defaultBranch = + decoder + .decode(head.stdout) + .trim() + .replace(/^refs\/heads\//, "") || "main"; + return { defaultBranch, refs }; + } + + async readBlob(hash: string): Promise<{ data: string } | null> { + if ((await this.objectType(hash)) !== "blob") { + return null; + } + const object = await this.git(["cat-file", "blob", hash]); + return { data: Buffer.from(object.stdout).toString("base64") }; + } + + async readTree(hash: string): Promise | null> { + if ((await this.objectType(hash)) !== "tree") { + return null; + } + const tree = await this.git(["ls-tree", "-z", hash]); + return decoder + .decode(tree.stdout) + .split("\0") + .filter(Boolean) + .map((line) => { + const match = /^(\d+) \w+ ([0-9a-f]{40})\t([\s\S]+)$/.exec(line); + if (!match?.[1] || !match[2] || match[3] === undefined) { + throw new Error("Invalid git tree output"); + } + const mode = match[1] === "040000" ? "40000" : match[1]; + return { + name: match[3], + mode, + hash: match[2], + type: treeEntryType(mode), + }; + }); + } + + async readCommit(hash: string): Promise { + if ((await this.objectType(hash)) !== "commit") { + return null; + } + const object = await this.git(["cat-file", "commit", hash]); + return parseCommit(hash, decoder.decode(object.stdout)); + } + + async readFile(ref: string, path: string): Promise<{ data: string } | null> { + const commit = await this.resolveCommit(ref); + if (!commit) { + return null; + } + const entry = await this.git(["ls-tree", "-z", commit, "--", path], { + allowFailure: true, + }); + if (entry.status !== 0 || entry.stdout.length === 0) { + return null; + } + const line = decoder.decode(entry.stdout).split("\0", 1)[0] ?? ""; + const match = /^(\d+) \w+ [0-9a-f]{40}\t([\s\S]+)$/.exec(line); + if (!match?.[1] || match[2] !== path || match[1] === "040000") { + return null; + } + const object = await this.git(["cat-file", "blob", `${commit}:${path}`], { + allowFailure: true, + }); + return object.status === 0 + ? { data: Buffer.from(object.stdout).toString("base64") } + : null; + } + + async log(ref?: string, limit = 50, offset = 0): Promise { + const commit = await this.resolveCommit(ref ?? "HEAD", ref === undefined); + if (!commit) { + return []; + } + const effectiveLimit = Math.min(limit, 1000); + const revs = await this.git( + [ + "rev-list", + "--first-parent", + `--max-count=${effectiveLimit}`, + `--skip=${offset}`, + commit, + ], + { allowFailure: true } + ); + if (revs.status !== 0) { + return []; + } + const hashes = decoder + .decode(revs.stdout) + .trim() + .split("\n") + .filter(Boolean); + return Promise.all( + hashes.map(async (hash) => { + const commitMetadata = await this.readCommit(hash); + if (!commitMetadata) { + throw new Error(`Commit disappeared: ${hash}`); + } + return commitMetadata; + }) + ); + } + + async rootCommits(): Promise> { + const result = await this.git(["rev-list", "--max-parents=0", "--all"]); + return new Set( + decoder.decode(result.stdout).trim().split("\n").filter(Boolean) + ); + } + + private async objectType(hash: string): Promise { + const result = await this.git(["cat-file", "-t", hash], { + allowFailure: true, + }); + return result.status === 0 ? decoder.decode(result.stdout).trim() : null; + } + + private async resolveCommit( + ref: string, + allowHead = false + ): Promise { + const candidates = allowHead + ? ["HEAD"] + : /^[0-9a-f]{40}$/.test(ref) + ? [ref] + : [`refs/heads/${ref}`, `refs/tags/${ref}`]; + for (const candidate of candidates) { + const result = await this.git( + ["rev-parse", "--verify", "--quiet", `${candidate}^{commit}`], + { allowFailure: true } + ); + if (result.status === 0) { + return decoder.decode(result.stdout).trim(); + } + } + return null; + } +} + +function commitHeader(headers: string[], name: string): string | undefined { + return headers + .find((line) => line.startsWith(`${name} `)) + ?.slice(name.length + 1); +} + +function parseCommitIdentity( + headers: string[], + name: string +): { + person: { name: string; email: string }; + time: number; +} { + const match = /^(.*) <([^<>]*)> (\d+) [+-]\d{4}$/.exec( + commitHeader(headers, name) ?? "" + ); + if (!match?.[1] || match[2] === undefined || !match[3]) { + throw new Error(`Invalid commit ${name}`); + } + return { + person: { name: match[1], email: match[2] }, + time: Number(match[3]), + }; +} + +function parseCommit(hash: string, value: string): CommitMetadata { + const split = value.indexOf("\n\n"); + const headers = value.slice(0, split < 0 ? value.length : split).split("\n"); + const author = parseCommitIdentity(headers, "author"); + const committer = parseCommitIdentity(headers, "committer"); + const treeHash = commitHeader(headers, "tree"); + if (!treeHash) { + throw new Error("Invalid commit tree"); + } + return { + hash, + treeHash, + message: (split < 0 ? "" : value.slice(split + 2)).replace(/\n$/, ""), + author: author.person, + committer: committer.person, + parents: headers + .filter((line) => line.startsWith("parent ")) + .map((line) => line.slice(7)), + authoredAt: author.time, + committedAt: committer.time, + }; +} + +function treeEntryType(mode: string): string { + if (mode === "40000") { + return "tree"; + } + if (mode === "120000") { + return "symlink"; + } + if (mode === "160000") { + return "gitlink"; + } + if (mode === "100755") { + return "exec"; + } + return "blob"; +} diff --git a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts new file mode 100644 index 00000000000..26b0cf3edde --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts @@ -0,0 +1,583 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { mkdir, stat } from "node:fs/promises"; +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import { join } from "node:path"; +import { parentPort, workerData } from "node:worker_threads"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { GitClient, gitEnvironment, runGit } from "./git-client"; +import type { RepositoryState } from "./git-client"; + +const adminPath = "/__local_artifacts__"; +const componentPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const repositoryLocks = new Map>(); +const decoder = new TextDecoder(); + +if (parentPort) { + const port = parentPort; + void startGitSidecar(workerData.root).then((sidecar) => { + port.postMessage({ address: sidecar.address, secret: sidecar.secret }); + port.once("message", async () => { + await sidecar.close(); + port.close(); + }); + }); +} + +export interface GitSidecar { + address: string; + secret: string; + close(): Promise; +} + +interface AdminRequest { + action: + | "create" + | "delete" + | "fork" + | "import" + | "refs" + | "readBlob" + | "readTree" + | "readCommit" + | "file" + | "log"; + namespace: string; + name: string; + defaultBranch?: string; + defaultBranchOnly?: boolean; + sourceName?: string; + sourceBranch?: string; + sourceUrl?: string; + branch?: string; + depth?: number; + hash?: string; + ref?: string; + path?: string; + limit?: number; + offset?: number; + generation?: string; +} + +function createGitSecret(): string { + return randomBytes(32).toString("hex"); +} + +export async function startGitSidecar(root: string): Promise { + await mkdir(root, { recursive: true }); + await runGit(["--version"]); + const secret = createGitSecret(); + const server = createServer((request, response) => { + if (request.headers["x-local-artifacts-backend"] !== secret) { + response.writeHead(403).end("Forbidden"); + return; + } + void handleRequest(root, request, response).catch((error: unknown) => { + if (response.headersSent) { + response.destroy(error instanceof Error ? error : undefined); + } else { + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end( + error instanceof Error ? error.message : "Native Git backend failed" + ); + } + }); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + // The listener outlives this function; ArtifactsController closes it on disposal. + // `using` here would close it before workerd could make any requests. + server.unref(); + const addressInfo = server.address(); + if (!addressInfo || typeof addressInfo === "string") { + throw new Error("Unable to start native Git backend"); + } + const address = `127.0.0.1:${addressInfo.port}`; + return { + address, + secret, + close() { + server.closeAllConnections(); + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +function parseGitPath(pathname: string): { + namespace: string; + repository: string; + suffix: string; +} | null { + const match = pathname.match(/^\/git\/([^/]+)\/([^/]+\.git)(\/.*)?$/); + if (!match?.[1] || !match[2]) { + return null; + } + const namespace = decodeURIComponent(match[1]); + const repository = decodeURIComponent(match[2].slice(0, -4)); + validateComponent(namespace, "namespace"); + validateComponent(repository, "repository"); + return { namespace, repository, suffix: match[3] ?? "" }; +} + +// Native Git's CGI backend needs streaming Node requests and responses; keep +// packfiles on node:http rather than buffering them through a Worker router. +async function handleRequest( + root: string, + request: IncomingMessage, + response: ServerResponse +): Promise { + const url = new URL(request.url ?? "/", "http://local-artifacts.invalid"); + if (url.pathname === adminPath) { + await handleAdmin(root, request, response); + return; + } + const gitPath = parseGitPath(url.pathname); + if (!gitPath) { + response.writeHead(404).end("Not found"); + return; + } + const { namespace, repository, suffix } = gitPath; + const contentLength = Number(request.headers["content-length"] ?? 0); + if (!Number.isFinite(contentLength) || contentLength > 256 * 1024 * 1024) { + response.writeHead(413).end("Request body too large"); + return; + } + const path = repositoryPath(root, namespace, repository); + await withRepositoryLocks([path], async () => { + const generation = request.headers["x-local-artifacts-generation"]; + const configured = await new GitClient(path).generation(); + if (typeof generation !== "string" || generation !== configured) { + response.writeHead(409).end("Repository generation changed"); + return; + } + await runGitHttpBackend( + root, + path, + request, + response, + `/${storageComponent(namespace)}/${storageComponent(repository)}.git${suffix}`, + url.search.slice(1) + ); + }); +} + +async function handleAdmin( + root: string, + request: IncomingMessage, + response: ServerResponse +): Promise { + if (request.method !== "POST") { + response.writeHead(405).end("Method not allowed"); + return; + } + const body = JSON.parse( + decoder.decode(await readRequest(request)) + ) as AdminRequest; + validateComponent(body.namespace, "namespace"); + validateComponent(body.name, "repository"); + const repository = repositoryPath(root, body.namespace, body.name); + const lockPaths = [repository]; + if (body.action === "fork" && body.sourceName) { + validateComponent(body.sourceName, "repository"); + lockPaths.push(repositoryPath(root, body.namespace, body.sourceName)); + } + await withRepositoryLocks(lockPaths, async () => { + const result = await handleAction(root, body, new GitClient(repository)); + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(result)); + }); +} + +async function handleAction( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + switch (body.action) { + case "create": + return handleCreate(root, body, git); + case "delete": + await removeDir(git.repository); + return true; + case "fork": + return handleFork(root, body, git); + case "import": + return handleImport(root, body, git); + case "refs": + return git.state(); + case "readBlob": + return git.readBlob(required(body.hash, "hash")); + case "readTree": + return git.readTree(required(body.hash, "hash")); + case "readCommit": + return git.readCommit(required(body.hash, "hash")); + case "file": + return git.readFile( + required(body.ref, "ref"), + required(body.path, "path") + ); + case "log": + return git.log(body.ref, body.limit, body.offset); + } +} + +function handleCreate( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + return withNewRepository(root, body, git, () => + git.init(body.defaultBranch ?? "main") + ); +} + +function handleFork( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + const { sourceName, sourceBranch } = body; + if (!sourceName) { + throw new Error("Fork source is required"); + } + if (!sourceBranch) { + throw new Error("Fork source branch is required"); + } + const source = new GitClient( + repositoryPath(root, body.namespace, sourceName) + ); + return withNewRepository(root, body, git, () => + git.forkFrom(source, sourceBranch, body.defaultBranchOnly ?? true) + ); +} + +function handleImport( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + const { sourceUrl } = body; + if (!sourceUrl) { + throw new Error("Import source is required"); + } + return withNewRepository(root, body, git, () => + git.importFrom(sourceUrl, body.branch, body.depth) + ); +} + +async function withNewRepository( + root: string, + body: AdminRequest, + git: GitClient, + initialize: () => Promise +): Promise { + await mkdir(repositoryNamespacePath(root, body.namespace), { + recursive: true, + }); + if (await exists(git.repository)) { + throw new Error(`Repository "${body.name}" already exists on disk`); + } + try { + await initialize(); + await git.configure(required(body.generation, "generation")); + return await git.state(); + } catch (error) { + await removeDir(git.repository); + throw error; + } +} + +function required(value: string | undefined, name: string): string { + if (!value) { + throw new Error(`${name} is required`); + } + return value; +} + +function parseGitBackendHeaders(headerBytes: Buffer): { + status: number; + headers: Record; +} { + const lines = decoder.decode(headerBytes).split(/\r?\n/); + let status = 200; + const headers: Record = {}; + for (const line of lines) { + const separator = line.indexOf(":"); + if (separator < 0) { + continue; + } + const name = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + if (name.toLowerCase() === "status") { + status = Number(value.slice(0, 3)); + } else { + headers[name] = value; + } + } + return { status, headers }; +} + +async function runGitHttpBackend( + root: string, + repository: string, + request: IncomingMessage, + response: ServerResponse, + pathInfo: string, + query: string +): Promise { + const rootCommits = pathInfo.endsWith("/git-upload-pack") + ? await new GitClient(repository).rootCommits() + : new Set(); + const shallowFilter = filterRootShallowPackets(rootCommits); + return new Promise((resolve, reject) => { + const child = spawn("git", ["http-backend"], { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: root, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: pathInfo, + QUERY_STRING: query, + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + REMOTE_USER: "local-artifacts", + REMOTE_ADDR: request.socket.remoteAddress ?? "127.0.0.1", + }, + stdio: ["pipe", "pipe", "pipe"], + }); + let header = Buffer.alloc(0); + let headersSent = false; + let settled = false; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill(); + }, 300_000); + timeout.unref(); + const stderr: Buffer[] = []; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + request.off("aborted", abort); + response.off("close", abort); + if (error) { + reject(error); + } else { + resolve(); + } + }; + const abort = () => { + if (!response.writableEnded) { + child.kill(); + } + }; + const write = (chunk: Buffer) => { + if (!response.write(chunk)) { + child.stdout.pause(); + response.once("drain", () => child.stdout.resume()); + } + }; + const writeBody = (chunk: Buffer) => { + const filtered = shallowFilter.write(chunk); + if (filtered.length) { + write(filtered); + } + }; + request.once("aborted", abort); + response.once("close", abort); + child.stdin.on("error", (error) => { + if (!request.destroyed) { + finish(error); + } + }); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.stdout.on("data", (chunk: Buffer) => { + if (headersSent) { + writeBody(chunk); + return; + } + header = Buffer.concat([header, chunk]); + const crlf = header.indexOf("\r\n\r\n"); + const lf = header.indexOf("\n\n"); + const index = crlf >= 0 ? crlf : lf; + if (index < 0) { + if (header.length > 64 * 1024) { + child.kill(); + } + return; + } + const separatorLength = crlf >= 0 ? 4 : 2; + const { status, headers } = parseGitBackendHeaders( + header.subarray(0, index) + ); + response.writeHead(status, headers); + headersSent = true; + const remaining = header.subarray(index + separatorLength); + if (remaining.length) { + writeBody(remaining); + } + header = Buffer.alloc(0); + }); + child.stdout.on("end", () => { + const remaining = shallowFilter.end(); + if (remaining.length) { + write(remaining); + } + if (headersSent) { + response.end(); + } + }); + child.once("error", finish); + child.once("close", (status) => { + if (timedOut) { + finish(new Error("git http-backend timed out")); + return; + } + if (!headersSent) { + finish( + new Error( + decoder.decode(Buffer.concat(stderr)).trim() || + `git http-backend exited with status ${status}` + ) + ); + } else { + finish(); + } + }); + request.pipe(child.stdin); + }); +} + +function filterRootShallowPackets(rootCommits: Set): { + write(chunk: Buffer): Buffer; + end(): Buffer; +} { + let pending = Buffer.alloc(0); + let passthrough = rootCommits.size === 0; + return { + write(chunk) { + if (passthrough) { + return chunk; + } + pending = Buffer.concat([pending, chunk]); + const output: Buffer[] = []; + while (pending.length >= 4) { + const prefix = pending.subarray(0, 4).toString("ascii"); + if (!/^[0-9a-f]{4}$/i.test(prefix)) { + passthrough = true; + output.push(pending); + pending = Buffer.alloc(0); + break; + } + const length = Number.parseInt(prefix, 16); + if (length === 0 || length === 1 || length === 2) { + output.push(pending.subarray(0, 4)); + pending = pending.subarray(4); + continue; + } + if (length < 4 || pending.length < length) { + break; + } + const packet = pending.subarray(0, length); + const payload = packet.subarray(4).toString("ascii"); + const shallowHash = /^shallow ([0-9a-f]{40})\n?$/.exec(payload)?.[1]; + if (!shallowHash || !rootCommits.has(shallowHash)) { + output.push(packet); + } + pending = pending.subarray(length); + } + return Buffer.concat(output); + }, + end() { + const result = pending; + pending = Buffer.alloc(0); + return result; + }, + }; +} + +function readRequest(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => resolve(Buffer.concat(chunks))); + request.once("error", reject); + }); +} + +function repositoryPath(root: string, namespace: string, name: string): string { + return join( + repositoryNamespacePath(root, namespace), + `${storageComponent(name)}.git` + ); +} + +function repositoryNamespacePath(root: string, namespace: string): string { + return join(root, storageComponent(namespace)); +} + +function storageComponent(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function validateComponent(value: string, type: string): void { + if (!componentPattern.test(value)) { + throw new Error(`Invalid ${type}`); + } +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +} + +async function withRepositoryLocks( + paths: string[], + action: () => Promise +): Promise { + const keys = [...new Set(paths)].sort(); + const previous = keys.map( + (key) => repositoryLocks.get(key) ?? Promise.resolve() + ); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + for (const key of keys) { + repositoryLocks.set(key, current); + } + await Promise.all(previous); + try { + return await action(); + } finally { + release(); + for (const key of keys) { + if (repositoryLocks.get(key) === current) { + repositoryLocks.delete(key); + } + } + } +} diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 48242920859..737455ced87 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -1,31 +1,84 @@ +import { createHash } from "node:crypto"; +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import SCRIPT from "worker:artifacts/binding"; import { buildRemoteProxyProps, getEnvBindingsOfType, + getPersistPath, getRemoteProxyConnectionString, ProxyNodeBinding, remoteProxyClientWorker, } from "../shared"; -import type { Plugin } from "../shared"; +import type { Service, Socket } from "../../runtime"; +import type { MiniflareBinding, Plugin } from "../shared"; +import type { ArtifactsController } from "./controller"; + +export { ArtifactsController, type LocalArtifactsBackend } from "./controller"; +export type { GitSidecar } from "./git-sidecar"; export const ARTIFACTS_PLUGIN_NAME = "artifacts"; -// One shared remote-proxy service for every artifacts binding; per-binding -// config travels via props. const ARTIFACTS_REMOTE_SERVICE_NAME = `${ARTIFACTS_PLUGIN_NAME}:remote`; +const ENTRYPOINT = "LocalArtifactsNamespace"; +const OBJECT_CLASS = "LocalArtifactsNamespaceObject"; +const LOCAL_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +type ArtifactsBinding = Extract; +type ArtifactsEntry = [name: string, binding: ArtifactsBinding]; + +function isLocalArtifactsBinding(binding: ArtifactsBinding): boolean { + return binding.dev?.remote !== true; +} + +function getLocalNamespaceNames(bindings: ArtifactsEntry[]): Set { + return new Set( + bindings + .filter(([, binding]) => isLocalArtifactsBinding(binding)) + .map(([, binding]) => binding.namespace) + ); +} + +function validateLocalNamespace(namespace: string): void { + if (!LOCAL_NAMESPACE_PATTERN.test(namespace)) { + throw new Error(`Invalid local Artifacts namespace: ${namespace}`); + } +} + +function namespaceId(namespace: string): string { + return createHash("sha256").update(namespace).digest("hex"); +} + +function localServiceName(namespace: string): string { + return `${ARTIFACTS_PLUGIN_NAME}:${namespaceId(namespace)}`; +} export const ARTIFACTS_PLUGIN: Plugin = { bindingTypeDescription: "Artifacts", - async getBindings(options) { + getBindings(options) { return getEnvBindingsOfType(options.config, "artifacts").map( - ([name, binding]) => ({ - name, - service: { - name: ARTIFACTS_REMOTE_SERVICE_NAME, - props: buildRemoteProxyProps( - getRemoteProxyConnectionString(binding, options.dev), - name - ), - }, - }) + ([name, binding]) => { + // Offline by default; remote proxying requires explicit opt-in. + if (isLocalArtifactsBinding(binding)) { + validateLocalNamespace(binding.namespace); + return { + name, + service: { + name: localServiceName(binding.namespace), + entrypoint: ENTRYPOINT, + }, + }; + } + return { + name, + service: { + name: ARTIFACTS_REMOTE_SERVICE_NAME, + props: buildRemoteProxyProps( + getRemoteProxyConnectionString(binding, options.dev), + name + ), + }, + }; + } ); }, getNodeBindings(options) { @@ -36,16 +89,98 @@ export const ARTIFACTS_PLUGIN: Plugin = { ]) ); }, - async getServices({ options }) { - if (getEnvBindingsOfType(options.config, "artifacts").length === 0) { - return []; - } - - return [ - { + async getServices({ options, sharedOptions, tmpPath, artifactsController }) { + const bindings = getEnvBindingsOfType(options.config, "artifacts"); + const services: Service[] = []; + const sockets: Socket[] = []; + if (bindings.some(([, binding]) => !isLocalArtifactsBinding(binding))) { + services.push({ name: ARTIFACTS_REMOTE_SERVICE_NAME, worker: remoteProxyClientWorker(), - }, - ]; + }); + } + const namespaces = getLocalNamespaceNames(bindings); + const root = getPersistPath( + ARTIFACTS_PLUGIN_NAME, + tmpPath, + sharedOptions.resourcePersistencePath + ); + + for (const namespace of namespaces) { + const local = await createLocalNamespaceRuntime( + root, + namespace, + artifactsController + ); + sockets.push(local.socket); + services.push(...local.services); + } + return { services, extensions: [], sockets }; }, }; + +async function createLocalNamespaceRuntime( + root: string, + namespace: string, + artifactsController: ArtifactsController +): Promise<{ services: Service[]; socket: Socket }> { + const persistPath = path.join(root, namespaceId(namespace)); + const metadataPath = path.join(persistPath, "metadata").replaceAll("\\", "/"); + await mkdir(metadataPath, { recursive: true }); + const { sidecar, port } = await artifactsController.get( + path.join(persistPath, "git") + ); + const serviceName = localServiceName(namespace); + const storageName = `${serviceName}:storage`; + const backendName = `${serviceName}:backend`; + const socket: Socket = { + name: `${serviceName}:git`, + address: `127.0.0.1:${port}`, + http: {}, + service: { name: serviceName, entrypoint: ENTRYPOINT }, + }; + const storageService: Service = { + name: storageName, + disk: { path: metadataPath, writable: true }, + }; + const gitBackendService: Service = { + name: backendName, + external: { + address: sidecar.address, + http: { + injectRequestHeaders: [ + { name: "X-Local-Artifacts-Backend", value: sidecar.secret }, + ], + }, + }, + }; + const bindingService: Service = { + name: serviceName, + worker: { + compatibilityDate: "2026-09-03", + modules: [{ name: "binding.worker.js", esModule: SCRIPT() }], + bindings: [ + { + name: "config", + json: JSON.stringify({ + namespace, + origin: `http://127.0.0.1:${port}`, + }), + }, + { + name: "localArtifactsNamespace", + durableObjectNamespace: { className: OBJECT_CLASS }, + }, + { name: "gitBackend", service: { name: backendName } }, + ], + durableObjectNamespaces: [ + { className: OBJECT_CLASS, uniqueKey: serviceName }, + ], + durableObjectStorage: { localDisk: storageName }, + }, + }; + return { + services: [storageService, gitBackendService, bindingService], + socket, + }; +} diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 9c0658a0c3b..578f1a1deca 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -12,6 +12,7 @@ import type { Extension, Service, ServiceDesignator, + Socket, Worker_Binding, Worker_Module, } from "../../runtime"; @@ -21,6 +22,7 @@ import type { QueueConsumerSchema, QueueProducerSchema, } from "../../workers"; +import type { ArtifactsController } from "../artifacts/controller"; import type { ContainerPrivilegesCache } from "../core/container"; import type { DOContainerOptions } from "../do"; import type { HyperdriveProxyController } from "../hyperdrive/hyperdrive-proxy"; @@ -88,11 +90,14 @@ export interface PluginServicesOptions { devRegistryEnabled: boolean; containerPrivilegesCache: ContainerPrivilegesCache; hyperdriveProxyController: HyperdriveProxyController; + artifactsController: ArtifactsController; } export interface ServicesExtensions { services: Service[]; extensions: Extension[]; + // Internal plugin listeners (e.g. Git smart HTTP), deduplicated by name. + sockets?: Socket[]; } /** diff --git a/packages/miniflare/src/workers/artifacts/binding.worker.ts b/packages/miniflare/src/workers/artifacts/binding.worker.ts new file mode 100644 index 00000000000..075ea9b4aeb --- /dev/null +++ b/packages/miniflare/src/workers/artifacts/binding.worker.ts @@ -0,0 +1,1229 @@ +import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; + +const repoName = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +type TokenScope = "read" | "write"; + +type RepositorySource = + | { type: "fork"; repository: string } + | { type: "remote"; url: string }; + +interface RepositoryOptions { + description?: string | undefined; + readOnly?: boolean; + setDefaultBranch?: string; +} + +interface ForkOptions { + description?: string; + readOnly?: boolean; + defaultBranchOnly?: boolean; +} + +interface ImportRepositoryOptions { + source: { url: string; branch?: string; depth?: number }; + target: { name: string; opts?: { description?: string; readOnly?: boolean } }; +} + +interface RepositoryToken { + id: string; + tokenHash: string; + scope: TokenScope; + createdAt: string; + expiresAt: string; + revokedAt?: string; +} + +interface CreatedToken { + record: RepositoryToken; + plaintext: string; +} + +interface Repository { + id: string; + name: string; + namespace: string; + description?: string | undefined; + readOnly: boolean; + defaultBranch: string; + remote: string; + source?: RepositorySource | undefined; + createdAt: string; + updatedAt: string; + lastPushAt?: string | undefined; + tokens: RepositoryToken[]; +} + +interface RepositoryMetadata { + id: string; + name: string; + remote: string; + defaultBranch: string; + description: string | null; + source: string | null; + readOnly: boolean; + createdAt: string; + updatedAt: string; + lastPushAt: string | null; +} + +interface CreateRepositoryResult { + id: string; + name: string; + description: string | null; + defaultBranch: string; + remote: string; + token: string; +} + +interface CreateTokenResult { + id: string; + plaintext: string; + scope: TokenScope; + expiresAt: string; +} +type TokenInfo = Omit & { + state: "active" | "expired" | "revoked"; +}; + +interface RepositoryListOptions { + cursor?: string; + limit?: number; +} + +type RepositoryListResult = { + repos: Array< + RepositoryMetadata & { status: "ready"; jurisdiction: "unrestricted" } + >; + total: number; + cursor?: string; +}; + +interface RepositoryLogOptions { + ref?: string; + limit?: number; + offset?: number; +} + +interface BackendState { + defaultBranch: string; + refs: Record; +} + +type BackendRequest = + | { action: "create"; defaultBranch: string } + | { action: "delete" } + | { + action: "fork"; + sourceName: string; + sourceBranch: string; + defaultBranchOnly?: boolean; + } + | { action: "import"; sourceUrl: string; branch?: string; depth?: number } + | { action: "refs" } + | { action: "readBlob"; hash: string } + | { action: "readTree"; hash: string } + | { action: "readCommit"; hash: string } + | { action: "file"; ref: string; path: string } + | { action: "log"; ref?: string; limit?: number; offset?: number }; + +interface TreeEntry { + name: string; + mode: string; + hash: string; + type: "tree" | "blob" | "symlink" | "gitlink" | "exec"; +} + +interface CommitMetadata { + hash: string; + treeHash: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + parents: string[]; + authoredAt: number; + committedAt: number; +} + +interface LocalArtifactsRepo { + info(): Promise; + createToken(scope?: TokenScope, ttl?: number): Promise; + listTokens(): Promise<{ total: number; tokens: TokenInfo[] }>; + revokeToken(tokenOrId: string): Promise; + readBlob(hash: string): Promise; + readTree(hash: string): Promise; + readCommit(hash: string): Promise; + readFile(args: { ref: string; path: string }): Promise; + log(options?: RepositoryLogOptions): Promise; + fork(name: string, options?: ForkOptions): Promise; +} + +interface BackendBlob { + data: string; +} + +type ArtifactsErrorCode = + | "ALREADY_EXISTS" + | "NOT_FOUND" + | "INVALID_INPUT" + | "INVALID_REPO_NAME" + | "INVALID_TTL" + | "INVALID_URL" + | "REMOTE_AUTH_REQUIRED" + | "MEMORY_LIMIT" + | "UPSTREAM_UNAVAILABLE" + | "INTERNAL_ERROR"; + +const numericCodes: Record = { + INVALID_INPUT: 10100, + INVALID_REPO_NAME: 10101, + INVALID_TTL: 10103, + INVALID_URL: 10104, + REMOTE_AUTH_REQUIRED: 10106, + NOT_FOUND: 10200, + ALREADY_EXISTS: 10201, + INTERNAL_ERROR: 10400, + UPSTREAM_UNAVAILABLE: 10401, + MEMORY_LIMIT: 10402, +}; + +class ArtifactsError extends Error { + readonly code: ArtifactsErrorCode; + readonly numericCode: number; + + constructor(code: ArtifactsErrorCode, message: string) { + super(message); + this.name = "ArtifactsError"; + this.code = code; + this.numericCode = numericCodes[code]; + } +} + +interface LocalArtifactsEnvironment { + config: { namespace: string; origin: string }; + localArtifactsNamespace: DurableObjectNamespace; + gitBackend: Fetcher; +} + +type RepositoryMap = Record; + +export class LocalArtifactsNamespace extends WorkerEntrypoint { + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return publicCall(() => this.#namespace().create(name, options)); + } + + async get(name: string): Promise { + const repository = await publicCall(() => this.#namespace().get(name)); + return repository as unknown as LocalArtifactsRepo; + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return publicCall(() => this.#namespace().import(options)); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + return publicCall(() => this.#namespace().list(options)); + } + + async delete(name: string): Promise { + return publicCall(() => this.#namespace().delete(name)); + } + + override async fetch(request: Request): Promise { + return this.#namespace().fetch(request); + } + + #namespace(): DurableObjectStub { + const { localArtifactsNamespace, config } = this.env; + return localArtifactsNamespace.get( + localArtifactsNamespace.idFromName(config.namespace) + ); + } +} + +export class LocalArtifactsNamespaceObject extends DurableObject { + // The resolved tail lets the first mutation run immediately. exclusive() + // serializes subsequent metadata and backend mutations across awaits. + private operations = Promise.resolve(); + + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + + const repository = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + try { + const state = await this.backend(repository, { + action: "create", + defaultBranch: repository.defaultBranch, + }); + applyBackendState(repository, state); + repositories[name.toLowerCase()] = repository; + const token = await createToken(repository, "write"); + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist Artifacts repository" + ); + } + return createRepoResult(repository, token); + } catch (error) { + if (error instanceof ArtifactsError) { + throw error; + } + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to create Artifacts repository" + ); + } + }); + } + + async get(name: string): Promise { + validateName(name); + const repository = await this.getRepo(name); + return createRepoHandle(this, repository.name); + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return this.exclusive(async () => { + const { source, target, sourceUrl } = parseImportRequest(options); + const repositories = await this.repos(); + if (hasRepository(repositories, target.name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${target.name}` + ); + } + validateRepositoryOptions(target.opts ?? {}); + const repository = makeRepo( + target.name, + this.env.config.namespace, + this.env.config.origin, + target.opts ?? {} + ); + repository.source = { type: "remote", url: sourceUrl.href }; + const state = await this.backend(repository, { + action: "import", + sourceUrl: sourceUrl.href, + ...(source.branch === undefined ? {} : { branch: source.branch }), + ...(source.depth === undefined ? {} : { depth: source.depth }), + }); + applyBackendState(repository, state); + repository.updatedAt = new Date().toISOString(); + const token = await createToken(repository, "write"); + repositories[repository.name.toLowerCase()] = repository; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist imported Artifacts repository" + ); + } + return createRepoResult(repository, token); + }); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + const limit = options.limit ?? 50; + if (!Number.isInteger(limit) || limit < 1 || limit > 200) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 200." + ); + } + const offset = decodeCursor(options.cursor); + const repositories = Object.values(await this.repos()).sort((left, right) => + right.createdAt.localeCompare(left.createdAt) + ); + const page = repositories + .slice(offset, offset + limit) + .map((repository) => ({ + ...repoMetadata(repository), + status: "ready" as const, + // Local storage has no placement jurisdiction; this is the public default. + jurisdiction: "unrestricted" as const, + })); + const cursor = + offset + page.length < repositories.length + ? encodeCursor(offset + page.length) + : undefined; + return { + repos: page, + total: repositories.length, + ...(cursor === undefined ? {} : { cursor }), + }; + } + + async delete(name: string): Promise { + return this.exclusive(async () => { + validateName(name); + const repositories = await this.repos(); + if (!hasRepository(repositories, name)) { + return false; + } + const repository = getRepo(repositories, name); + await this.backend(repository, { action: "delete" }); + delete repositories[name.toLowerCase()]; + await this.saveRepos(repositories); + return true; + }); + } + + async createToken( + name: string, + scope: TokenScope = "write", + ttl?: number + ): Promise { + return this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name); + const token = await createToken(repository, scope, ttl); + await this.saveRepos(repositories); + return tokenResult(token); + }); + } + + async listTokens( + name: string + ): Promise<{ total: number; tokens: TokenInfo[] }> { + const repository = await this.getRepo(name); + const active = repository.tokens.filter( + (token) => !token.revokedAt && isActive(token) + ); + const tokens = active + .toReversed() + .slice(0, 30) + .map(({ tokenHash: _tokenHash, revokedAt: _revokedAt, ...token }) => ({ + ...token, + state: "active" as const, + })); + return { total: active.length, tokens }; + } + + async revokeToken(name: string, tokenOrId: string): Promise { + return this.exclusive(async () => { + if (!isTokenId(tokenOrId) && !isArtifactToken(tokenOrId)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid tokenOrId: must be a token id or artifact token." + ); + } + const repositories = await this.repos(); + const repository = getRepo(repositories, name); + const tokenHash = isArtifactToken(tokenOrId) + ? await hashToken(tokenOrId) + : undefined; + const token = repository.tokens.find( + (candidate) => + candidate.id === tokenOrId || + (tokenHash !== undefined && + constantTimeEqual(candidate.tokenHash, tokenHash)) + ); + if (!token || token.revokedAt) { + return false; + } + token.revokedAt = new Date().toISOString(); + await this.saveRepos(repositories); + return true; + }); + } + + async fork( + sourceName: string, + name: string, + options: ForkOptions = {} + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + const source = getRepo(repositories, sourceName); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + const fork = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + let state: BackendState; + try { + state = await this.backend(fork, { + action: "fork", + sourceName: source.name, + sourceBranch: source.defaultBranch, + ...(options.defaultBranchOnly === undefined + ? {} + : { defaultBranchOnly: options.defaultBranchOnly }), + }); + } catch { + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to fork Artifacts repository" + ); + } + applyBackendState(fork, state); + fork.source = { type: "fork", repository: source.name }; + const token = await createToken(fork, "write"); + repositories[name.toLowerCase()] = fork; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(fork); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist forked Artifacts repository" + ); + } + return createRepoResult(fork, token); + }); + } + + override async fetch(request: Request): Promise { + return handleGitRequest(this, this.env.config.namespace, request); + } + + async repos(): Promise { + const repositories = + (await this.ctx.storage.get("repos")) ?? + (Object.create(null) as RepositoryMap); + // The local Git listener can move after a process restart. + for (const repository of Object.values(repositories)) { + repository.remote = `${this.env.config.origin}/git/${repository.namespace}/${repository.name}.git`; + } + return repositories; + } + + saveRepos(repositories: RepositoryMap): Promise { + return this.ctx.storage.put("repos", repositories); + } + + async getRepo(name: string): Promise { + return getRepo(await this.repos(), name); + } + + async backend( + repository: Repository, + request: BackendRequest + ): Promise { + const response = await this.env.gitBackend.fetch( + "http://local-artifacts-backend/__local_artifacts__", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ...request, + namespace: repository.namespace, + name: repository.name, + generation: repository.id, + }), + } + ); + if (!response.ok) { + const message = await response.text(); + if (request.action === "import") { + throw classifyImportError(message); + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } + return response.json(); + } + + private async deleteBackend(repository: Repository): Promise { + try { + await this.backend(repository, { action: "delete" }); + } catch { + // Preserve the original persistence error; native delete is idempotent on retry. + } + } + + gitFetch(repository: Repository, request: Request): Promise { + const headers = new Headers(request.headers); + headers.delete("Authorization"); + headers.set("X-Local-Artifacts-Generation", repository.id); + const url = new URL(request.url); + url.pathname = url.pathname.replace( + /^\/git\/[^/]+\/[^/]+\.git/, + `/git/${repository.namespace}/${repository.name}.git` + ); + return this.env.gitBackend.fetch( + new Request(url, new Request(request, { headers })) + ); + } + + async syncAfterPush( + name: string, + previousRefs: Record + ): Promise { + await this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name); + const state = await this.backend(repository, { + action: "refs", + }); + if (JSON.stringify(state.refs) !== JSON.stringify(previousRefs)) { + repository.updatedAt = new Date().toISOString(); + repository.lastPushAt = repository.updatedAt; + } + await this.saveRepos(repositories); + }); + } + + private async exclusive(action: () => Promise): Promise { + const previous = this.operations; + let release!: () => void; + this.operations = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await action(); + } finally { + release(); + } + } +} + +function createRepoHandle( + namespace: LocalArtifactsNamespaceObject, + name: string +): LocalArtifactsRepo { + const read = (request: BackendRequest): Promise => + publicCall(async () => + namespace.backend(await namespace.getRepo(name), request) + ); + const readFile = async (args: { + ref: string; + path: string; + }): Promise => { + if (!args?.ref) { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: ref required."); + } + if (!args?.path) { + throw new ArtifactsError("INVALID_INPUT", "Invalid path: path required."); + } + const path = normalizePath(args.path); + if (path === null) { + return null; + } + const result = await read({ + action: "file", + ref: args.ref, + path, + }); + if (result === null) { + return null; + } + const bytes = decodeBase64(result.data); + return new Blob([bytes], { type: contentType(args.path, bytes) }); + }; + + return { + info: () => + publicCall(async () => repoMetadata(await namespace.getRepo(name))), + createToken: (scope: TokenScope = "write", ttl?: number) => + publicCall(() => namespace.createToken(name, scope, ttl)), + listTokens: () => publicCall(() => namespace.listTokens(name)), + revokeToken: (tokenOrId: string) => + publicCall(() => namespace.revokeToken(name, tokenOrId)), + readBlob: async (hash: string) => { + validateHash(hash); + const result = await read({ + action: "readBlob", + hash, + }); + return result === null ? null : new Blob([decodeBase64(result.data)]); + }, + readTree: async (hash: string) => { + validateHash(hash); + return await read({ action: "readTree", hash }); + }, + readCommit: async (hash: string) => { + validateHash(hash); + return await read({ action: "readCommit", hash }); + }, + readFile, + log: (options: RepositoryLogOptions = {}) => { + validateLogOptions(options); + return read({ + action: "log", + ...(options.ref === undefined ? {} : { ref: options.ref }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.offset === undefined ? {} : { offset: options.offset }), + }); + }, + fork: (targetName: string, options: ForkOptions = {}) => + publicCall(() => namespace.fork(name, targetName, options)), + }; +} + +function parseGitRoute( + request: Request, + namespace: string +): { name: string; write: boolean; push: boolean } | null { + const url = new URL(request.url); + const match = url.pathname.match(/^\/git\/([^/]+)\/([^/]+)\.git(?:\/(.*))?$/); + if (!match || match[1] !== namespace || !match[2]) { + return null; + } + const path = match[3] ?? ""; + const service = url.searchParams.get("service"); + const discovery = + request.method === "GET" && + path === "info/refs" && + (service === "git-upload-pack" || service === "git-receive-pack"); + const pack = + request.method === "POST" && + (path === "git-upload-pack" || path === "git-receive-pack"); + if (!discovery && !pack) { + return null; + } + return { + name: match[2], + write: path === "git-receive-pack" || service === "git-receive-pack", + push: request.method === "POST" && path === "git-receive-pack", + }; +} + +async function handleGitRequest( + state: LocalArtifactsNamespaceObject, + namespace: string, + request: Request +): Promise { + const route = parseGitRoute(request, namespace); + if (!route) { + return new Response("Not found", { status: 404 }); + } + let repository: Repository; + try { + repository = await state.getRepo(route.name); + } catch (error) { + if (error instanceof ArtifactsError && error.code === "NOT_FOUND") { + return new Response("Not found", { status: 404 }); + } + throw error; + } + const authorizationError = await authorize( + repository, + request, + route.write ? "write" : "read" + ); + if (authorizationError) { + return authorizationError; + } + if (route.write && repository.readOnly) { + return new Response("Artifacts repository is read-only", { status: 403 }); + } + + const previousRefs = route.push + ? (await state.backend(repository, { action: "refs" })).refs + : undefined; + const response = await state.gitFetch(repository, request); + if (!previousRefs) { + return response; + } + + const body = await response.arrayBuffer(); + if (response.ok) { + try { + await state.syncAfterPush(repository.name, previousRefs); + } catch { + // Git has already accepted the push; metadata bookkeeping must not fail the client request. + } + } + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function authorize( + repository: Repository, + request: Request, + scope: TokenScope +): Promise { + const header = request.headers.get("Authorization") ?? ""; + let value = ""; + if (header.startsWith("Bearer ")) { + value = header.slice(7); + } else if (header.startsWith("Basic ")) { + try { + value = atob(header.slice(6)).split(":").slice(1).join(":"); + } catch { + value = ""; + } + } + const valueHash = await hashToken(value); + const token = repository.tokens.find( + (candidate) => + !candidate.revokedAt && + constantTimeEqual(candidate.tokenHash, valueHash) && + isActive(candidate) + ); + if (!token) { + return new Response("Invalid Artifacts token", { + status: 401, + headers: { "WWW-Authenticate": 'Basic realm="Artifacts"' }, + }); + } + if (scope === "write" && token.scope !== "write") { + return new Response("Artifacts token requires write scope", { + status: 403, + }); + } + return undefined; +} + +function applyBackendState(repository: Repository, state: BackendState): void { + repository.defaultBranch = state.defaultBranch; +} + +function makeRepo( + name: string, + namespace: string, + origin: string, + options: RepositoryOptions +): Repository { + const time = new Date().toISOString(); + return { + id: randomBase36(16), + name, + namespace, + description: options.description, + readOnly: Boolean(options.readOnly), + defaultBranch: options.setDefaultBranch ?? "main", + remote: `${origin}/git/${namespace}/${name}.git`, + createdAt: time, + updatedAt: time, + lastPushAt: undefined, + tokens: [], + }; +} + +function repoMetadata(repository: Repository): RepositoryMetadata { + return { + id: repository.id, + name: repository.name, + remote: repository.remote, + defaultBranch: repository.defaultBranch, + description: repository.description ?? null, + source: + repository.source === undefined + ? null + : repository.source.type === "fork" + ? `artifacts:${repository.namespace}/${repository.source.repository}` + : `git:${repository.source.url}`, + readOnly: repository.readOnly, + createdAt: repository.createdAt, + updatedAt: repository.updatedAt, + lastPushAt: repository.lastPushAt ?? null, + }; +} + +function createRepoResult( + repository: Repository, + token: CreatedToken +): CreateRepositoryResult { + return { + id: repository.id, + name: repository.name, + description: repository.description ?? null, + defaultBranch: repository.defaultBranch, + remote: repository.remote, + token: token.plaintext, + }; +} + +function tokenResult(token: CreatedToken): CreateTokenResult { + return { + id: token.record.id, + plaintext: token.plaintext, + scope: token.record.scope, + expiresAt: token.record.expiresAt, + }; +} + +function getRepo(repositories: RepositoryMap, name: string): Repository { + if (!hasRepository(repositories, name)) { + throw new ArtifactsError("NOT_FOUND", `Repository not found: ${name}.`); + } + return repositories[name.toLowerCase()] as Repository; +} + +function hasRepository(repositories: RepositoryMap, name: string): boolean { + return Object.hasOwn(repositories, name.toLowerCase()); +} + +function parseImportRequest(options: ImportRepositoryOptions): { + source: ImportRepositoryOptions["source"]; + target: ImportRepositoryOptions["target"]; + sourceUrl: URL; +} { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts import options are required" + ); + } + const { source, target } = options; + validateName(target?.name); + let sourceUrl: URL; + try { + sourceUrl = new URL(source?.url); + } catch { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be a valid URL." + ); + } + if (sourceUrl.protocol !== "https:") { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be an HTTPS URL." + ); + } + return { source, target, sourceUrl }; +} + +function validateName(name: unknown): asserts name is string { + if (typeof name !== "string" || !repoName.test(name)) { + throw new ArtifactsError("INVALID_REPO_NAME", "Invalid repo name."); + } +} + +function validateRepositoryOptions( + options: RepositoryOptions | ForkOptions +): void { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository options must be an object" + ); + } + if ( + options.description !== undefined && + typeof options.description !== "string" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository description must be a string" + ); + } + if (options.readOnly !== undefined && typeof options.readOnly !== "boolean") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository readOnly must be a boolean" + ); + } + if ("setDefaultBranch" in options && options.setDefaultBranch !== undefined) { + validateBranch(options.setDefaultBranch); + } + if ( + "defaultBranchOnly" in options && + options.defaultBranchOnly !== undefined && + typeof options.defaultBranchOnly !== "boolean" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts defaultBranchOnly must be a boolean" + ); + } +} + +function validateBranch(branch: unknown): asserts branch is string { + if ( + typeof branch !== "string" || + branch.length < 1 || + branch.length > 256 || + branch.startsWith("-") || + branch.startsWith(".") || + branch.endsWith("/") || + branch.endsWith(".") || + branch.endsWith(".lock") || + branch === "@" || + branch.includes("..") || + branch.includes("//") || + branch.includes("@{") || + branch.split("/").some((segment) => segment.startsWith(".")) || + [...branch].some((character) => character.charCodeAt(0) <= 32) || + /[~^:?*[\\]/.test(branch) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts default branch is invalid" + ); + } +} + +function validateLogOptions( + options: unknown +): asserts options is RepositoryLogOptions { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts log options must be an object" + ); + } + const { ref, limit, offset } = options as RepositoryLogOptions; + if (ref !== undefined && typeof ref !== "string") { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: must be a string."); + } + if ( + limit !== undefined && + (!Number.isInteger(limit) || limit < 1 || limit > 1000) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 1000." + ); + } + if (offset !== undefined && (!Number.isInteger(offset) || offset < 0)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid offset: must be a non-negative integer." + ); + } +} + +async function createToken( + repository: Repository, + scope: TokenScope = "write", + ttl = 86_400 +): Promise { + if (scope !== "read" && scope !== "write") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts token scope must be read or write" + ); + } + if (!Number.isInteger(ttl) || ttl < 60 || ttl > 31_536_000) { + throw new ArtifactsError( + "INVALID_TTL", + `Invalid TTL ${ttl}: must be between 60 and 31536000 seconds.` + ); + } + const createdAt = new Date(); + const expiresAt = new Date(createdAt.getTime() + ttl * 1000); + const plaintext = `art_v2_x_${randomHex(20)}?expires=${Math.floor(expiresAt.getTime() / 1000)}`; + const record = { + id: randomBase36(16), + tokenHash: await hashToken(plaintext), + scope, + createdAt: createdAt.toISOString(), + expiresAt: expiresAt.toISOString(), + }; + repository.tokens.push(record); + return { record, plaintext }; +} + +function isActive(token: Pick): boolean { + return Date.parse(token.expiresAt) > Date.now(); +} + +function randomHex(bytes: number): string { + return [...crypto.getRandomValues(new Uint8Array(bytes))] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +async function hashToken(value: string): Promise { + const secret = value.split("?", 1)[0] ?? ""; + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(secret) + ); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function constantTimeEqual(left: string, right: string): boolean { + if (left.length !== right.length) { + return false; + } + let difference = 0; + for (let index = 0; index < left.length; index++) { + difference |= left.charCodeAt(index) ^ right.charCodeAt(index); + } + return difference === 0; +} + +function randomBase36(length: number): string { + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"; + return [...crypto.getRandomValues(new Uint8Array(length))] + .map((byte) => alphabet[byte % alphabet.length]) + .join(""); +} + +function isTokenId(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-z]{16}$/.test(value); +} + +function isArtifactToken(value: unknown): value is string { + return ( + typeof value === "string" && + /^art_v(?:1_[0-9a-f]{40}|2_[xeuf]_[0-9a-f]{40})(?:\?expires=\d+)?$/.test( + value + ) + ); +} + +function validateHash(hash: unknown): asserts hash is string { + if (!hash) { + throw new ArtifactsError("INVALID_INPUT", "Invalid hash: hash required."); + } + if (typeof hash !== "string" || !/^[0-9a-f]{40}$/.test(hash)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid hash: expected a 40-character SHA-1 hex hash." + ); + } +} + +function normalizePath(path: string): string | null { + const segments = path + .split("/") + .filter((segment) => segment !== "" && segment !== "."); + if (segments.includes("..") || segments.length === 0) { + return null; + } + return segments.join("/"); +} + +function contentType(path: string, bytes: Uint8Array): string { + const extension = path.toLowerCase().match(/\.[^./]+$/)?.[0]; + const known: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".ico": "image/x-icon", + ".pdf": "application/pdf", + }; + if (extension && known[extension]) { + return known[extension]; + } + return bytes.subarray(0, 8192).includes(0) + ? "application/octet-stream" + : "text/plain;charset=utf-8"; +} + +function decodeBase64(value: string): Uint8Array { + return Uint8Array.from(atob(value), (character) => character.charCodeAt(0)); +} + +function encodeCursor(offset: number): string { + return btoa(JSON.stringify({ offset })); +} + +function decodeCursor(cursor: string | undefined): number { + if (cursor === undefined) { + return 0; + } + try { + const value = JSON.parse(atob(cursor)) as { offset?: unknown }; + return typeof value.offset === "number" && + Number.isInteger(value.offset) && + value.offset >= 0 + ? value.offset + : 0; + } catch { + return 0; + } +} + +async function publicCall(action: () => Promise): Promise { + try { + return await action(); + } catch (error) { + if (isArtifactsError(error)) { + throw error; + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } +} + +function isArtifactsError(error: unknown): error is ArtifactsError { + return ( + error instanceof Error && + error.name === "ArtifactsError" && + "code" in error && + typeof error.code === "string" && + "numericCode" in error && + typeof error.numericCode === "number" + ); +} + +function classifyImportError(message: string): ArtifactsError { + const normalized = message.toLowerCase(); + if ( + normalized.includes("authentication failed") || + normalized.includes("could not read username") || + normalized.includes("http 401") + ) { + return new ArtifactsError( + "REMOTE_AUTH_REQUIRED", + "Authentication is required to access the remote repository." + ); + } + if ( + normalized.includes("remote branch") && + normalized.includes("not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote branch not found."); + } + if ( + normalized.includes("not found") || + normalized.includes("repository not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote repository not found."); + } + if (normalized.includes("does not appear to be a git repository")) { + return new ArtifactsError( + "INVALID_URL", + "The URL does not point to a git repository." + ); + } + return new ArtifactsError( + "UPSTREAM_UNAVAILABLE", + "The upstream service is unavailable. Please retry." + ); +} diff --git a/packages/miniflare/test/config/v4-convert.spec.ts b/packages/miniflare/test/config/v4-convert.spec.ts index bd6ca18210c..3e156aad756 100644 --- a/packages/miniflare/test/config/v4-convert.spec.ts +++ b/packages/miniflare/test/config/v4-convert.spec.ts @@ -4,6 +4,27 @@ import { convertV4MiniflareOptions } from "../../src/config/v4-convert"; import type { RemoteProxyConnectionString } from "../../src/plugins/shared"; describe("convertV4MiniflareOptions", () => { + test("Artifacts defaults to local and only an explicit connection is remote", ({ + expect, + }) => { + const remoteProxyConnectionString = new URL( + "http://127.0.0.1:8787" + ) as RemoteProxyConnectionString; + const converted = convertV4MiniflareOptions({ + name: "worker", + script: "export default {};", + artifacts: { + LOCAL: { namespace: "local" }, + REMOTE: { namespace: "remote", remoteProxyConnectionString }, + }, + }); + expect(converted.workers[0].config.env?.LOCAL).toMatchObject({ + dev: { remote: false }, + }); + expect(converted.workers[0].config.env?.REMOTE).toMatchObject({ + dev: { remote: true }, + }); + }); test("preserves opaque K2 stream IDs through v4 parsing and conversion", ({ expect, }) => { diff --git a/packages/miniflare/test/plugins/artifacts/index.spec.ts b/packages/miniflare/test/plugins/artifacts/index.spec.ts new file mode 100644 index 00000000000..9d7097e766f --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/index.spec.ts @@ -0,0 +1,811 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { devNull } from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { + GitClient, + gitEnvironment, +} from "../../../src/plugins/artifacts/git-client"; +import { startGitSidecar } from "../../../src/plugins/artifacts/git-sidecar"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; +import type { MiniflareOptions } from "miniflare"; + +const exec = promisify(execFile); +const SCRIPT = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name === undefined ? env.REPOS : await env.REPOS.get(name); + if (method === "keys") return Response.json(Object.keys(target)); + const result = await target[method](...args); + if (result instanceof Blob) { + return Response.json({ bytes: [...new Uint8Array(await result.arrayBuffer())], type: result.type }); + } + return Response.json(result); + } catch (error) { + return Response.json({ error: error.message, name: error.name, code: error.code, numericCode: error.numericCode }, { status: 400 }); + } + } +}; +`; + +function options( + namespace = "test", + binding = "REPOS", + remote: boolean | null = false +): MiniflareOptions { + return { + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + [binding]: { + type: "artifacts", + namespace, + ...(remote === null ? {} : { dev: { remote } }), + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }; +} + +function rpcRequest( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + return mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); +} + +async function rpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +) { + const response = await rpcRequest(mf, method, args, name); + const result = (await response.json()) as any; + if (!response.ok) { + throw new Error(result.error); + } + return result; +} + +async function rpcFailure( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +) { + const response = await rpcRequest(mf, method, args, name); + return { + status: response.status, + ...((await response.json()) as { + error: string; + name: string; + code: string; + numericCode: number; + }), + }; +} + +function git(args: string[], cwd?: string) { + return exec("git", args, { + cwd, + env: { + ...process.env, + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: devNull, + GIT_TERMINAL_PROMPT: "0", + GIT_AUTHOR_NAME: "Miniflare", + GIT_AUTHOR_EMAIL: "miniflare@example.com", + GIT_COMMITTER_NAME: "Miniflare", + GIT_COMMITTER_EMAIL: "miniflare@example.com", + }, + }); +} + +async function pushFixture(mf: Miniflare, directory: string) { + const created = await rpc(mf, "create", ["Repo"]); + await git(["init", "--initial-branch=main", directory]); + await writeFile(path.join(directory, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(directory, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await git(["add", "."], directory); + await git(["commit", "-m", "fixture"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "main", + ], + directory + ); + return created; +} + +test("artifacts: real RPC exposes current methods, not metadata or legacy methods", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["Repo"]); + const keys = await rpc(mf, "keys", [], "Repo"); + expect(keys).toContain("readFile"); + expect(keys).not.toContain("id"); + expect(keys).not.toContain("name"); + expect(keys).not.toContain("file"); + expect(keys).not.toContain("raw"); + expect( + await rpc(mf, "readFile", [{ ref: "HEAD", path: "missing" }], "Repo") + ).toBeNull(); +}); + +test("artifacts: native Git subprocesses do not inherit Cloudflare or host credentials", ({ + expect, +}) => { + const env = gitEnvironment({ + PATH: "/usr/bin", + CLOUDFLARE_API_TOKEN: "production-secret", + GIT_CONFIG_GLOBAL: "/home/user/.gitconfig", + GIT_CONFIG_PARAMETERS: "'credential.helper=unsafe'", + GIT_ASKPASS: "/home/user/askpass", + GITHUB_TOKEN: "github-secret", + }); + expect(env.PATH).toBe("/usr/bin"); + expect(env.CLOUDFLARE_API_TOKEN).toBeUndefined(); + expect(env.GIT_CONFIG_PARAMETERS).toBeUndefined(); + expect(env.GIT_ASKPASS).toBeUndefined(); + expect(env.GITHUB_TOKEN).toBeUndefined(); + expect(env.GIT_CONFIG_VALUE_0).toBe(""); + expect(env.GIT_CONFIG_GLOBAL).toBe(devNull); +}); + +test("artifacts: Git client imports and reads a local fixture", async ({ + expect, +}) => { + const directory = await useTmp(); + const source = path.join(directory, "source"); + await git(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "hello.txt"), "hello artifacts\n"); + await git(["add", "."], source); + await git(["commit", "-m", "fixture"], source); + + // The public binding requires HTTPS; file:// only exercises native Git here. + const imported = new GitClient(path.join(directory, "imported.git")); + await imported.importFrom(pathToFileURL(source).href, "main", 1); + await imported.configure("local-generation"); + expect(await imported.generation()).toBe("local-generation"); + expect((await imported.state()).defaultBranch).toBe("main"); + expect((await imported.readFile("main", "hello.txt"))?.data).toBe( + Buffer.from("hello artifacts\n").toString("base64") + ); +}); + +test("artifacts: omitted remote remains offline", async ({ expect }) => { + const mf = new Miniflare(options("test", "REPOS", null)); + useDispose(mf); + expect((await rpc(mf, "create", ["offline"])).name).toBe("offline"); +}); + +test("artifacts: rejects invalid local namespace before starting services", async ({ + expect, +}) => { + const mf = new Miniflare(options("../invalid")); + await expect(mf.ready).rejects.toThrow( + "Invalid local Artifacts namespace: ../invalid" + ); + // dispose() preserves a startup failure after cleaning up the instance. + await expect(mf.dispose()).rejects.toThrow( + "Invalid local Artifacts namespace: ../invalid" + ); +}); + +test("artifacts: names are case-insensitive for create, get, fork and delete", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["Repo"]); + expect((await rpc(mf, "info", [], "rEpO")).id).toBe(created.id); + await expect(rpc(mf, "create", ["repo"])).rejects.toThrow(/already exists/i); + await expect(rpc(mf, "fork", ["REPO"], "repo")).rejects.toThrow( + /already exists/i + ); + expect(await rpc(mf, "delete", ["REPO"])).toBe(true); + expect(await rpc(mf, "delete", ["repo"])).toBe(false); +}); + +test("artifacts: supports Node binding proxies", async ({ expect }) => { + const mf = new Miniflare(options()); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { + create(name: string): Promise<{ name: string }>; + list(): Promise<{ total: number }>; + }; + }>(); + expect(await REPOS.create("node-repo")).toMatchObject({ name: "node-repo" }); + expect(await REPOS.list()).toMatchObject({ total: 1 }); +}); + +test("artifacts: aliases share a namespace while distinct namespaces are isolated", async ({ + expect, +}) => { + const mf = new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { type: "artifacts", namespace: "shared" }, + ALIAS: { type: "artifacts", namespace: "shared" }, + OTHER: { type: "artifacts", namespace: "other" }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); + useDispose(mf); + const bindings = await mf.getBindings<{ + REPOS: { create(name: string): Promise }; + ALIAS: { list(): Promise<{ total: number }> }; + OTHER: { list(): Promise<{ total: number }> }; + }>(); + await bindings.REPOS.create("shared-repo"); + expect((await bindings.ALIAS.list()).total).toBe(1); + expect((await bindings.OTHER.list()).total).toBe(0); +}); + +test("artifacts: two Workers can share a namespace through different bindings", async ({ + expect, +}) => { + const worker = (name: string, binding: string) => ({ + config: { + name, + compatibilityDate: "2026-09-03", + env: { [binding]: { type: "artifacts" as const, namespace: "shared" } }, + manifest: singleModuleManifest(SCRIPT), + }, + }); + const mf = new Miniflare({ + cf: false, + workers: [worker("first", "REPOS"), worker("second", "ALIAS")], + }); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { create(name: string): Promise<{ name: string }> }; + }>("first"); + const { ALIAS } = await mf.getBindings<{ + ALIAS: { list(): Promise<{ total: number }> }; + }>("second"); + await REPOS.create("shared-repo"); + expect((await ALIAS.list()).total).toBe(1); +}); + +test("artifacts: native Git reads commits, trees, blobs and files", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const source = path.join(await useTmp(), "source"); + await pushFixture(mf, source); + const { stdout } = await git(["rev-parse", "HEAD"], source); + const hash = stdout.trim(); + const commit = await rpc(mf, "readCommit", [hash], "Repo"); + expect(commit.hash).toBe(hash); + expect((await rpc(mf, "log", [], "Repo"))[0].hash).toBe(hash); + const tree = await rpc(mf, "readTree", [commit.treeHash], "Repo"); + const entry = tree.find(({ name }: { name: string }) => name === "hello.txt"); + expect(entry).toBeDefined(); + expect((await rpc(mf, "readBlob", [entry.hash], "Repo")).bytes).toEqual([ + ...new TextEncoder().encode("hello artifacts\n"), + ]); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "hello.txt" }], "Repo") + ).toEqual({ + bytes: [...new TextEncoder().encode("hello artifacts\n")], + type: "text/plain;charset=utf-8", + }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "bytes.bin" }], "Repo") + ).toEqual({ bytes: [0, 255, 128, 1], type: "application/octet-stream" }); +}); + +test("artifacts: native Git push/clone and token revocation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = await useTmp(); + const created = await pushFixture(mf, path.join(directory, "source")); + expect(new URL(created.remote).hostname).toBe("127.0.0.1"); + const token = await rpc(mf, "createToken", ["read"], "Repo"); + await git([ + "-c", + `http.extraHeader=Authorization: Bearer ${token.plaintext}`, + "clone", + created.remote.replace("Repo.git", "repo.git"), + path.join(directory, "clone"), + ]); + expect( + await readFile(path.join(directory, "clone", "hello.txt"), "utf8") + ).toBe("hello artifacts\n"); + const advertise = `${created.remote}/info/refs?service=git-upload-pack`; + const discovery = await fetch(advertise, { + headers: { + Authorization: `Bearer ${token.plaintext}`, + "Git-Protocol": "version=2", + }, + }); + expect(discovery.status).toBe(200); + expect(discovery.headers.get("content-type")).toContain( + "git-upload-pack-advertisement" + ); + await discovery.body?.cancel(); + const unsupported = await fetch(`${created.remote}/HEAD`, { + headers: { Authorization: `Bearer ${created.token}` }, + }); + expect(unsupported.status).toBe(404); + await unsupported.body?.cancel(); + const unauthenticated = await fetch(advertise); + expect(unauthenticated.status).toBe(401); + await unauthenticated.body?.cancel(); + await rpc(mf, "revokeToken", [token.id], "Repo"); + const revoked = await fetch(advertise, { + headers: { Authorization: `Bearer ${token.plaintext}` }, + }); + expect(revoked.status).toBe(401); + await revoked.body?.cancel(); +}); + +test("artifacts: read-only and read-scoped tokens reject Git pushes", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = await useTmp(); + const writable = await pushFixture(mf, path.join(directory, "source")); + const readToken = await rpc(mf, "createToken", ["read"], "Repo"); + const withReadToken = await fetch( + `${writable.remote}/info/refs?service=git-receive-pack`, + { + headers: { Authorization: `Bearer ${readToken.plaintext}` }, + } + ); + expect(withReadToken.status).toBe(403); + await withReadToken.body?.cancel(); + const readOnly = await rpc( + mf, + "fork", + ["read-only", { readOnly: true }], + "Repo" + ); + const writeAttempt = await fetch( + `${readOnly.remote}/info/refs?service=git-receive-pack`, + { + headers: { Authorization: `Bearer ${readOnly.token}` }, + } + ); + expect(writeAttempt.status).toBe(403); + await writeAttempt.body?.cancel(); + const readAttempt = await fetch( + `${readOnly.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${readOnly.token}` } } + ); + expect(readAttempt.status).toBe(200); + await readAttempt.body?.cancel(); +}); + +test("artifacts: metadata and Git data survive restart and binding rename", async ({ + expect, +}) => { + const root = await useTmp(); + const first = new Miniflare({ ...options(), resourcePersistencePath: root }); + useDispose(first); + const created = await pushFixture(first, path.join(await useTmp(), "source")); + await first.dispose(); + const second = new Miniflare({ + ...options("test", "RENAMED"), + resourcePersistencePath: root, + }); + useDispose(second); + const { RENAMED } = await second.getBindings<{ + RENAMED: { + get(name: string): Promise<{ + info(): Promise<{ id: string; remote: string }>; + readFile(args: { ref: string; path: string }): Promise; + }>; + }; + }>(); + const repo = await RENAMED.get("repo"); + const info = await repo.info(); + expect(info.id).toBe(created.id); + expect( + await (await repo.readFile({ ref: "main", path: "hello.txt" })).text() + ).toBe("hello artifacts\n"); + const response = await fetch( + `${info.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${created.token}` } } + ); + expect(response.status).toBe(200); + await response.body?.cancel(); +}); + +test("artifacts: namespaces and instances are isolated; reload preserves state", async ({ + expect, +}) => { + const first = new Miniflare(options()); + const second = new Miniflare(options()); + useDispose(first); + useDispose(second); + await rpc(first, "create", ["repo"]); + expect((await rpc(second, "list")).total).toBe(0); + await first.setOptions(options()); + expect((await rpc(first, "list")).total).toBe(1); + await first.setOptions(options("another")); + expect((await rpc(first, "list")).total).toBe(0); +}); + +test("artifacts: lists pages and deletes a repository and its handle", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + for (const name of ["one", "two", "three"]) { + await rpc(mf, "create", [name]); + } + const first = await rpc(mf, "list", [{ limit: 2 }]); + expect(first).toMatchObject({ total: 3 }); + expect(first.repos).toHaveLength(2); + expect(first.repos[0]).toMatchObject({ + status: "ready", + jurisdiction: "unrestricted", + }); + expect(first.cursor).toEqual(expect.any(String)); + const second = await rpc(mf, "list", [{ limit: 2, cursor: first.cursor }]); + expect(second).toMatchObject({ total: 3 }); + expect(second.repos).toHaveLength(1); + expect(second.cursor).toBeUndefined(); + expect( + [...first.repos, ...second.repos].map((repo) => repo.name).sort() + ).toEqual(["one", "three", "two"]); + const { REPOS } = await mf.getBindings<{ + REPOS: { + get(name: string): Promise<{ info(): Promise<{ name: string }> }>; + }; + }>(); + const handle = await REPOS.get("ONE"); + expect((await handle.info()).name).toBe("one"); + expect(await rpc(mf, "delete", ["one"])).toBe(true); + await expect(async () => handle.info()).rejects.toThrow(/not found/i); + expect(await rpcFailure(mf, "get", ["one"])).toMatchObject({ + code: "NOT_FOUND", + }); + expect((await rpc(mf, "list")).total).toBe(2); +}); + +test("artifacts: token metadata, revocation and validation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["repo"]); + const readToken = await rpc(mf, "createToken", ["read", 60], "repo"); + expect(readToken).toMatchObject({ + scope: "read", + expiresAt: expect.any(String), + }); + const listed = await rpc(mf, "listTokens", [], "repo"); + expect(listed.total).toBe(2); + expect(JSON.stringify(listed)).not.toContain(created.token); + expect(JSON.stringify(listed)).not.toContain(readToken.plaintext); + expect( + listed.tokens.every((token: { state: string }) => token.state === "active") + ).toBe(true); + expect(await rpc(mf, "revokeToken", [readToken.plaintext], "repo")).toBe( + true + ); + expect(await rpc(mf, "revokeToken", [readToken.id], "repo")).toBe(false); + expect((await rpc(mf, "listTokens", [], "repo")).total).toBe(1); + expect( + await rpcFailure(mf, "createToken", ["write", 59], "repo") + ).toMatchObject({ + status: 400, + name: "ArtifactsError", + code: "INVALID_TTL", + }); + expect( + await rpcFailure(mf, "createToken", ["write", 31_536_001], "repo") + ).toMatchObject({ + code: "INVALID_TTL", + }); + expect(await rpcFailure(mf, "revokeToken", ["bad"], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); +}); + +test("artifacts: rejects invalid inputs and returns null for missing objects", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + expect(await rpcFailure(mf, "create", ["invalid/name"])).toMatchObject({ + code: "INVALID_REPO_NAME", + numericCode: 10101, + }); + await rpc(mf, "create", ["repo"]); + expect(await rpcFailure(mf, "list", [{ limit: 0 }])).toMatchObject({ + code: "INVALID_INPUT", + }); + expect(await rpcFailure(mf, "readBlob", ["bad"], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); + const absentHash = "0".repeat(40); + expect(await rpc(mf, "readBlob", [absentHash], "repo")).toBeNull(); + expect(await rpc(mf, "readTree", [absentHash], "repo")).toBeNull(); + expect(await rpc(mf, "readCommit", [absentHash], "repo")).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "missing" }], "repo") + ).toBeNull(); + expect( + await rpcFailure(mf, "readFile", [{ ref: "main" }], "repo") + ).toMatchObject({ + code: "INVALID_INPUT", + }); + expect(await rpcFailure(mf, "log", [{ limit: 0 }], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); + expect( + await rpcFailure(mf, "import", [ + { + source: { url: "http://example.test/repo.git" }, + target: { name: "imported" }, + }, + ]) + ).toMatchObject({ + code: "INVALID_INPUT", + }); + expect( + await rpcFailure(mf, "import", [ + { source: { url: "not a url" }, target: { name: "imported" } }, + ]) + ).toMatchObject({ + code: "INVALID_INPUT", + }); +}); + +test("artifacts: Git credentials are scoped to the repository and operation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const first = await rpc(mf, "create", ["first"]); + const second = await rpc(mf, "create", ["second"]); + const readToken = await rpc(mf, "createToken", ["read"], "first"); + const firstRead = `${first.remote}/info/refs?service=git-upload-pack`; + const firstWrite = `${first.remote}/info/refs?service=git-receive-pack`; + const secondRead = `${second.remote}/info/refs?service=git-upload-pack`; + const basic = `Basic ${Buffer.from(`git:${readToken.plaintext}`).toString("base64")}`; + const read = await fetch(firstRead, { headers: { Authorization: basic } }); + expect(read.status).toBe(200); + await read.body?.cancel(); + const write = await fetch(firstWrite, { headers: { Authorization: basic } }); + expect(write.status).toBe(403); + await write.body?.cancel(); + const wrongRepo = await fetch(secondRead, { + headers: { Authorization: `Bearer ${first.token}` }, + }); + expect(wrongRepo.status).toBe(401); + await wrongRepo.body?.cancel(); +}); + +test("artifacts: forks an empty repo and selects which branches to copy", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["empty"]); + const emptyFork = await rpc(mf, "fork", ["empty-fork"], "empty"); + expect(emptyFork.defaultBranch).toBe("main"); + + const directory = path.join(await useTmp(), "source"); + const created = await pushFixture(mf, directory); + await git(["checkout", "-b", "feature"], directory); + await writeFile(path.join(directory, "feature.txt"), "feature branch\n"); + await git(["add", "."], directory); + await git(["commit", "-m", "feature"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "feature", + ], + directory + ); + const mainOnly = await rpc(mf, "fork", ["main-only"], "Repo"); + const all = await rpc( + mf, + "fork", + ["all", { defaultBranchOnly: false, description: "all refs" }], + "Repo" + ); + expect(mainOnly.defaultBranch).toBe("main"); + expect((await rpc(mf, "info", [], "all")).description).toBe("all refs"); + expect( + await rpc(mf, "readFile", [{ ref: "feature", path: "feature.txt" }], "all") + ).toMatchObject({ bytes: [...new TextEncoder().encode("feature branch\n")] }); + expect( + await rpc( + mf, + "readFile", + [{ ref: "feature", path: "feature.txt" }], + "main-only" + ) + ).toBeNull(); + expect(all.token).toEqual(expect.any(String)); +}); + +test("artifacts: log pages commits and files resolve only valid paths", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = path.join(await useTmp(), "source"); + const created = await pushFixture(mf, directory); + const firstHash = (await git(["rev-parse", "HEAD"], directory)).stdout.trim(); + await writeFile(path.join(directory, "second.txt"), "second\n"); + await mkdir(path.join(directory, "folder")); + await writeFile(path.join(directory, "folder", "nested.txt"), "nested\n"); + await git(["add", "."], directory); + await git(["commit", "-m", "second"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "main", + ], + directory + ); + const secondHash = ( + await git(["rev-parse", "HEAD"], directory) + ).stdout.trim(); + expect( + (await rpc(mf, "log", [{ limit: 1 }], "Repo")).map( + (commit: { hash: string }) => commit.hash + ) + ).toEqual([secondHash]); + expect( + (await rpc(mf, "log", [{ limit: 1, offset: 1 }], "Repo")).map( + (commit: { hash: string }) => commit.hash + ) + ).toEqual([firstHash]); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "../hello.txt" }], "Repo") + ).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "folder" }], "Repo") + ).toBeNull(); + expect( + await rpc( + mf, + "readFile", + [{ ref: "main", path: "folder/nested.txt" }], + "Repo" + ) + ).toMatchObject({ bytes: [...new TextEncoder().encode("nested\n")] }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "missing" }], "Repo") + ).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "missing", path: "hello.txt" }], "Repo") + ).toBeNull(); +}); + +test("artifacts: concurrent creations serialize without duplicate repositories", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await Promise.all( + Array.from({ length: 8 }, (_, index) => + rpc(mf, "create", [`repo-${index}`]) + ) + ); + const duplicate = await Promise.allSettled([ + rpc(mf, "create", ["same"]), + rpc(mf, "create", ["same"]), + ]); + expect(duplicate.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + expect((await rpc(mf, "list")).total).toBe(9); +}); + +test("artifacts: an explicit reset removes both metadata and Git repositories", async ({ + expect, +}) => { + const persistence = await useTmp(); + const first = new Miniflare({ + ...options(), + resourcePersistencePath: persistence, + }); + useDispose(first); + const created = await pushFixture(first, path.join(await useTmp(), "source")); + await first.dispose(); + await removeDir(path.join(persistence, "artifacts")); + const second = new Miniflare({ + ...options(), + resourcePersistencePath: persistence, + }); + useDispose(second); + expect((await rpc(second, "list")).total).toBe(0); + const fresh = await rpc(second, "create", ["Repo"]); + expect(fresh.id).not.toBe(created.id); + const oldToken = await fetch( + `${fresh.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${created.token}` } } + ); + expect(oldToken.status).toBe(401); + await oldToken.body?.cancel(); +}); + +test("artifacts: native Git absence fails before starting a sidecar listener", async ({ + expect, +}) => { + const root = await useTmp(); + const previousPath = process.env.PATH; + process.env.PATH = root; + try { + await expect(startGitSidecar(path.join(root, "repos"))).rejects.toThrow( + /spawn git|ENOENT/ + ); + } finally { + process.env.PATH = previousPath; + } +}); + +test("artifacts: disposing the sidecar closes its private listener", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + const address = `http://${sidecar.address}/missing`; + try { + const response = await fetch(address, { + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + }); + expect(response.status).toBe(404); + await response.body?.cancel(); + } finally { + await sidecar.close(); + } + await expect(fetch(address)).rejects.toThrow(); +}); diff --git a/packages/remote-bindings/src/maybe-start-or-update-session.test.ts b/packages/remote-bindings/src/maybe-start-or-update-session.test.ts index e88470ad7e8..2b0f8100568 100644 --- a/packages/remote-bindings/src/maybe-start-or-update-session.test.ts +++ b/packages/remote-bindings/src/maybe-start-or-update-session.test.ts @@ -35,6 +35,20 @@ describe("maybeStartOrUpdateRemoteProxySession", () => { } ); + it("keeps Artifacts offline unless remote is explicitly enabled", ({ + expect, + }) => { + const local = { type: "artifacts" as const, namespace: "local" }; + const remote = { + type: "artifacts" as const, + namespace: "remote", + remote: true, + }; + expect(pickRemoteBindings({ LOCAL: local, REMOTE: remote })).toEqual({ + REMOTE: remote, + }); + }); + it("updates an existing session when all remote bindings are removed", async ({ expect, }) => { diff --git a/packages/workers-utils/src/config/binding-local-support.ts b/packages/workers-utils/src/config/binding-local-support.ts index 724bb7f2619..636a8a8d96a 100644 --- a/packages/workers-utils/src/config/binding-local-support.ts +++ b/packages/workers-utils/src/config/binding-local-support.ts @@ -58,6 +58,7 @@ const BINDING_LOCAL_SUPPORT: Record< // TODO: Miniflare currently ignores `remote: true` on queues, tracked in #13727. queue: "local-and-remote", flagship: "local-and-remote", + artifacts: "local-and-remote", vectorize: "remote", mtls_certificate: "remote", @@ -70,7 +71,6 @@ const BINDING_LOCAL_SUPPORT: Record< ai_search_namespace: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", media: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", - artifacts: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", vpc_service: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", vpc_network: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", agent_memory: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", diff --git a/packages/workers-utils/tests/config/binding-local-support.test.ts b/packages/workers-utils/tests/config/binding-local-support.test.ts index 63c5eb6b989..2f7ee953d98 100644 --- a/packages/workers-utils/tests/config/binding-local-support.test.ts +++ b/packages/workers-utils/tests/config/binding-local-support.test.ts @@ -90,6 +90,12 @@ describe("validateBindingRemoteSetting", () => { expect(() => validateBindingRemoteSetting("flagship", undefined, warn) ).not.toThrow(); + expect(() => + validateBindingRemoteSetting("artifacts", false, warn) + ).not.toThrow(); + expect(() => + validateBindingRemoteSetting("artifacts", undefined, warn) + ).not.toThrow(); expect(warn).not.toHaveBeenCalled(); }); }); diff --git a/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts b/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts index 7cbd44e459a..4db0ec4e663 100644 --- a/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts +++ b/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts @@ -523,12 +523,14 @@ describe("dev with remote bindings", { sequential: true, retry: 2 }, () => { { binding: "MY_ARTIFACTS", namespace: "default", + remote: true, }, ], }, expectedProxyWorkerBindings: { MY_ARTIFACTS: { namespace: "default", + remote: true, type: "artifacts", }, }, @@ -671,6 +673,36 @@ describe("dev with remote bindings", { sequential: true, retry: 2 }, () => { await wranglerStopped; }); + it("keeps Artifacts local when remote is omitted", async ({ expect }) => { + await seed({ + "wrangler.jsonc": JSON.stringify({ + name: "worker", + main: "index.js", + compatibility_date: "2025-01-01", + artifacts: [{ binding: "MY_ARTIFACTS", namespace: "default" }], + }), + "index.js": `export default { fetch() { return new Response("hello") } }`, + }); + const wranglerStopped = runWrangler("dev --port=0 --inspector-port=0"); + await vi.waitFor(() => expect(std.out).toMatch(/Ready/), { + timeout: devReadyTimeout, + }); + expect(startRemoteProxySessionCallCount).toBe(0); + expect(proxyWorkerBindings).toBeUndefined(); + expect(workerOptions).toEqual([ + expect.objectContaining({ + artifacts: { + MY_ARTIFACTS: { + namespace: "default", + remoteProxyConnectionString: undefined, + }, + }, + }), + ]); + await stopWrangler(); + await wranglerStopped; + }); + it("should not recreate the remote proxy session on reload when auth has not changed", async () => { const { maybeStartOrUpdateRemoteProxySession } = await import("../../api/remoteBindings"); diff --git a/packages/wrangler/src/dev/miniflare/index.ts b/packages/wrangler/src/dev/miniflare/index.ts index da6c613fc7d..b0ffa13e563 100644 --- a/packages/wrangler/src/dev/miniflare/index.ts +++ b/packages/wrangler/src/dev/miniflare/index.ts @@ -960,7 +960,9 @@ export function buildMiniflareBindingOptions( binding.binding, { namespace: binding.namespace, - remoteProxyConnectionString, + remoteProxyConnectionString: binding.remote + ? remoteProxyConnectionString + : undefined, }, ]) ), From 91146b896641ddddedbe5824343d12b6c4fc5524 Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Tue, 6 Oct 2026 17:08:05 +0100 Subject: [PATCH 02/21] [miniflare] Fix Artifacts diagram and Windows Git configuration --- .changeset/local-artifacts-binding.md | 1 - packages/miniflare/src/plugins/artifacts/README.md | 2 +- packages/miniflare/src/plugins/artifacts/git-client.ts | 3 ++- packages/miniflare/test/plugins/artifacts/index.spec.ts | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.changeset/local-artifacts-binding.md b/.changeset/local-artifacts-binding.md index e402cd20fa2..546c6af0120 100644 --- a/.changeset/local-artifacts-binding.md +++ b/.changeset/local-artifacts-binding.md @@ -2,7 +2,6 @@ "miniflare": minor "wrangler": minor "@cloudflare/workers-utils": minor -"@cloudflare/remote-bindings": minor --- Add a local Artifacts binding simulator backed by native Git for Worker development diff --git a/packages/miniflare/src/plugins/artifacts/README.md b/packages/miniflare/src/plugins/artifacts/README.md index 52f8f0cf077..1844a2f9b7b 100644 --- a/packages/miniflare/src/plugins/artifacts/README.md +++ b/packages/miniflare/src/plugins/artifacts/README.md @@ -17,7 +17,7 @@ sequenceDiagram Dev->>MF: Configure Artifacts binding MF->>Plugin: getBindings() - Plugin->>Plugin: Select local; validate namespace + Plugin->>Plugin: Select local and validate namespace Plugin-->>MF: Bind to local namespace entrypoint MF->>Plugin: getServices() Plugin->>Controller: get(Git storage path) diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts index 1bd680a818c..3989456f735 100644 --- a/packages/miniflare/src/plugins/artifacts/git-client.ts +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -31,7 +31,8 @@ export function gitEnvironment( return { ...env, GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: devNull, + // Git for Windows cannot open Node's devNull path; use its NUL alias. + GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : devNull, GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "credential.helper", GIT_CONFIG_VALUE_0: "", diff --git a/packages/miniflare/test/plugins/artifacts/index.spec.ts b/packages/miniflare/test/plugins/artifacts/index.spec.ts index 9d7097e766f..e866362d058 100644 --- a/packages/miniflare/test/plugins/artifacts/index.spec.ts +++ b/packages/miniflare/test/plugins/artifacts/index.spec.ts @@ -178,7 +178,9 @@ test("artifacts: native Git subprocesses do not inherit Cloudflare or host crede expect(env.GIT_ASKPASS).toBeUndefined(); expect(env.GITHUB_TOKEN).toBeUndefined(); expect(env.GIT_CONFIG_VALUE_0).toBe(""); - expect(env.GIT_CONFIG_GLOBAL).toBe(devNull); + expect(env.GIT_CONFIG_GLOBAL).toBe( + process.platform === "win32" ? "NUL" : devNull + ); }); test("artifacts: Git client imports and reads a local fixture", async ({ From bde02ee113ee2b153b972d9f33a3789652aca4e9 Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Tue, 6 Oct 2026 17:23:39 +0100 Subject: [PATCH 03/21] [miniflare] Use filesystem-safe Artifacts storage keys --- packages/miniflare/src/plugins/artifacts/index.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 737455ced87..2abe6fec3d7 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -49,7 +49,8 @@ function namespaceId(namespace: string): string { } function localServiceName(namespace: string): string { - return `${ARTIFACTS_PLUGIN_NAME}:${namespaceId(namespace)}`; + // Workerd also uses this as the Durable Object storage key/directory name. + return `${ARTIFACTS_PLUGIN_NAME}-${namespaceId(namespace)}`; } export const ARTIFACTS_PLUGIN: Plugin = { @@ -131,10 +132,10 @@ async function createLocalNamespaceRuntime( path.join(persistPath, "git") ); const serviceName = localServiceName(namespace); - const storageName = `${serviceName}:storage`; - const backendName = `${serviceName}:backend`; + const storageName = `${serviceName}-storage`; + const backendName = `${serviceName}-backend`; const socket: Socket = { - name: `${serviceName}:git`, + name: `${serviceName}-git`, address: `127.0.0.1:${port}`, http: {}, service: { name: serviceName, entrypoint: ENTRYPOINT }, From 2ddaf967400515e673d8a543f6ca5ee55f912b44 Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Tue, 6 Oct 2026 17:33:27 +0100 Subject: [PATCH 04/21] [miniflare] Shorten Artifacts namespace storage paths --- packages/miniflare/src/plugins/artifacts/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 2abe6fec3d7..17ab9f2ada3 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -45,7 +45,8 @@ function validateLocalNamespace(namespace: string): void { } function namespaceId(namespace: string): string { - return createHash("sha256").update(namespace).digest("hex"); + // 128 bits keeps storage paths short enough for workerd's SQLite files on Windows. + return createHash("sha256").update(namespace).digest("hex").slice(0, 32); } function localServiceName(namespace: string): string { From e334470b96fc204a0fab012a965f664bf0140473 Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Tue, 6 Oct 2026 17:54:40 +0100 Subject: [PATCH 05/21] [miniflare] Isolate Git configuration on Windows --- .../miniflare/src/plugins/artifacts/git-client.ts | 12 +++++++++--- .../miniflare/test/plugins/artifacts/index.spec.ts | 13 ++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts index 3989456f735..ac7f3c7e9d0 100644 --- a/packages/miniflare/src/plugins/artifacts/git-client.ts +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -1,6 +1,8 @@ import { Buffer } from "node:buffer"; import { spawn } from "node:child_process"; -import { devNull } from "node:os"; +import { randomUUID } from "node:crypto"; +import { devNull, tmpdir } from "node:os"; +import path from "node:path"; const decoder = new TextDecoder(); type GitOptions = { allowFailure?: boolean; timeout?: number }; @@ -31,8 +33,12 @@ export function gitEnvironment( return { ...env, GIT_CONFIG_NOSYSTEM: "1", - // Git for Windows cannot open Node's devNull path; use its NUL alias. - GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : devNull, + // Git for Windows cannot read null-device paths. A unique, nonexistent + // config path prevents it from falling back to the host's Git settings. + GIT_CONFIG_GLOBAL: + process.platform === "win32" + ? path.join(tmpdir(), `miniflare-artifacts-empty-${randomUUID()}`) + : devNull, GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "credential.helper", GIT_CONFIG_VALUE_0: "", diff --git a/packages/miniflare/test/plugins/artifacts/index.spec.ts b/packages/miniflare/test/plugins/artifacts/index.spec.ts index e866362d058..61c60d69555 100644 --- a/packages/miniflare/test/plugins/artifacts/index.spec.ts +++ b/packages/miniflare/test/plugins/artifacts/index.spec.ts @@ -109,10 +109,7 @@ function git(args: string[], cwd?: string) { return exec("git", args, { cwd, env: { - ...process.env, - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: devNull, - GIT_TERMINAL_PROMPT: "0", + ...gitEnvironment(), GIT_AUTHOR_NAME: "Miniflare", GIT_AUTHOR_EMAIL: "miniflare@example.com", GIT_COMMITTER_NAME: "Miniflare", @@ -178,9 +175,11 @@ test("artifacts: native Git subprocesses do not inherit Cloudflare or host crede expect(env.GIT_ASKPASS).toBeUndefined(); expect(env.GITHUB_TOKEN).toBeUndefined(); expect(env.GIT_CONFIG_VALUE_0).toBe(""); - expect(env.GIT_CONFIG_GLOBAL).toBe( - process.platform === "win32" ? "NUL" : devNull - ); + if (process.platform === "win32") { + expect(env.GIT_CONFIG_GLOBAL).toContain("miniflare-artifacts-empty-"); + } else { + expect(env.GIT_CONFIG_GLOBAL).toBe(devNull); + } }); test("artifacts: Git client imports and reads a local fixture", async ({ From 37e8d141e68c0a18f9f397a31739cad6444d195c Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Tue, 6 Oct 2026 18:20:38 +0100 Subject: [PATCH 06/21] [miniflare] Test isolated Artifacts sidecar creation --- .../test/plugins/artifacts/index.spec.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/miniflare/test/plugins/artifacts/index.spec.ts b/packages/miniflare/test/plugins/artifacts/index.spec.ts index 61c60d69555..113c08de6d9 100644 --- a/packages/miniflare/test/plugins/artifacts/index.spec.ts +++ b/packages/miniflare/test/plugins/artifacts/index.spec.ts @@ -794,6 +794,33 @@ test("artifacts: native Git absence fails before starting a sidecar listener", a } }); +test("artifacts: sidecar creates a repository without host Git config", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "create", + namespace: "test", + name: "repo", + generation: "test-generation", + defaultBranch: "main", + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + expect(JSON.parse(body)).toEqual({ defaultBranch: "main", refs: {} }); + } finally { + await sidecar.close(); + } +}); + test("artifacts: disposing the sidecar closes its private listener", async ({ expect, }) => { From c415ffc778ae11f3581091443e184240eff47d64 Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Wed, 7 Oct 2026 11:59:34 +0100 Subject: [PATCH 07/21] [miniflare] Preserve shallow Git boundaries and show local Artifacts mode --- .../src/plugins/artifacts/git-client.ts | 7 -- .../src/plugins/artifacts/git-sidecar.ts | 68 +------------------ .../test/plugins/artifacts/index.spec.ts | 66 +++++++++++++++++- packages/workers-utils/src/print-bindings.ts | 6 +- .../tests/print-bindings.test.ts | 24 +++++++ .../src/__tests__/print-bindings.test.ts | 35 ++++++---- 6 files changed, 118 insertions(+), 88 deletions(-) diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts index ac7f3c7e9d0..927e716278b 100644 --- a/packages/miniflare/src/plugins/artifacts/git-client.ts +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -314,13 +314,6 @@ export class GitClient { ); } - async rootCommits(): Promise> { - const result = await this.git(["rev-list", "--max-parents=0", "--all"]); - return new Set( - decoder.decode(result.stdout).trim().split("\n").filter(Boolean) - ); - } - private async objectType(hash: string): Promise { const result = await this.git(["cat-file", "-t", hash], { allowFailure: true, diff --git a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts index 26b0cf3edde..80342b57162 100644 --- a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts +++ b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts @@ -164,7 +164,6 @@ async function handleRequest( } await runGitHttpBackend( root, - path, request, response, `/${storageComponent(namespace)}/${storageComponent(repository)}.git${suffix}`, @@ -331,16 +330,11 @@ function parseGitBackendHeaders(headerBytes: Buffer): { async function runGitHttpBackend( root: string, - repository: string, request: IncomingMessage, response: ServerResponse, pathInfo: string, query: string ): Promise { - const rootCommits = pathInfo.endsWith("/git-upload-pack") - ? await new GitClient(repository).rootCommits() - : new Set(); - const shallowFilter = filterRootShallowPackets(rootCommits); return new Promise((resolve, reject) => { const child = spawn("git", ["http-backend"], { env: { @@ -393,12 +387,6 @@ async function runGitHttpBackend( response.once("drain", () => child.stdout.resume()); } }; - const writeBody = (chunk: Buffer) => { - const filtered = shallowFilter.write(chunk); - if (filtered.length) { - write(filtered); - } - }; request.once("aborted", abort); response.once("close", abort); child.stdin.on("error", (error) => { @@ -409,7 +397,7 @@ async function runGitHttpBackend( child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); child.stdout.on("data", (chunk: Buffer) => { if (headersSent) { - writeBody(chunk); + write(chunk); return; } header = Buffer.concat([header, chunk]); @@ -430,15 +418,11 @@ async function runGitHttpBackend( headersSent = true; const remaining = header.subarray(index + separatorLength); if (remaining.length) { - writeBody(remaining); + write(remaining); } header = Buffer.alloc(0); }); child.stdout.on("end", () => { - const remaining = shallowFilter.end(); - if (remaining.length) { - write(remaining); - } if (headersSent) { response.end(); } @@ -464,54 +448,6 @@ async function runGitHttpBackend( }); } -function filterRootShallowPackets(rootCommits: Set): { - write(chunk: Buffer): Buffer; - end(): Buffer; -} { - let pending = Buffer.alloc(0); - let passthrough = rootCommits.size === 0; - return { - write(chunk) { - if (passthrough) { - return chunk; - } - pending = Buffer.concat([pending, chunk]); - const output: Buffer[] = []; - while (pending.length >= 4) { - const prefix = pending.subarray(0, 4).toString("ascii"); - if (!/^[0-9a-f]{4}$/i.test(prefix)) { - passthrough = true; - output.push(pending); - pending = Buffer.alloc(0); - break; - } - const length = Number.parseInt(prefix, 16); - if (length === 0 || length === 1 || length === 2) { - output.push(pending.subarray(0, 4)); - pending = pending.subarray(4); - continue; - } - if (length < 4 || pending.length < length) { - break; - } - const packet = pending.subarray(0, length); - const payload = packet.subarray(4).toString("ascii"); - const shallowHash = /^shallow ([0-9a-f]{40})\n?$/.exec(payload)?.[1]; - if (!shallowHash || !rootCommits.has(shallowHash)) { - output.push(packet); - } - pending = pending.subarray(length); - } - return Buffer.concat(output); - }, - end() { - const result = pending; - pending = Buffer.alloc(0); - return result; - }, - }; -} - function readRequest(request: IncomingMessage): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; diff --git a/packages/miniflare/test/plugins/artifacts/index.spec.ts b/packages/miniflare/test/plugins/artifacts/index.spec.ts index 113c08de6d9..ef372b0eaeb 100644 --- a/packages/miniflare/test/plugins/artifacts/index.spec.ts +++ b/packages/miniflare/test/plugins/artifacts/index.spec.ts @@ -105,11 +105,12 @@ async function rpcFailure( }; } -function git(args: string[], cwd?: string) { +function git(args: string[], cwd?: string, extraEnv: NodeJS.ProcessEnv = {}) { return exec("git", args, { cwd, env: { ...gitEnvironment(), + ...extraEnv, GIT_AUTHOR_NAME: "Miniflare", GIT_AUTHOR_EMAIL: "miniflare@example.com", GIT_COMMITTER_NAME: "Miniflare", @@ -821,6 +822,69 @@ test("artifacts: sidecar creates a repository without host Git config", async ({ } }); +test("artifacts: shallow imports stay shallow when cloned with Git protocol v2", async ({ + expect, +}) => { + const workspace = await useTmp(); + const source = path.join(workspace, "source"); + await git(["init", "--initial-branch=main", source]); + for (const index of [1, 2, 3]) { + await writeFile(path.join(source, "README"), `version ${index}\n`); + await git(["add", "README"], source); + await git(["commit", "-m", `version ${index}`], source); + } + + const sidecar = await startGitSidecar(path.join(workspace, "repos")); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "import", + namespace: "test", + name: "repo", + generation: "shallow-generation", + sourceUrl: pathToFileURL(source).href, + branch: "main", + depth: 1, + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + + const gitHeaders = { + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `X-Local-Artifacts-Backend: ${sidecar.secret}`, + GIT_CONFIG_KEY_2: "http.extraHeader", + GIT_CONFIG_VALUE_2: "X-Local-Artifacts-Generation: shallow-generation", + }; + const clone = path.join(workspace, "clone"); + const remote = `http://${sidecar.address}/git/test/repo.git`; + await git( + ["-c", "protocol.version=2", "clone", remote, clone], + undefined, + gitHeaders + ); + expect( + (await git(["rev-parse", "--is-shallow-repository"], clone)).stdout.trim() + ).toBe("true"); + expect( + (await git(["rev-list", "--count", "HEAD"], clone)).stdout.trim() + ).toBe("1"); + await git( + ["-c", "protocol.version=2", "fetch", "origin"], + clone, + gitHeaders + ); + } finally { + await sidecar.close(); + } +}); + test("artifacts: disposing the sidecar closes its private listener", async ({ expect, }) => { diff --git a/packages/workers-utils/src/print-bindings.ts b/packages/workers-utils/src/print-bindings.ts index f07de4ce3bb..dd13beca9ee 100644 --- a/packages/workers-utils/src/print-bindings.ts +++ b/packages/workers-utils/src/print-bindings.ts @@ -457,12 +457,14 @@ export function printBindings( if (artifacts.length > 0) { output.push( - ...artifacts.map(({ binding, namespace }) => { + ...artifacts.map(({ binding, namespace, remote }) => { return { name: binding, type: getBindingTypeFriendlyName("artifacts"), value: namespace, - mode: getMode({ isSimulatedLocally: false }), + mode: getMode({ + isSimulatedLocally: context.remoteBindingsDisabled || !remote, + }), }; }) ); diff --git a/packages/workers-utils/tests/print-bindings.test.ts b/packages/workers-utils/tests/print-bindings.test.ts index ccefdc7ff2d..f29177ab10b 100644 --- a/packages/workers-utils/tests/print-bindings.test.ts +++ b/packages/workers-utils/tests/print-bindings.test.ts @@ -80,6 +80,30 @@ describe("printBindings", () => { expect(output).toMatch(/K2 Stream\s+not supported/); }); + test.for([ + { label: "default", remote: undefined, disabled: false, mode: "local" }, + { label: "explicit local", remote: false, disabled: false, mode: "local" }, + { label: "explicit remote", remote: true, disabled: false, mode: "remote" }, + { label: "remote disabled", remote: true, disabled: true, mode: "local" }, + ])( + "labels Artifacts bindings as $mode for $label", + ({ remote, disabled, mode }, { expect }) => { + const output = captureBindings( + { + REPOS: { + type: "artifacts", + namespace: "review", + ...(remote === undefined ? {} : { remote }), + }, + }, + { local: true, remoteBindingsDisabled: disabled } + ); + expect(output).toMatch( + new RegExp(`env\\.REPOS \\(review\\)\\s+Artifacts\\s+${mode}`) + ); + } + ); + test.for([{ class_name: "Sandbox" }, { name: "Sandbox" }])( "prints Durable Object-managed containers using their configured identity %j", (identity, { expect }) => { diff --git a/packages/wrangler/src/__tests__/print-bindings.test.ts b/packages/wrangler/src/__tests__/print-bindings.test.ts index 8723079ca5d..68ab3548f5e 100644 --- a/packages/wrangler/src/__tests__/print-bindings.test.ts +++ b/packages/wrangler/src/__tests__/print-bindings.test.ts @@ -176,16 +176,27 @@ describe("printBindings -- Flagship bindings", () => { }); describe("printBindings -- Artifacts bindings", () => { - it("shows Artifacts bindings", ({ expect }) => { - const output = callPrintBindings({ - MY_ARTIFACTS: { - type: "artifacts", - namespace: "default", - }, - }); - - expect(output).toContain("MY_ARTIFACTS"); - expect(output).toContain("Artifacts"); - expect(output).toContain("default"); - }); + it.for([ + { label: "default", remote: undefined, mode: "local" }, + { label: "explicit local", remote: false, mode: "local" }, + { label: "explicit remote", remote: true, mode: "remote" }, + ])( + "shows $label Artifacts bindings as $mode", + ({ remote, mode }, { expect }) => { + const output = callPrintBindings( + { + MY_ARTIFACTS: { + type: "artifacts", + namespace: "default", + ...(remote === undefined ? {} : { remote }), + }, + }, + true + ); + + expect(output).toMatch( + new RegExp(`env\\.MY_ARTIFACTS \\(default\\)\\s+Artifacts\\s+${mode}`) + ); + } + ); }); From b360a37e1d2cdd07f33bb54e15ce14bbfd2a2bf6 Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Wed, 7 Oct 2026 12:44:16 +0100 Subject: [PATCH 08/21] [miniflare] Shorten local Artifacts storage paths and reject draft layouts --- .../miniflare/src/plugins/artifacts/README.md | 9 +- .../src/plugins/artifacts/git-sidecar.ts | 30 +--- .../miniflare/src/plugins/artifacts/index.ts | 9 +- .../src/plugins/artifacts/storage.ts | 40 +++++ .../test/plugins/artifacts/storage.spec.ts | 165 ++++++++++++++++++ 5 files changed, 229 insertions(+), 24 deletions(-) create mode 100644 packages/miniflare/src/plugins/artifacts/storage.ts create mode 100644 packages/miniflare/test/plugins/artifacts/storage.spec.ts diff --git a/packages/miniflare/src/plugins/artifacts/README.md b/packages/miniflare/src/plugins/artifacts/README.md index 1844a2f9b7b..48a1342fca0 100644 --- a/packages/miniflare/src/plugins/artifacts/README.md +++ b/packages/miniflare/src/plugins/artifacts/README.md @@ -65,7 +65,14 @@ the sidecar stores the bare Git repositories separately. `git-sidecar.ts` handles HTTP routing and repository locks, while `git-client.ts` owns repository-scoped Git commands and result parsing. Read operations such as `readFile()` also use the binding Worker and native Git backend. Node callers -reach the same binding through Miniflare's Node binding proxy. +reach the same binding through Miniflare's Node binding proxy. Namespace and +repository names are hashed into short, namespace-scoped on-disk paths; this +leaves room for Git packfiles and workerd SQLite files on platforms with path +length limits. Older draft storage layouts are not migrated automatically: +startup rejects a detected old layout rather than silently creating empty +repositories. Back up any local repositories before resetting the Artifacts +persistence directory. Do not delete the persistence directory without first +checking whether it holds work you need. ## Git clone and push diff --git a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts index 80342b57162..ae34483645d 100644 --- a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts +++ b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts @@ -1,16 +1,20 @@ import { Buffer } from "node:buffer"; import { spawn } from "node:child_process"; -import { createHash, randomBytes } from "node:crypto"; +import { randomBytes } from "node:crypto"; import { mkdir, stat } from "node:fs/promises"; import { createServer, type IncomingMessage, type ServerResponse, } from "node:http"; -import { join } from "node:path"; import { parentPort, workerData } from "node:worker_threads"; import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; import { GitClient, gitEnvironment, runGit } from "./git-client"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "./storage"; import type { RepositoryState } from "./git-client"; const adminPath = "/__local_artifacts__"; @@ -70,6 +74,7 @@ function createGitSecret(): string { export async function startGitSidecar(root: string): Promise { await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); await runGit(["--version"]); const secret = createGitSecret(); const server = createServer((request, response) => { @@ -166,7 +171,7 @@ async function handleRequest( root, request, response, - `/${storageComponent(namespace)}/${storageComponent(repository)}.git${suffix}`, + `/${repositoryDirectory(namespace, repository)}${suffix}`, url.search.slice(1) ); }); @@ -282,9 +287,7 @@ async function withNewRepository( git: GitClient, initialize: () => Promise ): Promise { - await mkdir(repositoryNamespacePath(root, body.namespace), { - recursive: true, - }); + await mkdir(root, { recursive: true }); if (await exists(git.repository)) { throw new Error(`Repository "${body.name}" already exists on disk`); } @@ -457,21 +460,6 @@ function readRequest(request: IncomingMessage): Promise { }); } -function repositoryPath(root: string, namespace: string, name: string): string { - return join( - repositoryNamespacePath(root, namespace), - `${storageComponent(name)}.git` - ); -} - -function repositoryNamespacePath(root: string, namespace: string): string { - return join(root, storageComponent(namespace)); -} - -function storageComponent(value: string): string { - return createHash("sha256").update(value).digest("hex"); -} - function validateComponent(value: string, type: string): void { if (!componentPattern.test(value)) { throw new Error(`Invalid ${type}`); diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 17ab9f2ada3..d611b1c8b91 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -10,6 +10,7 @@ import { ProxyNodeBinding, remoteProxyClientWorker, } from "../shared"; +import { assertSupportedGitLayout } from "./storage"; import type { Service, Socket } from "../../runtime"; import type { MiniflareBinding, Plugin } from "../shared"; import type { ArtifactsController } from "./controller"; @@ -50,7 +51,6 @@ function namespaceId(namespace: string): string { } function localServiceName(namespace: string): string { - // Workerd also uses this as the Durable Object storage key/directory name. return `${ARTIFACTS_PLUGIN_NAME}-${namespaceId(namespace)}`; } @@ -126,9 +126,12 @@ async function createLocalNamespaceRuntime( namespace: string, artifactsController: ArtifactsController ): Promise<{ services: Service[]; socket: Socket }> { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); const persistPath = path.join(root, namespaceId(namespace)); const metadataPath = path.join(persistPath, "metadata").replaceAll("\\", "/"); await mkdir(metadataPath, { recursive: true }); + await assertSupportedGitLayout(metadataPath); const { sidecar, port } = await artifactsController.get( path.join(persistPath, "git") ); @@ -175,8 +178,10 @@ async function createLocalNamespaceRuntime( }, { name: "gitBackend", service: { name: backendName } }, ], + // Keys must be unique across namespaces in one workerd, but should + // not repeat the long service prefix in SQLite's directory name. durableObjectNamespaces: [ - { className: OBJECT_CLASS, uniqueKey: serviceName }, + { className: OBJECT_CLASS, uniqueKey: namespaceId(namespace) }, ], durableObjectStorage: { localDisk: storageName }, }, diff --git a/packages/miniflare/src/plugins/artifacts/storage.ts b/packages/miniflare/src/plugins/artifacts/storage.ts new file mode 100644 index 00000000000..ac693dc1bcc --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/storage.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { readdir } from "node:fs/promises"; +import path from "node:path"; + +/** + * Keep repository paths short: native Git adds long pack and lock filenames. + * Hash the tuple rather than nesting namespace and repository hashes. Each + * identifier retains 128 bits and no user input becomes a path component. + */ +export function repositoryDirectory(namespace: string, name: string): string { + const id = createHash("sha256") + .update(JSON.stringify([namespace, name])) + .digest("hex") + .slice(0, 32); + return `${id}.git`; +} + +export function repositoryPath( + root: string, + namespace: string, + name: string +): string { + return path.join(root, repositoryDirectory(namespace, name)); +} + +/** Do not quietly abandon repositories written by earlier unreleased layouts. */ +export async function assertSupportedGitLayout(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + if ( + entries.some( + (entry) => + entry.isDirectory() && + (/^[0-9a-f]{64}$/.test(entry.name) || /^artifacts[-:]/.test(entry.name)) + ) + ) { + throw new Error( + "Local Artifacts storage uses an older draft layout. Back up your local repositories, then reset the Artifacts persistence directory before restarting." + ); + } +} diff --git a/packages/miniflare/test/plugins/artifacts/storage.spec.ts b/packages/miniflare/test/plugins/artifacts/storage.spec.ts new file mode 100644 index 00000000000..a87a2935673 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/storage.spec.ts @@ -0,0 +1,165 @@ +import { createHash } from "node:crypto"; +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { GitClient, runGit } from "../../../src/plugins/artifacts/git-client"; +import { startGitSidecar } from "../../../src/plugins/artifacts/git-sidecar"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "../../../src/plugins/artifacts/storage"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; + +const namespace = "namespace-with-a-long-name-for-platform-testing"; + +test("artifacts: repository paths are compact, deterministic and namespace-scoped", ({ + expect, +}) => { + const directory = repositoryDirectory(namespace, "repo"); + expect(directory).toMatch(/^[0-9a-f]{32}\.git$/); + expect(repositoryDirectory(namespace, "repo")).toBe(directory); + expect(repositoryDirectory("other", "repo")).not.toBe(directory); + expect(repositoryDirectory(namespace, "other")).not.toBe(directory); + expect(repositoryDirectory("ab", "c")).not.toBe( + repositoryDirectory("a", "bc") + ); + expect(repositoryDirectory("../namespace", "../repo")).toMatch( + /^[0-9a-f]{32}\.git$/ + ); +}); + +test("artifacts: sidecar rejects legacy storage without removing it", async ({ + expect, +}) => { + const root = await useTmp(); + const legacy = path.join(root, "a".repeat(64)); + await mkdir(legacy); + await writeFile(path.join(legacy, "sentinel"), "preserve me"); + await expect(startGitSidecar(root)).rejects.toThrow( + /older draft layout.*Back up/ + ); + expect(await readFile(path.join(legacy, "sentinel"), "utf8")).toBe( + "preserve me" + ); +}); + +test("artifacts: legacy metadata layout is detected without resetting it", async ({ + expect, +}) => { + const root = await useTmp(); + const id = createHash("sha256").update(namespace).digest("hex").slice(0, 32); + const metadata = path.join(root, "artifacts", id, "metadata"); + const legacy = path.join(metadata, `artifacts-${id}`); + await mkdir(legacy, { recursive: true }); + await writeFile(path.join(legacy, "sentinel"), "preserve me"); + await expect(assertSupportedGitLayout(metadata)).rejects.toThrow( + /older draft layout.*Back up/ + ); + expect(await readFile(path.join(legacy, "sentinel"), "utf8")).toBe( + "preserve me" + ); +}); + +test("artifacts: namespace metadata and Git data survive restart under paths with spaces", async ({ + expect, +}) => { + const root = path.join(await useTmp(), "development project"); + const config = { + name: "test", + compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts" as const, namespace } }, + manifest: singleModuleManifest( + `export default { fetch() { return new Response("ok"); } };` + ), + }; + type Bindings = { + REPOS: { + create( + name: string + ): Promise<{ id: string; remote: string; token: string }>; + get(name: string): Promise<{ info(): Promise<{ id: string }> }>; + }; + }; + const options = { + cf: false as const, + resourcePersistencePath: root, + workers: [{ config }], + }; + const first = new Miniflare(options); + useDispose(first); + const { REPOS } = await first.getBindings(); + const created = await REPOS.create("repo"); + const id = created.id; + await first.dispose(); + + const second = new Miniflare(options); + useDispose(second); + const restarted = await second.getBindings(); + expect(await (await restarted.REPOS.get("repo")).info()).toMatchObject({ + id, + }); + const namespaces = await readdir(path.join(root, "artifacts")); + expect(namespaces).toHaveLength(1); + const persisted = path.join(root, "artifacts", namespaces[0]); + expect(await readdir(path.join(persisted, "git"))).toEqual([ + repositoryDirectory(namespace, "repo"), + ]); + expect( + (await readdir(path.join(persisted, "metadata"))).length + ).toBeGreaterThan(0); +}); + +test("artifacts: long Git storage roots can initialize and repack repositories", async ({ + expect, +}) => { + const workspace = await useTmp(); + const root = path.join(workspace, "project with spaces " + "x".repeat(32)); + const sidecar = await startGitSidecar(root); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "create", + namespace, + name: "repo", + generation: "test-generation", + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + const repository = repositoryPath(root, namespace, "repo"); + const source = path.join(workspace, "source"); + await runGit(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "long-path regression\n"); + await runGit(["-C", source, "add", "README"]); + await runGit([ + "-C", + source, + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-m", + "fixture", + ]); + await runGit(["-C", source, "push", repository, "main"]); + await runGit(["-C", repository, "repack", "-ad"]); + expect( + (await readdir(path.join(repository, "objects", "pack"))).some((file) => + file.endsWith(".pack") + ) + ).toBe(true); + expect(await new GitClient(repository).generation()).toBe( + "test-generation" + ); + } finally { + await sidecar.close(); + } +}); From 2e5cdb4420b1dfa461de092e012ca025ee67bb3b Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Wed, 7 Oct 2026 13:04:12 +0100 Subject: [PATCH 09/21] [miniflare] Cover local HTTPS Artifacts imports with trusted Git CA --- .../src/plugins/artifacts/git-client.ts | 4 + .../plugins/artifacts/https-import.spec.ts | 453 ++++++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 packages/miniflare/test/plugins/artifacts/https-import.spec.ts diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts index 927e716278b..3c5c92b1210 100644 --- a/packages/miniflare/src/plugins/artifacts/git-client.ts +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -22,6 +22,10 @@ export function gitEnvironment( "TEMP", "TMPDIR", "LANG", + // Explicit host CA bundles may be used to trust a private HTTPS Git origin. + // Do not inherit other Git TLS overrides (especially GIT_SSL_NO_VERIFY), + // credential helpers, or HTTP headers from the host environment. + "GIT_SSL_CAINFO", ]; const env: NodeJS.ProcessEnv = {}; for (const key of keep) { diff --git a/packages/miniflare/test/plugins/artifacts/https-import.spec.ts b/packages/miniflare/test/plugins/artifacts/https-import.spec.ts new file mode 100644 index 00000000000..2a474ef3ccf --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/https-import.spec.ts @@ -0,0 +1,453 @@ +import { execFile, spawn } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:https"; +import path from "node:path"; +import { promisify } from "node:util"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { singleModuleManifest, useTmp } from "../../test-shared"; + +const exec = promisify(execFile); +const WORKER = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name ? await env.REPOS.get(name) : env.REPOS; + const result = await target[method](...args); + if (result instanceof Blob) { + return Response.json({ bytes: [...new Uint8Array(await result.arrayBuffer())] }); + } + return Response.json(result); + } catch (error) { + return Response.json({ error: error.message, code: error.code }, { status: 400 }); + } + } +}; +`; + +type ImportOptions = { + source: { url: string; branch?: string; depth?: number }; + target: { name: string }; +}; + +async function git(args: string[], cwd?: string): Promise { + const { stdout } = await exec("git", args, { + cwd, + env: { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }); + return stdout.trim(); +} + +async function makeSource(directory: string): Promise<{ + first: string; + second: string; + feature: string; +}> { + const source = path.join(directory, "source"); + await git(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "first\n"); + await git(["add", "README"], source); + await git(["commit", "-m", "first"], source); + const first = await git(["rev-parse", "HEAD"], source); + await writeFile(path.join(source, "README"), "second\n"); + await git(["commit", "-am", "second"], source); + const second = await git(["rev-parse", "HEAD"], source); + await git(["checkout", "-b", "feature"], source); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git(["add", "feature.txt"], source); + await git(["commit", "-m", "feature"], source); + const feature = await git(["rev-parse", "HEAD"], source); + const bare = path.join(directory, "source.git"); + await git(["clone", "--bare", source, bare]); + await git(["symbolic-ref", "HEAD", "refs/heads/main"], bare); + return { first, second, feature }; +} + +/** Serve native Git's smart HTTP CGI over a local, self-signed HTTPS origin. */ +async function startHttpsGit(directory: string) { + const keyPath = path.join(directory, "key.pem"); + const certPath = path.join(directory, "cert.pem"); + await exec("openssl", [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + "-keyout", + keyPath, + "-out", + certPath, + ]); + const server = createServer( + { key: await readFile(keyPath), cert: await readFile(certPath) }, + (request, response) => { + const auth = `Basic ${Buffer.from("reader:correct-password").toString("base64")}`; + if (request.headers.authorization !== auth) { + response + .writeHead(401, { "WWW-Authenticate": 'Basic realm="fixture"' }) + .end(); + return; + } + if ( + interruptUpload && + request.url?.startsWith("/source.git/git-upload-pack") + ) { + interrupted++; + response.writeHead(200, { + "Content-Type": "application/x-git-upload-pack-result", + }); + response.write("0008NAK\n"); + response.destroy(); + return; + } + const url = new URL(request.url ?? "/", "https://localhost"); + const child = spawn("git", ["http-backend"], { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: directory, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: url.pathname, + QUERY_STRING: url.search.slice(1), + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + }, + stdio: ["pipe", "pipe", "pipe"], + }); + const output: Buffer[] = []; + const errors: Buffer[] = []; + request.pipe(child.stdin); + child.stdout.on("data", (chunk: Buffer) => output.push(chunk)); + child.stderr.on("data", (chunk: Buffer) => errors.push(chunk)); + response.once("close", () => child.kill()); + child.once("error", () => response.destroy()); + child.once("close", (status) => { + if (response.destroyed) { + return; + } + const bytes = Buffer.concat(output); + const separator = bytes.indexOf("\r\n\r\n") >= 0 ? "\r\n\r\n" : "\n\n"; + const end = bytes.indexOf(separator); + if (end < 0) { + response.writeHead(500).end(Buffer.concat(errors)); + return; + } + const headers = bytes.subarray(0, end).toString().split(/\r?\n/); + const statusLine = headers.find((line) => line.startsWith("Status:")); + const statusCode = statusLine + ? Number(statusLine.match(/^Status: (\d{3})/)?.[1]) + : 200; + for (const line of headers) { + const colon = line.indexOf(":"); + if (colon > 0 && !line.startsWith("Status:")) { + response.setHeader( + line.slice(0, colon), + line.slice(colon + 1).trim() + ); + } + } + response.writeHead(status === 0 ? statusCode : 500); + response.end(bytes.subarray(end + separator.length)); + }); + } + ); + let interruptUpload = false; + let interrupted = 0; + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("HTTPS fixture did not bind a port"); + } + return { + certPath, + url: `https://localhost:${address.port}/source.git`, + get interrupted() { + return interrupted; + }, + set interruptUpload(value: boolean) { + interruptUpload = value; + }, + async close() { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +async function call( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise<{ ok: boolean; body: unknown }> { + const response = await mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); + return { ok: response.ok, body: await response.json() }; +} + +function importOptions( + url: string, + name: string, + branch?: string, + depth?: number +): ImportOptions { + return { + source: { url, ...(branch ? { branch } : {}), ...(depth ? { depth } : {}) }, + target: { name }, + }; +} + +async function withFixture( + callback: (context: { + mf: Miniflare; + fixture: Awaited>; + commits: Awaited>; + directory: string; + }) => Promise, + trust = true +) { + const directory = await useTmp(); + const commits = await makeSource(directory); + const fixture = await startHttpsGit(directory); + const previousCA = process.env.GIT_SSL_CAINFO; + // The controller's sidecar snapshots this environment at startup. Git gets + // this CA only via GIT_SSL_CAINFO; neither Node nor workerd trust is changed. + if (trust) { + process.env.GIT_SSL_CAINFO = fixture.certPath; + } else { + delete process.env.GIT_SSL_CAINFO; + } + let mf: Miniflare | undefined; + try { + mf = new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts", namespace: "https-import" } }, + manifest: singleModuleManifest(WORKER), + }, + }, + ], + }); + await mf.ready; + restoreCA(previousCA); + await callback({ mf, fixture, commits, directory }); + } finally { + restoreCA(previousCA); + try { + await mf?.dispose(); + } finally { + await fixture.close(); + } + } +} + +function restoreCA(value: string | undefined): void { + if (value === undefined) { + delete process.env.GIT_SSL_CAINFO; + } else { + process.env.GIT_SSL_CAINFO = value; + } +} + +test("HTTPS import: full import exposes refs, file, log and Git clone through the public binding", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, commits, directory }) => { + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const imported = await call(mf, "import", [importOptions(url, "full")]); + expect(imported.ok, JSON.stringify(imported.body)).toBe(true); + expect(imported.body).toMatchObject({ + name: "full", + defaultBranch: "main", + }); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "full")) + .body + ).toEqual({ bytes: [...Buffer.from("second\n")] }); + expect((await call(mf, "log", [], "full")).body).toMatchObject([ + { hash: commits.second }, + { hash: commits.first }, + ]); + expect( + ( + await call( + mf, + "readFile", + [{ ref: "feature", path: "feature.txt" }], + "full" + ) + ).body + ).toEqual({ bytes: [...Buffer.from("feature\n")] }); + const info = (await call(mf, "info", [], "full")).body as { + remote: string; + token?: string; + }; + const token = (imported.body as { token: string }).token; + await exec( + "git", + [ + "-c", + `http.extraHeader=Authorization: Bearer ${token}`, + "clone", + info.remote, + path.join(directory, "clone"), + ], + { env: gitEnvironment() } + ); + expect( + await readFile(path.join(directory, "clone", "README"), "utf8") + ).toBe("second\n"); + }); +}); + +test("HTTPS import: depth and branch constrain the imported history and refs", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, commits }) => { + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const result = await call(mf, "import", [ + importOptions(url, "shallow", "feature", 1), + ]); + expect(result.ok, JSON.stringify(result.body)).toBe(true); + expect(result.body).toMatchObject({ defaultBranch: "feature" }); + expect((await call(mf, "log", [], "shallow")).body).toMatchObject([ + { hash: commits.feature }, + ]); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "shallow")) + .body + ).toBeNull(); + }); +}); + +test("HTTPS import: Basic URL credentials, missing repos/branches, duplicate targets and retries", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const invalid = fixture.url.replace( + "https://", + "https://reader:wrong-password@" + ); + for (const [url, name] of [ + [fixture.url, "no-auth"], + [invalid, "wrong-auth"], + ]) { + const failure = await call(mf, "import", [importOptions(url, name)]); + expect(failure.ok).toBe(false); + expect(failure.body).toMatchObject({ code: "REMOTE_AUTH_REQUIRED" }); + } + const missing = await call(mf, "import", [ + importOptions(valid.replace("source.git", "missing.git"), "absent"), + ]); + expect(missing.ok).toBe(false); + expect(missing.body).toMatchObject({ code: "NOT_FOUND" }); + const branch = await call(mf, "import", [ + importOptions(valid, "absent-branch", "nonexistent"), + ]); + expect(branch.ok).toBe(false); + expect(branch.body).toMatchObject({ code: "NOT_FOUND" }); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + expect( + (await call(mf, "import", [importOptions(valid, "absent")])).ok + ).toBe(true); + const duplicate = await call(mf, "import", [ + importOptions(valid, "ABSENT"), + ]); + expect(duplicate.ok).toBe(false); + expect(duplicate.body).toMatchObject({ code: "ALREADY_EXISTS" }); + }); +}); + +test("HTTPS import: untrusted TLS fails closed without a CA", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const failed = await call(mf, "import", [ + importOptions(valid, "tls-failed"), + ]); + expect(failed.ok).toBe(false); + expect(failed.body).toMatchObject({ code: "UPSTREAM_UNAVAILABLE" }); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + }, false); +}); + +test("HTTPS import: interrupted upload cleans partial target and retry succeeds", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + fixture.interruptUpload = true; + const failed = await call(mf, "import", [importOptions(valid, "retry")]); + expect(failed.ok).toBe(false); + expect(failed.body).toMatchObject({ code: "UPSTREAM_UNAVAILABLE" }); + expect(fixture.interrupted).toBeGreaterThan(0); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + fixture.interruptUpload = false; + expect((await call(mf, "import", [importOptions(valid, "retry")])).ok).toBe( + true + ); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "retry")) + .body + ).toEqual({ bytes: [...Buffer.from("second\n")] }); + }); +}); + +test("Git child only inherits explicit CA trust, never host TLS bypass or credentials", ({ + expect, +}) => { + const env = gitEnvironment({ + PATH: "/usr/bin", + GIT_SSL_CAINFO: "/tmp/explicit-ca.pem", + GIT_SSL_NO_VERIFY: "true", + GIT_ASKPASS: "/tmp/askpass", + GIT_CONFIG_PARAMETERS: "'http.extraHeader=Authorization: secret'", + }); + expect(env.GIT_SSL_CAINFO).toBe("/tmp/explicit-ca.pem"); + expect(env.GIT_SSL_NO_VERIFY).toBeUndefined(); + expect(env.GIT_ASKPASS).toBeUndefined(); + expect(env.GIT_CONFIG_PARAMETERS).toBeUndefined(); +}); From fd2ec4ac8002998300e2f587a6721c54c3d7c74f Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Wed, 7 Oct 2026 13:04:24 +0100 Subject: [PATCH 10/21] [miniflare] Test local Artifacts expiry and restart lifecycle --- .../test/plugins/artifacts/lifecycle.spec.ts | 488 ++++++++++++++++++ 1 file changed, 488 insertions(+) create mode 100644 packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts diff --git a/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts b/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts new file mode 100644 index 00000000000..edca7b0f101 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts @@ -0,0 +1,488 @@ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { transform } from "esbuild"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; +import type { MiniflareOptions } from "miniflare"; + +const namespace = "lifecycle"; +const workerSource = path.resolve( + __dirname, + "../../../src/workers/artifacts/binding.worker.ts" +); +const clientScript = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name === undefined ? env.REPOS : await env.REPOS.get(name); + return Response.json(await target[method](...args)); + } catch (error) { + return Response.json({ code: error.code, message: error.message }, { status: 400 }); + } + } +};`; + +function options(persistencePath?: string): MiniflareOptions { + return { + cf: false, + ...(persistencePath === undefined + ? {} + : { resourcePersistencePath: persistencePath }), + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts", namespace } }, + manifest: singleModuleManifest(clientScript), + }, + }, + ], + }; +} + +type RpcResult = Record; + +async function rpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + const response = await mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); + const result = (await response.json()) as RpcResult; + if (!response.ok) { + throw new Error(String(result.message)); + } + return result; +} + +// This variant runs the actual binding worker and its DO inside workerd. Only +// this test's source is rewritten: there is no clock endpoint in shipped code. +async function clockedWorker(): Promise { + let source = await readFile(workerSource, "utf8"); + const replacements = [ + ["const createdAt = new Date();", "const createdAt = new Date(testClock);"], + [ + "return Date.parse(token.expiresAt) > Date.now();", + "return Date.parse(token.expiresAt) > testClock;", + ], + [ + "\t// The resolved tail lets the first mutation run immediately.", + "\tsetTestClock(time: number): void { testClock = time; }\n\t// The resolved tail lets the first mutation run immediately.", + ], + ]; + for (const [original, replacement] of replacements) { + if (source.split(original).length !== 2) { + throw new Error(`Artifacts clock test source changed: ${original}`); + } + source = source.replace(original, replacement); + } + source = `let testClock = 0;\n${source}\n${clockedFetch}`; + return (await transform(source, { loader: "ts", format: "esm" })).code; +} + +const clockedFetch = ` +export default { + async fetch(request, env) { + const { action, name, method, args = [], time, token, scope } = await request.json(); + const namespace = env.localArtifactsNamespace; + const state = namespace.get(namespace.idFromName("lifecycle")); + if (action === "clock") { + await state.setTestClock(time); + return Response.json(true); + } + if (action === "git") { + return state.fetch(new Request( + "http://localhost/git/lifecycle/repo.git/info/refs?service=git-" + scope + "-pack", + { headers: { Authorization: "Bearer " + token } } + )); + } + try { + const target = name === undefined ? state : await state.get(name); + return Response.json(await target[method](...args)); + } catch (error) { + return Response.json({ code: error.code, message: error.message }, { status: 400 }); + } + } +};`; + +function clockOptions( + script: string, + persistencePath?: string +): MiniflareOptions { + return { + cf: false, + ...(persistencePath === undefined + ? {} + : { resourcePersistencePath: persistencePath }), + workers: [ + { + config: { + name: "clocked", + compatibilityDate: "2026-09-03", + manifest: singleModuleManifest(script), + env: { + config: { + type: "json", + value: { namespace, origin: "http://localhost" }, + }, + localArtifactsNamespace: { + type: "durable-object", + worker: "clocked", + exportName: "LocalArtifactsNamespaceObject", + }, + gitBackend: { type: "worker", worker: "backend" }, + }, + exports: { + LocalArtifactsNamespaceObject: { + type: "durable-object", + storage: "sqlite", + }, + }, + }, + }, + { + config: { + name: "backend", + compatibilityDate: "2026-09-03", + manifest: singleModuleManifest(`export default { + async fetch(request) { + if (new URL(request.url).pathname !== "/__local_artifacts__") + return new Response("Git advertisement", { status: 200 }); + const { action } = await request.json(); + if (action === "create" || action === "fork") + return Response.json({ defaultBranch: "main", refs: {} }); + return Response.json({ refs: {} }); + } + }`), + }, + }, + ], + }; +} + +async function clockRequest( + mf: Miniflare, + request: Record +): Promise { + return mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify(request), + }); +} + +async function setClock(mf: Miniflare, time: number): Promise { + const response = await clockRequest(mf, { action: "clock", time }); + await response.body?.cancel(); +} + +async function clockRpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + const response = await clockRequest(mf, { method, args, name }); + const result = (await response.json()) as RpcResult; + if (!response.ok) { + throw new Error(JSON.stringify(result)); + } + return result; +} + +async function gitStatus( + mf: Miniflare, + token: string, + scope: "upload" | "receive" +): Promise { + const response = await clockRequest(mf, { + action: "git", + token, + scope, + }); + await response.body?.cancel(); + return response.status; +} + +const start = Date.parse("2026-01-01T00:00:00.000Z"); + +test("artifacts: token expiry is exclusive at the exact workerd clock boundary and persists", async ({ + expect, +}) => { + const script = await clockedWorker(); + const root = await useTmp(); + const first = new Miniflare(clockOptions(script, root)); + useDispose(first); + await setClock(first, start); + await clockRpc(first, "create", ["repo"]); + const read = await clockRpc(first, "createToken", ["read", 60], "repo"); + const write = await clockRpc(first, "createToken", ["write", 60], "repo"); + const expiration = start + 60_000; + expect(read.expiresAt).toBe(new Date(expiration).toISOString()); + expect(String(read.plaintext)).toContain(`?expires=${expiration / 1000}`); + expect(write.expiresAt).toBe(read.expiresAt); + const beforeExpiry = await clockRpc(first, "listTokens", [], "repo"); + expect(beforeExpiry.tokens).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: read.id, + scope: "read", + state: "active", + expiresAt: read.expiresAt, + }), + expect.objectContaining({ + id: write.id, + scope: "write", + state: "active", + expiresAt: write.expiresAt, + }), + ]) + ); + expect(JSON.stringify(beforeExpiry)).not.toContain(String(read.plaintext)); + await setClock(first, expiration - 1); + expect((await clockRpc(first, "listTokens", [], "repo")).total).toBe(3); + expect(await gitStatus(first, String(read.plaintext), "upload")).toBe(200); + expect(await gitStatus(first, String(read.plaintext), "receive")).toBe(403); + expect(await gitStatus(first, String(write.plaintext), "receive")).toBe(200); + expect(await gitStatus(first, String(write.plaintext), "upload")).toBe(200); + + await setClock(first, expiration); + expect((await clockRpc(first, "listTokens", [], "repo")).total).toBe(1); + expect(await gitStatus(first, String(read.plaintext), "upload")).toBe(401); + expect(await gitStatus(first, String(write.plaintext), "receive")).toBe(401); + expect(await gitStatus(first, String(read.plaintext), "receive")).toBe(401); + await first.dispose(); + + const restarted = new Miniflare(clockOptions(script, root)); + useDispose(restarted); + await setClock(restarted, expiration + 1); + expect((await clockRpc(restarted, "listTokens", [], "repo")).total).toBe(1); + expect(await gitStatus(restarted, String(write.plaintext), "upload")).toBe( + 401 + ); + expect(await gitStatus(restarted, String(read.plaintext), "upload")).toBe( + 401 + ); + const replacement = await clockRpc( + restarted, + "createToken", + ["read", 60], + "repo" + ); + expect( + await gitStatus(restarted, String(replacement.plaintext), "upload") + ).toBe(200); + expect( + await clockRpc(restarted, "revokeToken", [replacement.id], "repo") + ).toBe(true); + expect( + await gitStatus(restarted, String(replacement.plaintext), "upload") + ).toBe(401); +}); + +test("artifacts: concurrent same-name creates and forks commit one winner each", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["source"]); + const creates = await Promise.allSettled([ + rpc(mf, "create", ["Target"]), + rpc(mf, "create", ["target"]), + ]); + expect(creates.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + const forks = await Promise.allSettled([ + rpc(mf, "fork", ["Child"], "source"), + rpc(mf, "fork", ["child"], "source"), + ]); + expect(forks.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + expect((await rpc(mf, "list")).total).toBe(3); + const child = await rpc(mf, "info", [], "child"); + expect(child.source).toBe(`artifacts:${namespace}/source`); +}); + +test("artifacts: import and fork metadata remain distinct across concurrent creations", async ({ + expect, +}) => { + // The backend is a local stub: this exercises namespace state transitions, + // not network cloning (which is covered by the native Git tests). + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + const results = await Promise.allSettled([ + clockRpc(mf, "import", [ + { + source: { + url: "https://example.com/source.git", + branch: "release", + depth: 1, + }, + target: { name: "Imported", opts: { description: "remote copy" } }, + }, + ]), + clockRpc(mf, "import", [ + { + source: { url: "https://example.com/other.git" }, + target: { name: "imported" }, + }, + ]), + ]); + expect(results.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + const imported = await clockRpc(mf, "info", [], "imported"); + expect(imported.source).toMatch( + /^git:https:\/\/example\.com\/(source|other)\.git$/ + ); + const fork = await clockRpc( + mf, + "fork", + ["forked", { readOnly: true }], + "imported" + ); + expect(fork.name).toBe("forked"); + expect(await clockRpc(mf, "info", [], "forked")).toMatchObject({ + readOnly: true, + source: `artifacts:${namespace}/${imported.name}`, + }); + expect((await clockRpc(mf, "list")).total).toBe(2); +}); + +test("artifacts: concurrent independent mutations preserve all repositories and tokens", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await Promise.all( + ["one", "two", "three"].map((name) => rpc(mf, "create", [name])) + ); + await Promise.all( + ["one", "two", "three"].map(async (name) => { + await Promise.all([ + rpc(mf, "createToken", ["read", 60], name), + rpc(mf, "fork", [`${name}-fork`], name), + ]); + }) + ); + for (const name of ["one", "two", "three"]) { + expect((await rpc(mf, "listTokens", [], name)).total).toBe(2); + expect((await rpc(mf, "info", [], `${name}-fork`)).source).toBe( + `artifacts:${namespace}/${name}` + ); + } + expect((await rpc(mf, "list")).total).toBe(6); +}); + +async function killChild(child: ReturnType): Promise { + const closed = child.exitCode === null ? once(child, "close") : undefined; + if (process.platform !== "win32" && child.pid !== undefined) { + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") { + throw error; + } + } + } else if (child.exitCode === null) { + child.kill("SIGKILL"); + } + await closed; +} + +// The child deliberately never calls dispose(): the parent kills it after +// create() has committed, then starts a new listener against the same root. +test("artifacts: abrupt child-process exit leaves committed repositories restartable", async ({ + expect, +}) => { + const root = await useTmp(); + const marker = path.join(root, "ready"); + const childScript = path.join(root, "child.cjs"); + const modulePath = path.resolve(__dirname, "../../../dist/src/index.js"); + await writeFile( + childScript, + ` +const { Miniflare } = require(${JSON.stringify(modulePath)}); +const { writeFileSync } = require("node:fs"); +const mf = new Miniflare({ + cf: false, + resourcePersistencePath: ${JSON.stringify(root)}, + workers: [{ config: { + name: "", compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts", namespace: "lifecycle" } }, + manifest: { mainModule: "index.mjs", modulesRoot: ${JSON.stringify(root)}, modules: { "index.mjs": { type: "esm", contents: "export default { fetch() { return new Response('ok') } }" } } } + }}] +}); +(async () => { + const { REPOS } = await mf.getBindings(); + const created = await REPOS.create("survivor"); + writeFileSync(${JSON.stringify(marker)}, JSON.stringify(created)); + process.stdout.write("READY\\n"); +})().catch(error => { process.stderr.write(String(error)); process.exitCode = 1; }); +` + ); + const child = spawn(process.execPath, [childScript], { + stdio: ["ignore", "pipe", "pipe"], + // workerd is a subprocess: kill the process group to avoid orphaning it. + detached: process.platform !== "win32", + }); + let listener = ""; + let token = ""; + let stderr = ""; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + try { + if (child.stdout === null) { + throw new Error("Child stdout pipe unavailable"); + } + await Promise.race([ + once(child.stdout, "data", { signal: AbortSignal.timeout(15_000) }), + once(child, "exit").then(() => { + throw new Error(`Child exited before ready: ${stderr}`); + }), + ]); + const created = JSON.parse(await readFile(marker, "utf8")) as { + remote: string; + token: string; + }; + expect(created.remote).toMatch(/^http:\/\/127\.0\.0\.1:/); + listener = `${created.remote}/info/refs?service=git-upload-pack`; + token = created.token; + } finally { + await killChild(child); + } + if (process.platform !== "win32") { + await expect( + fetch(listener, { signal: AbortSignal.timeout(2_000) }) + ).rejects.toThrow(); + } + const restarted = new Miniflare(options(root)); + useDispose(restarted); + const info = await rpc(restarted, "info", [], "survivor"); + expect(info.name).toBe("survivor"); + const advertised = await fetch( + `${info.remote}/info/refs?service=git-upload-pack`, + { + headers: { Authorization: `Bearer ${token}` }, + } + ); + expect(advertised.status).toBe(200); + await advertised.body?.cancel(); + const created = await rpc(restarted, "create", ["after-restart"]); + expect(created.name).toBe("after-restart"); + expect((await rpc(restarted, "list")).total).toBe(2); +}); From 5b2221223a1a421f746147f7c215d8675e10468f Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Wed, 7 Oct 2026 13:04:34 +0100 Subject: [PATCH 11/21] [miniflare] Add local Artifacts contract comparison harness --- .../test/plugins/artifacts/contract.spec.ts | 263 ++++++++++++++ .../test/plugins/artifacts/contract/README.md | 23 ++ .../plugins/artifacts/contract/compare.ts | 141 ++++++++ .../test/plugins/artifacts/contract/live.ts | 49 +++ .../plugins/artifacts/contract/scenarios.ts | 331 ++++++++++++++++++ 5 files changed, 807 insertions(+) create mode 100644 packages/miniflare/test/plugins/artifacts/contract.spec.ts create mode 100644 packages/miniflare/test/plugins/artifacts/contract/README.md create mode 100644 packages/miniflare/test/plugins/artifacts/contract/compare.ts create mode 100644 packages/miniflare/test/plugins/artifacts/contract/live.ts create mode 100644 packages/miniflare/test/plugins/artifacts/contract/scenarios.ts diff --git a/packages/miniflare/test/plugins/artifacts/contract.spec.ts b/packages/miniflare/test/plugins/artifacts/contract.spec.ts new file mode 100644 index 00000000000..35891f176b1 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract.spec.ts @@ -0,0 +1,263 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { singleModuleManifest, useDispose } from "../../test-shared"; +import { compare, normalize } from "./contract/compare"; +import { runApprovedLiveComparison } from "./contract/live"; +import { runScenarios } from "./contract/scenarios"; +import type { + Created, + ArtifactsBinding, + Observation, +} from "./contract/scenarios"; + +const exec = promisify(execFile); +const SCRIPT = `export default { fetch() { return new Response("contract"); } };`; + +async function prepareLocalFixture(created: Created): Promise { + const source = await mkdtemp(path.join(tmpdir(), "artifacts-contract-")); + const environment = { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + GIT_AUTHOR_DATE: "2020-01-02T03:04:05Z", + GIT_COMMITTER_DATE: "2020-01-02T03:04:05Z", + }; + const git = async (...args: string[]) => + exec("git", args, { cwd: source, env: environment }); + try { + await git("init", "--initial-branch=main"); + await mkdir(path.join(source, "folder")); + await writeFile(path.join(source, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(source, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await writeFile(path.join(source, "folder", "nested.txt"), "nested\n"); + await git("add", "."); + await git("commit", "-m", "first"); + await writeFile(path.join(source, "second.txt"), "second\n"); + await git("add", "."); + await git("commit", "-m", "second"); + // Keep the token out of argv, stderr, and test output. The child alone sees the header. + const pushEnv = { + ...environment, + GIT_CONFIG_COUNT: "2", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `Authorization: Bearer ${created.token}`, + }; + await exec("git", ["push", created.remote, "main"], { + cwd: source, + env: pushEnv, + }); + await git("checkout", "-b", "feature"); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git("add", "."); + await git("commit", "-m", "feature"); + await exec("git", ["push", created.remote, "feature"], { + cwd: source, + env: pushEnv, + }); + } finally { + await removeDir(source); + } +} + +function local(): Miniflare { + return new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts", namespace: "contract-local" } }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); +} + +test("shared Artifacts contract runs against the local RPC binding and cleans up", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const observations = await runScenarios( + REPOS, + "contract-test-", + prepareLocalFixture + ); + const find = (step: string) => + observations.find((item) => item.step === step); + expect(observations.length).toBeGreaterThan(50); + expect(find("file binary")).toMatchObject({ + outcome: "value", + value: { + blob: { type: "application/octet-stream", bytes: [0, 255, 128, 1] }, + }, + }); + expect(find("file text")).toMatchObject({ + outcome: "value", + value: { blob: { type: "text/plain;charset=utf-8" } }, + }); + expect(find("duplicate case")).toMatchObject({ outcome: "error" }); + expect(find("retained after delete")).toMatchObject({ outcome: "error" }); + for (const kind of ["blob", "tree", "commit"]) { + expect(find(`${kind} present`)).toMatchObject({ outcome: "value" }); + expect(find(`${kind} missing`)).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find(`${kind} wrong type`)).toMatchObject({ + outcome: "value", + value: null, + }); + } + expect(find("file missing ref")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("file missing path")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("list next page")).toMatchObject({ outcome: "value" }); + expect(find("token revoke wrong repo")).toMatchObject({ + outcome: "value", + value: false, + }); + expect(find("fork all feature")).toMatchObject({ + outcome: "value", + value: { blob: { bytes: [...new TextEncoder().encode("feature\n")] } }, + }); + expect(find("fork main feature")).toMatchObject({ + outcome: "value", + value: null, + }); + expect((await REPOS.list()).total).toBe(0); +}); + +test("comparator preserves exact errors, Blob data, paths and non-loopback URLs", ({ + expect, +}) => { + const base: Observation[] = [ + { + step: "file", + outcome: "value", + value: { blob: { bytes: [0, 255], type: "text/plain" } }, + }, + { + step: "error", + outcome: "error", + value: { + name: "ArtifactsError", + code: "NOT_FOUND", + numericCode: 10200, + message: "not found", + }, + }, + { + step: "repo", + outcome: "value", + value: { + id: "first", + createdAt: "2020-01-01T00:00:00Z", + remote: "http://127.0.0.1:1234/git/repo.git", + name: "repo", + }, + }, + ]; + const variable = structuredClone(base); + (variable[2].value as Record).id = "second"; + (variable[2].value as Record).createdAt = + "2026-01-01T00:00:00Z"; + (variable[2].value as Record).remote = + "http://127.0.0.1:5678/git/repo.git"; + expect(compare(base, variable)).toEqual([]); + const different = structuredClone(variable); + (different[0].value as { blob: { bytes: number[] } }).blob.bytes[1] = 254; + (different[1].value as Record).numericCode = 10201; + (different[2].value as Record).remote = + "https://example.test/git/repo.git"; + expect( + compare(base, different).map(({ step, path, kind }) => [step, path, kind]) + ).toEqual([ + ["file", "$.blob.bytes[1]", "value"], + ["error", "$.numericCode", "error"], + ["repo", "$.remote", "value"], + ]); + expect( + normalize([ + { + step: "err", + outcome: "error", + value: { message: "id first at 2020-01-01T00:00:00Z" }, + }, + ])[0].value + ).toEqual({ message: "id first at 2020-01-01T00:00:00Z" }); + expect(compare(base, base.slice(1))[0].kind).toBe("missing-step"); + expect(compare(base, [...base, base[0]])[0]).toMatchObject({ + kind: "missing-step", + path: "$[3]", + }); + const reordered = structuredClone(base); + reordered.reverse(); + expect(compare(base, reordered)[0].kind).toBe("missing-step"); + const changed = structuredClone(base); + (changed[0].value as { blob: { type: string } }).blob.type = + "application/octet-stream"; + expect(compare(base, changed)).toMatchObject([ + { path: "$.blob.type", kind: "value" }, + ]); +}); + +test("live comparison refuses unapproved or reused fixtures without calling either binding", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const options = { + namespace: "contract-local", + approvedDisposableNamespace: "not-approved", + fixturePrefix: "contract-test-", + approvedDisposableFixturePrefix: "contract-test-", + confirmation: "I_APPROVE_DISPOSABLE_ARTIFACTS_COMPARISON" as const, + local: REPOS, + live: REPOS, + prepareLocalFixture, + prepareLiveFixture: prepareLocalFixture, + }; + await expect(runApprovedLiveComparison(options)).rejects.toThrow(/approval/); + await expect( + runApprovedLiveComparison({ + ...options, + approvedDisposableNamespace: "contract-local", + }) + ).rejects.toThrow(/distinct/); + expect((await REPOS.list()).total).toBe(0); + await REPOS.create("contract-test-main"); + await expect( + runScenarios(REPOS, "contract-test-", prepareLocalFixture) + ).rejects.toThrow(/not empty/); + expect((await REPOS.list()).total).toBe(1); + await REPOS.delete("contract-test-main"); + await expect( + runApprovedLiveComparison({ + ...options, + approvedDisposableNamespace: "contract-local", + approvedDisposableFixturePrefix: "other-prefix-", + }) + ).rejects.toThrow(/approval/); +}); diff --git a/packages/miniflare/test/plugins/artifacts/contract/README.md b/packages/miniflare/test/plugins/artifacts/contract/README.md new file mode 100644 index 00000000000..54f4a81ecff --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/README.md @@ -0,0 +1,23 @@ +# Artifacts observable contract (draft) + +`../contract.spec.ts` runs `runScenarios()` against Miniflare's real RPC binding, with a deterministic, locally pushed Git fixture. The shared scenarios capture returned objects, errors (`name`, `message`, `code`, `numericCode`), Blob MIME types and bytes, and missing/null outcomes. They cover namespace create/get/list/delete (including case and pagination boundaries), retained handles after delete/recreate, tokens and wrong-repository revocation, Git objects/files/refs, and empty/branch-selective/read-only forks. Git HTTP authorization is covered separately in `../index.spec.ts`; this suite does not expose a public RPC HTTP endpoint. + +**Do not treat local observations as an approved remote specification.** The comparator reports field-level differences without updating either side. It normalizes only identity fields (`id`, `token`, `plaintext`), ISO timestamp fields, and the port of a loopback `remote` URL. In particular, error strings/codes, Blob bytes/types, Git hashes, cursor behavior, object fields, and non-loopback remote URLs remain exact. A non-loopback live remote compared to a loopback local remote will be reported, not silently accepted. Traces include credentials before comparison: do not print, persist, or upload raw observations. The comparison API returns normalized mismatches only. + +## Optional manual comparison (not run by CI) + +There is **no** standalone CLI, deploy script, automatic provisioning, authentication helper, or remote endpoint in this directory. After the namespace owner explicitly approves a **disposable** namespace and a unique fixture prefix, a trusted private test environment may obtain two real binding objects (local and live) and call `runApprovedLiveComparison` from `live.ts` in-process. Pass: + +- `namespace` and the identical `approvedDisposableNamespace` confirmed by the owner (the harness cannot verify binding-to-namespace mapping); +- `fixturePrefix` and the identical `approvedDisposableFixturePrefix` confirmed by the owner: a fresh lowercase prefix ending in `-` (for example, `contract-sample-`), reserved solely for this run. The preflight refuses a namespace containing _any_ repositories, because list counts and cursors are global; +- the literal `confirmation: "I_APPROVE_DISPOSABLE_ARTIFACTS_COMPARISON"`; +- `local`, `live`, and `prepareLocalFixture` / `prepareLiveFixture` callbacks. Each callback receives a fresh repo's `remote` and a short-lived write `token` and must push the same two main commits and one feature commit as the local fixture, using a protected credential channel (not argv, logs, or committed files). Supply bindings through an authorized Worker/test context, **not** an unrestricted public RPC fetch route. + +Never pass a production namespace, existing repository, shared prefix, or resource credential through a command line. The harness deletes only repository names whose create/fork RPC returned success in this run; it never deletes a namespace or provisions/deploys a Worker. A failed RPC may still have created a resource; inspect partial failures manually with owner approval. Concurrent writers to the same namespace are not safe: reserve the namespace for the duration of the run. Cleanup failures throw; inspect and remove only the named fixtures with owner approval. Do not run a live comparison without explicit authorization. The local spec is the only automatically executed path. + +## Unresolved contract decisions + +- Remote ownership/authorization, namespace-to-binding mapping, exclusive namespace reservation and disposal approval require an actual owner; none is implied by this test harness. +- Exact production behavior for missing/wrong-type Git objects, commit-hash refs, path normalization, malformed cursors, token visibility/revocation, and retained handles requires an approved remote run. Current local handle errors can differ between direct binding proxy and an in-Worker call; do not rewrite error observations to hide this. +- A returned production Git remote will not be normalized to the local loopback URL. Its difference needs an explicit owner decision, not a comparator exception. +- Local-only regression assertions are not evidence of live platform parity. No production calls have been made by this workstream. diff --git a/packages/miniflare/test/plugins/artifacts/contract/compare.ts b/packages/miniflare/test/plugins/artifacts/contract/compare.ts new file mode 100644 index 00000000000..71eabe1bf21 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/compare.ts @@ -0,0 +1,141 @@ +import type { Observation } from "./scenarios"; + +export interface Mismatch { + step: string; + path: string; + kind: "missing-step" | "outcome" | "error" | "shape" | "value"; + local: unknown; + live: unknown; +} + +/** Normalize only identity fields, timestamps and loopback origins; never object contents or error text. */ +export function normalize(trace: Observation[]): Observation[] { + const ids = new Map(); + function normalizeValue(value: unknown, key = ""): unknown { + if (Array.isArray(value)) { + return value.map((item) => normalizeValue(item)); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([field, item]) => [ + field, + normalizeValue(item, field), + ]) + ); + } + if (typeof value !== "string") { + return value; + } + if (["id", "token", "plaintext"].includes(key)) { + if (!ids.has(value)) { + ids.set(value, `<${key}:${ids.size + 1}>`); + } + return ids.get(value); + } + if ( + ["createdAt", "updatedAt", "lastPushAt", "expiresAt"].includes(key) && + /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z$/.test(value) + ) { + return "