From 317f09f23473b54bb3a334900167f7fee2b82a5a Mon Sep 17 00:00:00 2001 From: Leo Ostera Date: Fri, 9 Oct 2026 12:43:56 +0100 Subject: [PATCH 01/15] [miniflare] Add opt-in local Artifacts simulator --- .changeset/artifacts-opt-in-simulator.md | 5 + packages/miniflare/scripts/build.mjs | 8 +- packages/miniflare/src/index.ts | 23 +- .../miniflare/src/plugins/artifacts/README.md | 146 ++ .../src/plugins/artifacts/controller.ts | 150 ++ .../src/plugins/artifacts/git-sidecar.ts | 548 +++++++ .../miniflare/src/plugins/artifacts/index.ts | 192 ++- .../src/plugins/artifacts/storage.ts | 40 + .../miniflare/src/plugins/shared/index.ts | 5 + .../src/workers/artifacts/binding.worker.ts | 1255 +++++++++++++++++ .../miniflare/test/config/v4-convert.spec.ts | 21 + .../test/plugins/artifacts/contract.spec.ts | 269 ++++ .../test/plugins/artifacts/contract/README.md | 23 + .../plugins/artifacts/contract/compare.ts | 141 ++ .../test/plugins/artifacts/contract/live.ts | 49 + .../plugins/artifacts/contract/scenarios.ts | 331 +++++ .../plugins/artifacts/https-import.spec.ts | 524 +++++++ .../test/plugins/artifacts/index.spec.ts | 1047 ++++++++++++++ .../test/plugins/artifacts/lifecycle.spec.ts | 543 +++++++ .../test/plugins/artifacts/storage.spec.ts | 167 +++ 20 files changed, 5461 insertions(+), 26 deletions(-) create mode 100644 .changeset/artifacts-opt-in-simulator.md create mode 100644 packages/miniflare/src/plugins/artifacts/README.md create mode 100644 packages/miniflare/src/plugins/artifacts/controller.ts create mode 100644 packages/miniflare/src/plugins/artifacts/git-sidecar.ts create mode 100644 packages/miniflare/src/plugins/artifacts/storage.ts create mode 100644 packages/miniflare/src/workers/artifacts/binding.worker.ts create mode 100644 packages/miniflare/test/plugins/artifacts/contract.spec.ts create mode 100644 packages/miniflare/test/plugins/artifacts/contract/README.md create mode 100644 packages/miniflare/test/plugins/artifacts/contract/compare.ts create mode 100644 packages/miniflare/test/plugins/artifacts/contract/live.ts create mode 100644 packages/miniflare/test/plugins/artifacts/contract/scenarios.ts create mode 100644 packages/miniflare/test/plugins/artifacts/https-import.spec.ts create mode 100644 packages/miniflare/test/plugins/artifacts/index.spec.ts create mode 100644 packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts create mode 100644 packages/miniflare/test/plugins/artifacts/storage.spec.ts diff --git a/.changeset/artifacts-opt-in-simulator.md b/.changeset/artifacts-opt-in-simulator.md new file mode 100644 index 00000000000..047636b2940 --- /dev/null +++ b/.changeset/artifacts-opt-in-simulator.md @@ -0,0 +1,5 @@ +--- +"miniflare": minor +--- + +Add an opt-in local Artifacts binding simulator to Miniflare. Set `dev.remote: false` on an Artifacts binding to use persistent local repositories and authenticated Git smart HTTP backed by host Git 2.32 or newer. Omitted `dev.remote` retains the existing remote binding behavior. diff --git a/packages/miniflare/scripts/build.mjs b/packages/miniflare/scripts/build.mjs index 56ba5b35265..516f094f304 100644 --- a/packages/miniflare/scripts/build.mjs +++ b/packages/miniflare/scripts/build.mjs @@ -446,7 +446,13 @@ async function buildPackage() { logLevel: watch ? "info" : "warning", outdir: outPath, outbase: pkgRoot, - entryPoints: [indexPath, ...fixtureBuilds], + entryPoints: [ + indexPath, + // A separate thread keeps native Git available during synchronous Node + // binding-proxy calls, which block Miniflare's main event loop. + path.join(pkgRoot, "src/plugins/artifacts/git-sidecar.ts"), + ...fixtureBuilds, + ], }; if (watch) { diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 1284d267928..ef661dfc487 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -77,6 +77,7 @@ import { WORKER_BINDING_SERVICE_LOOPBACK, WORKFLOWS_PLUGIN_NAME, } from "./plugins"; +import { ArtifactsController } from "./plugins/artifacts/controller"; import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants"; import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version"; import { closeBrowserProcess } from "./plugins/browser-rendering/process"; @@ -888,6 +889,7 @@ export class Miniflare { #maybeInspectorProxyController?: InspectorProxyController; #previousRuntimeInspectorPort?: number; + #artifactsController = new ArtifactsController(); #containerPrivilegesCache = new ContainerPrivilegesCache(); #hyperdriveProxyController: HyperdriveProxyController = new HyperdriveProxyController(); @@ -2326,6 +2328,7 @@ export class Miniflare { queueConsumers, containerPrivilegesCache: this.#containerPrivilegesCache, hyperdriveProxyController: this.#hyperdriveProxyController, + artifactsController: this.#artifactsController, }; for (const [key, plugin] of this.#mergedPluginEntries) { const pluginServicesExtensions = await plugin.getServices({ @@ -2341,6 +2344,11 @@ export class Miniflare { } else { pluginServices = pluginServicesExtensions.services; extensions.push(...pluginServicesExtensions.extensions); + for (const socket of pluginServicesExtensions.sockets ?? []) { + if (!sockets.some((existing) => existing.name === socket.name)) { + sockets.push(socket); + } + } } for (const service of pluginServices) { @@ -2672,11 +2680,13 @@ export class Miniflare { async #assembleAndUpdateConfig(reusePorts = false) { this.#hyperdriveProxyController.beginUpdate(); + this.#artifactsController.beginUpdate(); try { await this.#assembleAndUpdateConfigInternal(reusePorts); } finally { - // If assembly or runtime startup failed, keep the old listeners and - // close only the new candidates. A successful update already committed. + // A failed update retains listeners used by the previous runtime and + // closes only candidates started during this attempt. + await this.#artifactsController.abortUpdate(); this.#hyperdriveProxyController.abortUpdate(); } } @@ -2800,6 +2810,7 @@ export class Miniflare { } } this.#hyperdriveProxyController.commitUpdate(activeExternalAddresses); + await this.#artifactsController.commitUpdate(); // Note: `updateConfig()` doesn't resolve until ports for all required // sockets have been recorded. At this point, `maybeSocketPorts` contains // all of `requiredSockets` as keys. @@ -3915,6 +3926,14 @@ export class Miniflare { } const runtimeCleanupOutcome = await runtimeDisposeOutcome; + try { + await this.#artifactsController.dispose(); + } catch (error) { + if (!independentCleanupFailed) { + independentCleanupFailed = true; + independentCleanupError = error; + } + } this.#devRegistry.unregisterWorkers(); try { await Promise.all( diff --git a/packages/miniflare/src/plugins/artifacts/README.md b/packages/miniflare/src/plugins/artifacts/README.md new file mode 100644 index 00000000000..59fd851347f --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/README.md @@ -0,0 +1,146 @@ +# Artifacts in Miniflare + +The Artifacts plugin implements the local Artifacts binding in workerd. This +first version requires Git 2.32 or newer installed globally on the host and +available on PATH. Miniflare checks the Git version at local startup and gives +installation or upgrade instructions if needed. A Node worker thread runs the +native Git backend. Set `dev.remote` to `false` explicitly to select local +storage; an omitted setting retains remote binding behavior in this stage. +We plan to open-source a JavaScript Git engine in the future; it is not +included in this version. + +## Local startup + +```mermaid +sequenceDiagram + participant Dev as Developer tool + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Controller as ArtifactsController + participant Sidecar as Node Git sidecar + participant Workerd as workerd + + Dev->>MF: Configure Artifacts binding + MF->>Plugin: getBindings() + Plugin->>Plugin: Select local and validate namespace + Plugin-->>MF: Bind to local namespace entrypoint + MF->>Plugin: getServices() + Plugin->>Plugin: Verify host Git is available on PATH + Plugin->>Controller: get(Git storage path) + Controller->>Sidecar: Start worker thread and private HTTP listener + Sidecar-->>Controller: Address and secret + Controller-->>Plugin: Sidecar endpoint and Git socket port + Plugin-->>MF: Binding Worker, DO storage, backend service, Git socket + MF->>Workerd: Start configured services and socket +``` + +`getBindings()` routes each local binding to a namespace-scoped service. +`getServices()` creates the binding Worker, namespace Durable Object, persistent +metadata disk, internal Git backend service, and Git socket. The controller +reuses a sidecar per Git storage path and closes it when Miniflare is disposed. +The socket used by Git clients and the sidecar's private HTTP listener are +**different endpoints**. + +## Local binding RPC + +For example, `env.REPOS.create("demo")`: + +```mermaid +sequenceDiagram + participant App as User Worker + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Disk as Local disk + + App->>Entry: env.REPOS.create("demo") + Entry->>DO: RPC create("demo") + DO->>DO: Validate and serialize mutation + DO->>Backend: POST internal create request + Backend->>Sidecar: Inject private backend header + Sidecar->>Disk: git init --bare + Sidecar-->>DO: Git state (via backend service) + DO->>Disk: Persist metadata and token hash + DO-->>Entry: Repository info and initial token + Entry-->>App: Create result +``` + +The Durable Object stores repository metadata and token hashes on local disk; +the sidecar stores the bare Git repositories separately. `git-sidecar.ts` +handles HTTP routing and repository locks, while `git-client.ts` owns +repository-scoped Git commands and result parsing. Read operations such as +`readFile()` also use the binding Worker and native Git backend. Node callers +reach the same binding through Miniflare's Node binding proxy. Namespace and +repository names are hashed into short, namespace-scoped on-disk paths; this +leaves room for Git packfiles and workerd SQLite files on platforms with path +length limits. Older draft storage layouts are not migrated automatically: +startup rejects a detected old layout rather than silently creating empty +repositories. Back up any local repositories before resetting the Artifacts +persistence directory. Do not delete the persistence directory without first +checking whether it holds work you need. + +## Git clone and push + +```mermaid +sequenceDiagram + participant Git as Git client + participant Socket as Git socket + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Native as Native Git + + Git->>Socket: Git smart HTTP at /git/:namespace/:repo.git + Socket->>Entry: Route request + Entry->>DO: fetch(request) + DO->>DO: Check endpoint, repository, token, and scope + DO->>Backend: Forward request without Authorization header + Backend->>Sidecar: Inject private backend header + Sidecar->>Native: Stream via git http-backend + Native-->>Sidecar: Discovery or pack response + Sidecar-->>DO: Git response (via backend service) + opt Successful receive-pack push + DO->>Backend: Read updated refs + Backend->>Sidecar: Internal refs request + Sidecar-->>DO: Refs (via backend service) + DO->>DO: Persist updated push metadata + end + DO-->>Entry: Git response + Entry-->>Socket: Git response + Socket-->>Git: Git response +``` + +The binding Worker checks the supported Git endpoints and authenticates +repository tokens before forwarding. The internal service injects the sidecar +secret; the client's `Authorization` header is removed before that hop. For a +successful push, the Durable Object refreshes repository metadata before +returning the response. The sidecar uses native `git http-backend` for Git smart +HTTP rather than reproducing Git's pack protocol. + +## Explicit remote binding + +When `dev.remote` is `true`, this binding uses the remote-bindings proxy instead +of creating local Artifacts storage or a local Git sidecar for that namespace. + +```mermaid +sequenceDiagram + participant App as User Worker + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Proxy as Remote-bindings proxy + participant CF as Cloudflare Artifacts + + MF->>Plugin: getBindings() with remote: true + Plugin-->>MF: Bind to remote proxy service + MF->>Proxy: Start remote proxy Worker + App->>Proxy: env.REPOS method call + Proxy->>CF: Remote binding request + CF-->>Proxy: Result + Proxy-->>App: Result +``` + +A local binding does not automatically synchronize with production Artifacts. +An explicit repository import may contact its specified HTTPS Git remote; that +is distinct from enabling a remote Artifacts binding. diff --git a/packages/miniflare/src/plugins/artifacts/controller.ts b/packages/miniflare/src/plugins/artifacts/controller.ts new file mode 100644 index 00000000000..1a69edefc9e --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/controller.ts @@ -0,0 +1,150 @@ +import { once } from "node:events"; +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import getPort from "get-port"; +import type { GitSidecar } from "./git-sidecar"; + +export interface LocalArtifactsBackend { + sidecar: GitSidecar; + port: number; +} + +// Node binding proxies make synchronous calls. The Git backend must run on +// another thread, otherwise those calls prevent it from answering workerd. +export class ArtifactsController { + #backends = new Map>(); + // Keep the old roots alive until workerd accepts the replacement config. + #activeRoots = new Set(); + #pendingRoots?: Set; + + beginUpdate(): void { + if (this.#pendingRoots) { + throw new Error("Artifacts runtime update already in progress"); + } + this.#pendingRoots = new Set(); + } + + async commitUpdate(): Promise { + const roots = this.#pendingRoots; + if (!roots) { + throw new Error("No Artifacts runtime update to commit"); + } + this.#pendingRoots = undefined; + this.#activeRoots = roots; + await this.#closeExcept(roots); + } + + async abortUpdate(): Promise { + if (!this.#pendingRoots) { + return; + } + this.#pendingRoots = undefined; + await this.#closeExcept(this.#activeRoots); + } + + get(root: string): Promise { + if (!this.#pendingRoots) { + throw new Error("Artifacts backend requested outside runtime update"); + } + this.#pendingRoots.add(root); + let backend = this.#backends.get(root); + if (!backend) { + backend = this.#start(root); + this.#backends.set(root, backend); + void backend.catch(() => { + if (this.#backends.get(root) === backend) { + this.#backends.delete(root); + } + }); + } + return backend; + } + + async #start(root: string): Promise { + const port = await getPort({ host: "127.0.0.1" }); + // __dirname is dist/src in the bundled Miniflare entry point. + const worker = new Worker( + path.join(__dirname, "plugins/artifacts/git-sidecar.js"), + { workerData: { root } } + ); + try { + const endpoint = await waitForSidecarReady(worker); + worker.on("error", (error) => { + process.emitWarning( + `Local Artifacts Git backend failed: ${error.message}` + ); + }); + return { + port, + sidecar: { + ...endpoint, + async close() { + if (worker.threadId === -1) { + return; + } + const exited = once(worker, "exit"); + worker.postMessage("close"); + await exited; + }, + }, + }; + } catch (error) { + await worker.terminate(); + throw error; + } + } + + async #closeExcept(roots: Set): Promise { + const retired = [...this.#backends].filter(([root]) => !roots.has(root)); + for (const [root] of retired) { + this.#backends.delete(root); + } + const results = await Promise.allSettled( + retired.map(([, backend]) => + backend.then( + ({ sidecar }) => sidecar.close(), + () => undefined + ) + ) + ); + for (const result of results) { + if (result.status === "rejected") { + process.emitWarning("Unable to stop a retired Artifacts Git backend"); + } + } + } + + async dispose(): Promise { + this.#pendingRoots = undefined; + this.#activeRoots.clear(); + await this.#closeExcept(this.#activeRoots); + } +} + +function waitForSidecarReady( + worker: Worker +): Promise> { + return new Promise((resolve, reject) => { + function cleanup() { + worker.off("message", ready); + worker.off("error", failed); + worker.off("exit", exited); + } + function ready(value: Pick) { + cleanup(); + resolve(value); + } + function failed(error: Error) { + cleanup(); + reject(error); + } + function exited(code: number) { + failed( + new Error(`Local Artifacts Git backend exited during startup (${code})`) + ); + } + worker.on("message", ready); + worker.on("error", failed); + worker.on("exit", exited); + }); +} diff --git a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts new file mode 100644 index 00000000000..3e003b66789 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts @@ -0,0 +1,548 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { mkdir, readdir, rename, stat } from "node:fs/promises"; +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import path from "node:path"; +import { parentPort, workerData } from "node:worker_threads"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { assertGitAvailable, GitClient, gitEnvironment } from "./git-client"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "./storage"; +import type { RepositoryState } from "./git-client"; + +const adminPath = "/__local_artifacts__"; +const componentPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +type RepositoryLocks = Map>; +const decoder = new TextDecoder(); + +if (parentPort) { + const port = parentPort; + void startGitSidecar(workerData.root).then((sidecar) => { + port.postMessage({ address: sidecar.address, secret: sidecar.secret }); + port.once("message", async () => { + await sidecar.close(); + port.close(); + }); + }); +} + +export interface GitSidecar { + address: string; + secret: string; + close(): Promise; +} + +interface AdminRequest { + action: + | "create" + | "delete" + | "fork" + | "import" + | "refs" + | "readBlob" + | "readTree" + | "readCommit" + | "file" + | "log"; + namespace: string; + name: string; + defaultBranch?: string; + defaultBranchOnly?: boolean; + sourceName?: string; + sourceBranch?: string; + sourceUrl?: string; + branch?: string; + depth?: number; + hash?: string; + ref?: string; + path?: string; + limit?: number; + offset?: number; + generation?: string; +} + +function createGitSecret(): string { + return randomBytes(32).toString("hex"); +} + +export async function startGitSidecar(root: string): Promise { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + await assertGitAvailable(); + await cleanupRetiredRepositories(root); + const secret = createGitSecret(); + const repositoryLocks: RepositoryLocks = new Map(); + const server = createServer((request, response) => { + if (request.headers["x-local-artifacts-backend"] !== secret) { + response.writeHead(403).end("Forbidden"); + return; + } + void handleRequest(root, repositoryLocks, request, response).catch( + (error: unknown) => { + if (response.headersSent) { + response.destroy(error instanceof Error ? error : undefined); + } else { + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end( + error instanceof Error ? error.message : "Native Git backend failed" + ); + } + } + ); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + // The listener outlives this function; ArtifactsController closes it on disposal. + // `using` here would close it before workerd could make any requests. + server.unref(); + const addressInfo = server.address(); + if (!addressInfo || typeof addressInfo === "string") { + throw new Error("Unable to start native Git backend"); + } + const address = `127.0.0.1:${addressInfo.port}`; + return { + address, + secret, + close() { + server.closeAllConnections(); + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +function parseGitPath(pathname: string): { + namespace: string; + repository: string; + suffix: string; +} | null { + const match = pathname.match(/^\/git\/([^/]+)\/([^/]+\.git)(\/.*)?$/); + if (!match?.[1] || !match[2]) { + return null; + } + const namespace = decodeURIComponent(match[1]); + const repository = decodeURIComponent(match[2].slice(0, -4)); + validateComponent(namespace, "namespace"); + validateComponent(repository, "repository"); + return { namespace, repository, suffix: match[3] ?? "" }; +} + +// Native Git's CGI backend needs streaming Node requests and responses; keep +// packfiles on node:http rather than buffering them through a Worker router. +async function handleRequest( + root: string, + locks: RepositoryLocks, + request: IncomingMessage, + response: ServerResponse +): Promise { + const url = new URL(request.url ?? "/", "http://local-artifacts.invalid"); + if (url.pathname === adminPath) { + await handleAdmin(root, locks, request, response); + return; + } + const gitPath = parseGitPath(url.pathname); + if (!gitPath) { + response.writeHead(404).end("Not found"); + return; + } + const { namespace, repository, suffix } = gitPath; + const contentLength = Number(request.headers["content-length"] ?? 0); + if (!Number.isFinite(contentLength) || contentLength > 256 * 1024 * 1024) { + response.writeHead(413).end("Request body too large"); + return; + } + const path = repositoryPath(root, namespace, repository); + await withRepositoryLocks(locks, [path], async () => { + const generation = request.headers["x-local-artifacts-generation"]; + const configured = await new GitClient(path).generation(); + if (typeof generation !== "string" || generation !== configured) { + response.writeHead(409).end("Repository generation changed"); + return; + } + await runGitHttpBackend( + root, + request, + response, + `/${repositoryDirectory(namespace, repository)}${suffix}`, + url.search.slice(1) + ); + }); +} + +async function handleAdmin( + root: string, + locks: RepositoryLocks, + request: IncomingMessage, + response: ServerResponse +): Promise { + if (request.method !== "POST") { + response.writeHead(405).end("Method not allowed"); + return; + } + const body = JSON.parse( + decoder.decode(await readRequest(request)) + ) as AdminRequest; + validateComponent(body.namespace, "namespace"); + validateComponent(body.name, "repository"); + const repository = repositoryPath(root, body.namespace, body.name); + const lockPaths = [repository]; + if (body.action === "fork" && body.sourceName) { + validateComponent(body.sourceName, "repository"); + lockPaths.push(repositoryPath(root, body.namespace, body.sourceName)); + } + await withRepositoryLocks(locks, lockPaths, async () => { + const result = await handleAction(root, body, new GitClient(repository)); + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(result)); + }); +} + +async function handleAction( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + switch (body.action) { + case "create": + return handleCreate(root, body, git); + case "delete": + await retireRepository(git.repository); + return true; + case "fork": + return handleFork(root, body, git); + case "import": + return handleImport(root, body, git); + case "refs": + return git.state(); + case "readBlob": + return git.readBlob(required(body.hash, "hash")); + case "readTree": + return git.readTree(required(body.hash, "hash")); + case "readCommit": + return git.readCommit(required(body.hash, "hash")); + case "file": + return git.readFile( + required(body.ref, "ref"), + required(body.path, "path") + ); + case "log": + return git.log(body.ref, body.limit, body.offset); + } +} + +function handleCreate( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + return withNewRepository(root, body, git, () => + git.init(body.defaultBranch ?? "main") + ); +} + +function handleFork( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + const { sourceName, sourceBranch } = body; + if (!sourceName) { + throw new Error("Fork source is required"); + } + if (!sourceBranch) { + throw new Error("Fork source branch is required"); + } + const source = new GitClient( + repositoryPath(root, body.namespace, sourceName) + ); + return withNewRepository(root, body, git, () => + git.forkFrom(source, sourceBranch, body.defaultBranchOnly ?? true) + ); +} + +function handleImport( + root: string, + body: AdminRequest, + git: GitClient +): Promise { + const { sourceUrl } = body; + if (!sourceUrl) { + throw new Error("Import source is required"); + } + return withNewRepository(root, body, git, () => + git.importFrom(sourceUrl, body.branch, body.depth) + ); +} + +async function withNewRepository( + root: string, + body: AdminRequest, + git: GitClient, + initialize: () => Promise +): Promise { + await mkdir(root, { recursive: true }); + if (await exists(git.repository)) { + // The namespace has no record for this name. A prior deletion may have + // committed metadata before the process stopped removing Git data. + // Never remove a directory that was not created by this simulator. + if ((await git.generation()) === null) { + throw new Error(`Repository "${body.name}" already exists on disk`); + } + await retireRepository(git.repository); + } + try { + await initialize(); + await git.configure(required(body.generation, "generation")); + return await git.state(); + } catch (error) { + await removeDir(git.repository); + throw error; + } +} + +// Moving the directory out of its active path is atomic on the same volume. +// A crash while removing its contents leaves only an inaccessible trash path; +// the next sidecar startup retries cleanup without blocking name reuse. +async function retireRepository(repository: string): Promise { + const retired = `${repository}.deleting-${randomBytes(16).toString("hex")}`; + try { + await rename(repository, retired); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + await removeDir(retired); +} + +async function cleanupRetiredRepositories(root: string): Promise { + for (const entry of await readdir(root, { withFileTypes: true })) { + if ( + entry.isDirectory() && + /^[0-9a-f]{32}\.git\.deleting-[0-9a-f]{32}$/.test(entry.name) + ) { + await removeDir(path.join(root, entry.name)); + } + } +} + +function required(value: string | undefined, name: string): string { + if (!value) { + throw new Error(`${name} is required`); + } + return value; +} + +function parseGitBackendHeaders(headerBytes: Buffer): { + status: number; + headers: Record; +} { + const lines = decoder.decode(headerBytes).split(/\r?\n/); + let status = 200; + const headers: Record = {}; + for (const line of lines) { + const separator = line.indexOf(":"); + if (separator < 0) { + continue; + } + const name = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + if (name.toLowerCase() === "status") { + status = Number(value.slice(0, 3)); + } else { + headers[name] = value; + } + } + return { status, headers }; +} + +async function runGitHttpBackend( + root: string, + request: IncomingMessage, + response: ServerResponse, + pathInfo: string, + query: string +): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", ["http-backend"], { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: root, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: pathInfo, + QUERY_STRING: query, + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + REMOTE_USER: "local-artifacts", + REMOTE_ADDR: request.socket.remoteAddress ?? "127.0.0.1", + }, + stdio: ["pipe", "pipe", "pipe"], + }); + let header = Buffer.alloc(0); + let headersSent = false; + let settled = false; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill(); + }, 300_000); + timeout.unref(); + const stderr: Buffer[] = []; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + request.off("aborted", abort); + response.off("close", abort); + if (error) { + reject(error); + } else { + resolve(); + } + }; + const abort = () => { + if (!response.writableEnded) { + child.kill(); + } + }; + const write = (chunk: Buffer) => { + if (!response.write(chunk)) { + child.stdout.pause(); + response.once("drain", () => child.stdout.resume()); + } + }; + request.once("aborted", abort); + response.once("close", abort); + child.stdin.on("error", (error) => { + if (!request.destroyed) { + finish(error); + } + }); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.stdout.on("data", (chunk: Buffer) => { + if (headersSent) { + write(chunk); + return; + } + header = Buffer.concat([header, chunk]); + const crlf = header.indexOf("\r\n\r\n"); + const lf = header.indexOf("\n\n"); + const index = crlf >= 0 ? crlf : lf; + if (index < 0) { + if (header.length > 64 * 1024) { + child.kill(); + } + return; + } + const separatorLength = crlf >= 0 ? 4 : 2; + const { status, headers } = parseGitBackendHeaders( + header.subarray(0, index) + ); + response.writeHead(status, headers); + headersSent = true; + const remaining = header.subarray(index + separatorLength); + if (remaining.length) { + write(remaining); + } + header = Buffer.alloc(0); + }); + child.stdout.on("end", () => { + if (headersSent) { + response.end(); + } + }); + child.once("error", finish); + child.once("close", (status) => { + if (timedOut) { + finish(new Error("git http-backend timed out")); + return; + } + if (!headersSent) { + finish( + new Error( + decoder.decode(Buffer.concat(stderr)).trim() || + `git http-backend exited with status ${status}` + ) + ); + } else { + finish(); + } + }); + request.pipe(child.stdin); + }); +} + +function readRequest(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => resolve(Buffer.concat(chunks))); + request.once("error", reject); + }); +} + +function validateComponent(value: string, type: string): void { + if (!componentPattern.test(value)) { + throw new Error(`Invalid ${type}`); + } +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +} + +async function withRepositoryLocks( + repositoryLocks: RepositoryLocks, + paths: string[], + action: () => Promise +): Promise { + const keys = [...new Set(paths)].sort(); + const previous = keys.map( + (key) => repositoryLocks.get(key) ?? Promise.resolve() + ); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + for (const key of keys) { + repositoryLocks.set(key, current); + } + await Promise.all(previous); + try { + return await action(); + } finally { + release(); + for (const key of keys) { + if (repositoryLocks.get(key) === current) { + repositoryLocks.delete(key); + } + } + } +} diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 48242920859..e39ce376c81 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -1,31 +1,87 @@ +import { createHash } from "node:crypto"; +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import SCRIPT from "worker:artifacts/binding"; import { buildRemoteProxyProps, getEnvBindingsOfType, + getPersistPath, getRemoteProxyConnectionString, ProxyNodeBinding, remoteProxyClientWorker, } from "../shared"; -import type { Plugin } from "../shared"; +import { assertGitAvailable } from "./git-client"; +import { assertSupportedGitLayout } from "./storage"; +import type { Service, Socket } from "../../runtime"; +import type { MiniflareBinding, Plugin } from "../shared"; +import type { ArtifactsController } from "./controller"; + +export { ArtifactsController, type LocalArtifactsBackend } from "./controller"; +export type { GitSidecar } from "./git-sidecar"; export const ARTIFACTS_PLUGIN_NAME = "artifacts"; -// One shared remote-proxy service for every artifacts binding; per-binding -// config travels via props. const ARTIFACTS_REMOTE_SERVICE_NAME = `${ARTIFACTS_PLUGIN_NAME}:remote`; +const ENTRYPOINT = "LocalArtifactsNamespace"; +const OBJECT_CLASS = "LocalArtifactsNamespaceObject"; +const LOCAL_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +type ArtifactsBinding = Extract; +type ArtifactsEntry = [name: string, binding: ArtifactsBinding]; + +function isLocalArtifactsBinding(binding: ArtifactsBinding): boolean { + return binding.dev?.remote === false; +} + +function getLocalNamespaceNames(bindings: ArtifactsEntry[]): Set { + return new Set( + bindings + .filter(([, binding]) => isLocalArtifactsBinding(binding)) + .map(([, binding]) => binding.namespace) + ); +} + +function validateLocalNamespace(namespace: string): void { + if (!LOCAL_NAMESPACE_PATTERN.test(namespace)) { + throw new Error(`Invalid local Artifacts namespace: ${namespace}`); + } +} + +function namespaceId(namespace: string): string { + // 128 bits keeps storage paths short enough for workerd's SQLite files on Windows. + return createHash("sha256").update(namespace).digest("hex").slice(0, 32); +} + +function localServiceName(namespace: string): string { + return `${ARTIFACTS_PLUGIN_NAME}-${namespaceId(namespace)}`; +} export const ARTIFACTS_PLUGIN: Plugin = { bindingTypeDescription: "Artifacts", - async getBindings(options) { + getBindings(options) { return getEnvBindingsOfType(options.config, "artifacts").map( - ([name, binding]) => ({ - name, - service: { - name: ARTIFACTS_REMOTE_SERVICE_NAME, - props: buildRemoteProxyProps( - getRemoteProxyConnectionString(binding, options.dev), - name - ), - }, - }) + ([name, binding]) => { + // In this stage, only explicit remote: false opts into local storage. + if (isLocalArtifactsBinding(binding)) { + validateLocalNamespace(binding.namespace); + return { + name, + service: { + name: localServiceName(binding.namespace), + entrypoint: ENTRYPOINT, + }, + }; + } + return { + name, + service: { + name: ARTIFACTS_REMOTE_SERVICE_NAME, + props: buildRemoteProxyProps( + getRemoteProxyConnectionString(binding, options.dev), + name + ), + }, + }; + } ); }, getNodeBindings(options) { @@ -36,16 +92,106 @@ export const ARTIFACTS_PLUGIN: Plugin = { ]) ); }, - async getServices({ options }) { - if (getEnvBindingsOfType(options.config, "artifacts").length === 0) { - return []; - } - - return [ - { + async getServices({ options, sharedOptions, tmpPath, artifactsController }) { + const bindings = getEnvBindingsOfType(options.config, "artifacts"); + const services: Service[] = []; + const sockets: Socket[] = []; + if (bindings.some(([, binding]) => !isLocalArtifactsBinding(binding))) { + services.push({ name: ARTIFACTS_REMOTE_SERVICE_NAME, worker: remoteProxyClientWorker(), - }, - ]; + }); + } + const namespaces = getLocalNamespaceNames(bindings); + if (namespaces.size > 0) { + await assertGitAvailable(); + } + const root = getPersistPath( + ARTIFACTS_PLUGIN_NAME, + tmpPath, + sharedOptions.resourcePersistencePath + ); + + for (const namespace of namespaces) { + const local = await createLocalNamespaceRuntime( + root, + namespace, + artifactsController + ); + sockets.push(local.socket); + services.push(...local.services); + } + return { services, extensions: [], sockets }; }, }; + +async function createLocalNamespaceRuntime( + root: string, + namespace: string, + artifactsController: ArtifactsController +): Promise<{ services: Service[]; socket: Socket }> { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + const persistPath = path.join(root, namespaceId(namespace)); + const metadataPath = path.join(persistPath, "metadata").replaceAll("\\", "/"); + await mkdir(metadataPath, { recursive: true }); + await assertSupportedGitLayout(metadataPath); + const { sidecar, port } = await artifactsController.get( + path.join(persistPath, "git") + ); + const serviceName = localServiceName(namespace); + const storageName = `${serviceName}-storage`; + const backendName = `${serviceName}-backend`; + const socket: Socket = { + name: `${serviceName}-git`, + address: `127.0.0.1:${port}`, + http: {}, + service: { name: serviceName, entrypoint: ENTRYPOINT }, + }; + const storageService: Service = { + name: storageName, + disk: { path: metadataPath, writable: true }, + }; + const gitBackendService: Service = { + name: backendName, + external: { + address: sidecar.address, + http: { + injectRequestHeaders: [ + { name: "X-Local-Artifacts-Backend", value: sidecar.secret }, + ], + }, + }, + }; + const bindingService: Service = { + name: serviceName, + worker: { + compatibilityDate: "2026-09-03", + modules: [{ name: "binding.worker.js", esModule: SCRIPT() }], + bindings: [ + { + name: "config", + json: JSON.stringify({ + namespace, + origin: `http://127.0.0.1:${port}`, + }), + }, + { + name: "localArtifactsNamespace", + durableObjectNamespace: { className: OBJECT_CLASS }, + }, + { name: "gitBackend", service: { name: backendName } }, + ], + // Keys must be unique across namespaces in one workerd, but should + // not repeat the long service prefix in SQLite's directory name. + durableObjectNamespaces: [ + { className: OBJECT_CLASS, uniqueKey: namespaceId(namespace) }, + ], + durableObjectStorage: { localDisk: storageName }, + }, + }; + return { + services: [storageService, gitBackendService, bindingService], + socket, + }; +} diff --git a/packages/miniflare/src/plugins/artifacts/storage.ts b/packages/miniflare/src/plugins/artifacts/storage.ts new file mode 100644 index 00000000000..ac693dc1bcc --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/storage.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { readdir } from "node:fs/promises"; +import path from "node:path"; + +/** + * Keep repository paths short: native Git adds long pack and lock filenames. + * Hash the tuple rather than nesting namespace and repository hashes. Each + * identifier retains 128 bits and no user input becomes a path component. + */ +export function repositoryDirectory(namespace: string, name: string): string { + const id = createHash("sha256") + .update(JSON.stringify([namespace, name])) + .digest("hex") + .slice(0, 32); + return `${id}.git`; +} + +export function repositoryPath( + root: string, + namespace: string, + name: string +): string { + return path.join(root, repositoryDirectory(namespace, name)); +} + +/** Do not quietly abandon repositories written by earlier unreleased layouts. */ +export async function assertSupportedGitLayout(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + if ( + entries.some( + (entry) => + entry.isDirectory() && + (/^[0-9a-f]{64}$/.test(entry.name) || /^artifacts[-:]/.test(entry.name)) + ) + ) { + throw new Error( + "Local Artifacts storage uses an older draft layout. Back up your local repositories, then reset the Artifacts persistence directory before restarting." + ); + } +} diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 9c0658a0c3b..578f1a1deca 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -12,6 +12,7 @@ import type { Extension, Service, ServiceDesignator, + Socket, Worker_Binding, Worker_Module, } from "../../runtime"; @@ -21,6 +22,7 @@ import type { QueueConsumerSchema, QueueProducerSchema, } from "../../workers"; +import type { ArtifactsController } from "../artifacts/controller"; import type { ContainerPrivilegesCache } from "../core/container"; import type { DOContainerOptions } from "../do"; import type { HyperdriveProxyController } from "../hyperdrive/hyperdrive-proxy"; @@ -88,11 +90,14 @@ export interface PluginServicesOptions { devRegistryEnabled: boolean; containerPrivilegesCache: ContainerPrivilegesCache; hyperdriveProxyController: HyperdriveProxyController; + artifactsController: ArtifactsController; } export interface ServicesExtensions { services: Service[]; extensions: Extension[]; + // Internal plugin listeners (e.g. Git smart HTTP), deduplicated by name. + sockets?: Socket[]; } /** diff --git a/packages/miniflare/src/workers/artifacts/binding.worker.ts b/packages/miniflare/src/workers/artifacts/binding.worker.ts new file mode 100644 index 00000000000..1ad28e69706 --- /dev/null +++ b/packages/miniflare/src/workers/artifacts/binding.worker.ts @@ -0,0 +1,1255 @@ +import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; + +const repoName = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +type TokenScope = "read" | "write"; + +type RepositorySource = + | { type: "fork"; repository: string } + | { type: "remote"; url: string }; + +interface RepositoryOptions { + description?: string | undefined; + readOnly?: boolean; + setDefaultBranch?: string; +} + +interface ForkOptions { + description?: string; + readOnly?: boolean; + defaultBranchOnly?: boolean; +} + +interface ImportRepositoryOptions { + source: { url: string; branch?: string; depth?: number }; + target: { name: string; opts?: { description?: string; readOnly?: boolean } }; +} + +interface RepositoryToken { + id: string; + tokenHash: string; + scope: TokenScope; + createdAt: string; + expiresAt: string; + revokedAt?: string; +} + +interface CreatedToken { + record: RepositoryToken; + plaintext: string; +} + +interface Repository { + id: string; + name: string; + namespace: string; + description?: string | undefined; + readOnly: boolean; + defaultBranch: string; + remote: string; + source?: RepositorySource | undefined; + createdAt: string; + updatedAt: string; + lastPushAt?: string | undefined; + tokens: RepositoryToken[]; +} + +interface RepositoryMetadata { + id: string; + name: string; + remote: string; + defaultBranch: string; + description: string | null; + source: string | null; + readOnly: boolean; + createdAt: string; + updatedAt: string; + lastPushAt: string | null; +} + +interface CreateRepositoryResult { + id: string; + name: string; + description: string | null; + defaultBranch: string; + remote: string; + token: string; +} + +interface CreateTokenResult { + id: string; + plaintext: string; + scope: TokenScope; + expiresAt: string; +} +type TokenInfo = Omit & { + state: "active" | "expired" | "revoked"; +}; + +interface RepositoryListOptions { + cursor?: string; + limit?: number; +} + +type RepositoryListResult = { + repos: Array< + RepositoryMetadata & { status: "ready"; jurisdiction: "unrestricted" } + >; + total: number; + cursor?: string; +}; + +interface RepositoryLogOptions { + ref?: string; + limit?: number; + offset?: number; +} + +interface BackendState { + defaultBranch: string; + refs: Record; +} + +type BackendRequest = + | { action: "create"; defaultBranch: string } + | { action: "delete" } + | { + action: "fork"; + sourceName: string; + sourceBranch: string; + defaultBranchOnly?: boolean; + } + | { action: "import"; sourceUrl: string; branch?: string; depth?: number } + | { action: "refs" } + | { action: "readBlob"; hash: string } + | { action: "readTree"; hash: string } + | { action: "readCommit"; hash: string } + | { action: "file"; ref: string; path: string } + | { action: "log"; ref?: string; limit?: number; offset?: number }; + +interface TreeEntry { + name: string; + mode: string; + hash: string; + type: "tree" | "blob" | "symlink" | "gitlink" | "exec"; +} + +interface CommitMetadata { + hash: string; + treeHash: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + parents: string[]; + authoredAt: number; + committedAt: number; +} + +interface LocalArtifactsRepo { + info(): Promise; + createToken(scope?: TokenScope, ttl?: number): Promise; + listTokens(): Promise<{ total: number; tokens: TokenInfo[] }>; + revokeToken(tokenOrId: string): Promise; + readBlob(hash: string): Promise; + readTree(hash: string): Promise; + readCommit(hash: string): Promise; + readFile(args: { ref: string; path: string }): Promise; + log(options?: RepositoryLogOptions): Promise; + fork(name: string, options?: ForkOptions): Promise; +} + +interface BackendBlob { + data: string; +} + +type ArtifactsErrorCode = + | "ALREADY_EXISTS" + | "NOT_FOUND" + | "INVALID_INPUT" + | "INVALID_REPO_NAME" + | "INVALID_TTL" + | "INVALID_URL" + | "REMOTE_AUTH_REQUIRED" + | "MEMORY_LIMIT" + | "UPSTREAM_UNAVAILABLE" + | "INTERNAL_ERROR"; + +const numericCodes: Record = { + INVALID_INPUT: 10100, + INVALID_REPO_NAME: 10101, + INVALID_TTL: 10103, + INVALID_URL: 10104, + REMOTE_AUTH_REQUIRED: 10106, + NOT_FOUND: 10200, + ALREADY_EXISTS: 10201, + INTERNAL_ERROR: 10400, + UPSTREAM_UNAVAILABLE: 10401, + MEMORY_LIMIT: 10402, +}; + +class ArtifactsError extends Error { + readonly code: ArtifactsErrorCode; + readonly numericCode: number; + + constructor(code: ArtifactsErrorCode, message: string) { + super(message); + this.name = "ArtifactsError"; + this.code = code; + this.numericCode = numericCodes[code]; + } +} + +interface LocalArtifactsEnvironment { + config: { namespace: string; origin: string }; + localArtifactsNamespace: DurableObjectNamespace; + gitBackend: Fetcher; +} + +type RepositoryMap = Record; + +export class LocalArtifactsNamespace extends WorkerEntrypoint { + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return publicCall(() => this.#namespace().create(name, options)); + } + + async get(name: string): Promise { + const repository = await publicCall(() => this.#namespace().get(name)); + return repository as unknown as LocalArtifactsRepo; + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return publicCall(() => this.#namespace().import(options)); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + return publicCall(() => this.#namespace().list(options)); + } + + async delete(name: string): Promise { + return publicCall(() => this.#namespace().delete(name)); + } + + override async fetch(request: Request): Promise { + return this.#namespace().fetch(request); + } + + #namespace(): DurableObjectStub { + const { localArtifactsNamespace, config } = this.env; + return localArtifactsNamespace.get( + localArtifactsNamespace.idFromName(config.namespace) + ); + } +} + +export class LocalArtifactsNamespaceObject extends DurableObject { + // The resolved tail lets the first mutation run immediately. exclusive() + // serializes subsequent metadata and backend mutations across awaits. + private operations = Promise.resolve(); + + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + + const repository = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + try { + const state = await this.backend(repository, { + action: "create", + defaultBranch: repository.defaultBranch, + }); + applyBackendState(repository, state); + repositories[name.toLowerCase()] = repository; + const token = await createToken(repository, "write"); + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist Artifacts repository" + ); + } + return createRepoResult(repository, token); + } catch (error) { + if (error instanceof ArtifactsError) { + throw error; + } + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to create Artifacts repository" + ); + } + }); + } + + async get(name: string): Promise { + validateName(name); + const repository = await this.getRepo(name); + return createRepoHandle(this, repository.name, repository.id); + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return this.exclusive(async () => { + const { source, target, sourceUrl } = parseImportRequest(options); + const repositories = await this.repos(); + if (hasRepository(repositories, target.name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${target.name}` + ); + } + validateRepositoryOptions(target.opts ?? {}); + const repository = makeRepo( + target.name, + this.env.config.namespace, + this.env.config.origin, + target.opts ?? {} + ); + // The clone URL may contain one-time credentials. Keep them out of + // repository metadata returned by info() and persisted in storage. + const publicSourceUrl = new URL(sourceUrl); + publicSourceUrl.username = ""; + publicSourceUrl.password = ""; + publicSourceUrl.search = ""; + publicSourceUrl.hash = ""; + repository.source = { type: "remote", url: publicSourceUrl.href }; + const state = await this.backend(repository, { + action: "import", + sourceUrl: sourceUrl.href, + ...(source.branch === undefined ? {} : { branch: source.branch }), + ...(source.depth === undefined ? {} : { depth: source.depth }), + }); + applyBackendState(repository, state); + repository.updatedAt = new Date().toISOString(); + const token = await createToken(repository, "write"); + repositories[repository.name.toLowerCase()] = repository; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist imported Artifacts repository" + ); + } + return createRepoResult(repository, token); + }); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + const limit = options.limit ?? 50; + if (!Number.isInteger(limit) || limit < 1 || limit > 200) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 200." + ); + } + const offset = decodeCursor(options.cursor); + const repositories = Object.values(await this.repos()).sort((left, right) => + right.createdAt.localeCompare(left.createdAt) + ); + const page = repositories + .slice(offset, offset + limit) + .map((repository) => ({ + ...repoMetadata(repository), + status: "ready" as const, + // Local storage has no placement jurisdiction; this is the public default. + jurisdiction: "unrestricted" as const, + })); + const cursor = + offset + page.length < repositories.length + ? encodeCursor(offset + page.length) + : undefined; + return { + repos: page, + total: repositories.length, + ...(cursor === undefined ? {} : { cursor }), + }; + } + + async delete(name: string): Promise { + return this.exclusive(async () => { + validateName(name); + const repositories = await this.repos(); + if (!hasRepository(repositories, name)) { + return false; + } + const repository = getRepo(repositories, name); + // Persist removal first. If storage fails, the Git contents and + // metadata must both remain usable. Git cleanup first retires the + // directory atomically; startup or a later creation cleans orphans. + delete repositories[name.toLowerCase()]; + await this.saveRepos(repositories); + await this.backend(repository, { action: "delete" }); + return true; + }); + } + + async createToken( + name: string, + scope: TokenScope = "write", + ttl?: number, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const token = await createToken(repository, scope, ttl); + await this.saveRepos(repositories); + return tokenResult(token); + }); + } + + async listTokens( + name: string, + expectedId?: string + ): Promise<{ total: number; tokens: TokenInfo[] }> { + const repository = await this.getRepo(name, expectedId); + const active = repository.tokens.filter( + (token) => !token.revokedAt && isActive(token) + ); + const tokens = active + .toReversed() + .slice(0, 30) + .map(({ tokenHash: _tokenHash, revokedAt: _revokedAt, ...token }) => ({ + ...token, + state: "active" as const, + })); + return { total: active.length, tokens }; + } + + async revokeToken( + name: string, + tokenOrId: string, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + if (!isTokenId(tokenOrId) && !isArtifactToken(tokenOrId)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid tokenOrId: must be a token id or artifact token." + ); + } + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const tokenHash = isArtifactToken(tokenOrId) + ? await hashToken(tokenOrId) + : undefined; + const token = repository.tokens.find( + (candidate) => + candidate.id === tokenOrId || + (tokenHash !== undefined && + constantTimeEqual(candidate.tokenHash, tokenHash)) + ); + if (!token || token.revokedAt) { + return false; + } + token.revokedAt = new Date().toISOString(); + await this.saveRepos(repositories); + return true; + }); + } + + async fork( + sourceName: string, + name: string, + options: ForkOptions = {}, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + const source = getRepo(repositories, sourceName, expectedId); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + const fork = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + let state: BackendState; + try { + state = await this.backend(fork, { + action: "fork", + sourceName: source.name, + sourceBranch: source.defaultBranch, + ...(options.defaultBranchOnly === undefined + ? {} + : { defaultBranchOnly: options.defaultBranchOnly }), + }); + } catch { + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to fork Artifacts repository" + ); + } + applyBackendState(fork, state); + fork.source = { type: "fork", repository: source.name }; + const token = await createToken(fork, "write"); + repositories[name.toLowerCase()] = fork; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(fork); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist forked Artifacts repository" + ); + } + return createRepoResult(fork, token); + }); + } + + override async fetch(request: Request): Promise { + return handleGitRequest(this, this.env.config.namespace, request); + } + + async repos(): Promise { + const repositories = + (await this.ctx.storage.get("repos")) ?? + (Object.create(null) as RepositoryMap); + // The local Git listener can move after a process restart. + for (const repository of Object.values(repositories)) { + repository.remote = `${this.env.config.origin}/git/${repository.namespace}/${repository.name}.git`; + } + return repositories; + } + + saveRepos(repositories: RepositoryMap): Promise { + return this.ctx.storage.put("repos", repositories); + } + + async getRepo(name: string, expectedId?: string): Promise { + return getRepo(await this.repos(), name, expectedId); + } + + async backend( + repository: Repository, + request: BackendRequest + ): Promise { + const response = await this.env.gitBackend.fetch( + "http://local-artifacts-backend/__local_artifacts__", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ...request, + namespace: repository.namespace, + name: repository.name, + generation: repository.id, + }), + } + ); + if (!response.ok) { + const message = await response.text(); + if (request.action === "import") { + throw classifyImportError(message); + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } + return response.json(); + } + + private async deleteBackend(repository: Repository): Promise { + try { + await this.backend(repository, { action: "delete" }); + } catch { + // Preserve the original persistence error; native delete is idempotent on retry. + } + } + + gitFetch(repository: Repository, request: Request): Promise { + const headers = new Headers(request.headers); + headers.delete("Authorization"); + headers.set("X-Local-Artifacts-Generation", repository.id); + const url = new URL(request.url); + url.pathname = url.pathname.replace( + /^\/git\/[^/]+\/[^/]+\.git/, + `/git/${repository.namespace}/${repository.name}.git` + ); + return this.env.gitBackend.fetch( + new Request(url, new Request(request, { headers })) + ); + } + + async syncAfterPush( + name: string, + previousRefs: Record + ): Promise { + await this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name); + const state = await this.backend(repository, { + action: "refs", + }); + if (JSON.stringify(state.refs) !== JSON.stringify(previousRefs)) { + repository.updatedAt = new Date().toISOString(); + repository.lastPushAt = repository.updatedAt; + } + await this.saveRepos(repositories); + }); + } + + private async exclusive(action: () => Promise): Promise { + const previous = this.operations; + let release!: () => void; + this.operations = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await action(); + } finally { + release(); + } + } +} + +function createRepoHandle( + namespace: LocalArtifactsNamespaceObject, + name: string, + id: string +): LocalArtifactsRepo { + const read = (request: BackendRequest): Promise => + publicCall(async () => + namespace.backend(await namespace.getRepo(name, id), request) + ); + const readFile = async (args: { + ref: string; + path: string; + }): Promise => { + if (!args?.ref) { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: ref required."); + } + if (!args?.path) { + throw new ArtifactsError("INVALID_INPUT", "Invalid path: path required."); + } + const path = normalizePath(args.path); + if (path === null) { + return null; + } + const result = await read({ + action: "file", + ref: args.ref, + path, + }); + if (result === null) { + return null; + } + const bytes = decodeBase64(result.data); + return new Blob([bytes], { type: contentType(args.path, bytes) }); + }; + + return { + info: () => + publicCall(async () => repoMetadata(await namespace.getRepo(name, id))), + createToken: (scope: TokenScope = "write", ttl?: number) => + publicCall(() => namespace.createToken(name, scope, ttl, id)), + listTokens: () => publicCall(() => namespace.listTokens(name, id)), + revokeToken: (tokenOrId: string) => + publicCall(() => namespace.revokeToken(name, tokenOrId, id)), + readBlob: async (hash: string) => { + validateHash(hash); + const result = await read({ + action: "readBlob", + hash, + }); + return result === null ? null : new Blob([decodeBase64(result.data)]); + }, + readTree: async (hash: string) => { + validateHash(hash); + return await read({ action: "readTree", hash }); + }, + readCommit: async (hash: string) => { + validateHash(hash); + return await read({ action: "readCommit", hash }); + }, + readFile, + log: (options: RepositoryLogOptions = {}) => { + validateLogOptions(options); + return read({ + action: "log", + ...(options.ref === undefined ? {} : { ref: options.ref }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.offset === undefined ? {} : { offset: options.offset }), + }); + }, + fork: (targetName: string, options: ForkOptions = {}) => + publicCall(() => namespace.fork(name, targetName, options, id)), + }; +} + +function parseGitRoute( + request: Request, + namespace: string +): { name: string; write: boolean; push: boolean } | null { + const url = new URL(request.url); + const match = url.pathname.match(/^\/git\/([^/]+)\/([^/]+)\.git(?:\/(.*))?$/); + if (!match || match[1] !== namespace || !match[2]) { + return null; + } + const path = match[3] ?? ""; + const service = url.searchParams.get("service"); + const discovery = + request.method === "GET" && + path === "info/refs" && + (service === "git-upload-pack" || service === "git-receive-pack"); + const pack = + request.method === "POST" && + (path === "git-upload-pack" || path === "git-receive-pack"); + if (!discovery && !pack) { + return null; + } + return { + name: match[2], + write: path === "git-receive-pack" || service === "git-receive-pack", + push: request.method === "POST" && path === "git-receive-pack", + }; +} + +async function handleGitRequest( + state: LocalArtifactsNamespaceObject, + namespace: string, + request: Request +): Promise { + const route = parseGitRoute(request, namespace); + if (!route) { + return new Response("Not found", { status: 404 }); + } + let repository: Repository; + try { + repository = await state.getRepo(route.name); + } catch (error) { + if (error instanceof ArtifactsError && error.code === "NOT_FOUND") { + return new Response("Not found", { status: 404 }); + } + throw error; + } + const authorizationError = await authorize( + repository, + request, + route.write ? "write" : "read" + ); + if (authorizationError) { + return authorizationError; + } + if (route.write && repository.readOnly) { + return new Response("Artifacts repository is read-only", { status: 403 }); + } + + const previousRefs = route.push + ? (await state.backend(repository, { action: "refs" })).refs + : undefined; + const response = await state.gitFetch(repository, request); + if (!previousRefs) { + return response; + } + + const body = await response.arrayBuffer(); + if (response.ok) { + try { + await state.syncAfterPush(repository.name, previousRefs); + } catch { + // Git has already accepted the push; metadata bookkeeping must not fail the client request. + } + } + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function authorize( + repository: Repository, + request: Request, + scope: TokenScope +): Promise { + const header = request.headers.get("Authorization") ?? ""; + let value = ""; + if (header.startsWith("Bearer ")) { + value = header.slice(7); + } else if (header.startsWith("Basic ")) { + try { + value = atob(header.slice(6)).split(":").slice(1).join(":"); + } catch { + value = ""; + } + } + const valueHash = await hashToken(value); + const token = repository.tokens.find( + (candidate) => + !candidate.revokedAt && + constantTimeEqual(candidate.tokenHash, valueHash) && + isActive(candidate) + ); + if (!token) { + return new Response("Invalid Artifacts token", { + status: 401, + headers: { "WWW-Authenticate": 'Basic realm="Artifacts"' }, + }); + } + if (scope === "write" && token.scope !== "write") { + return new Response("Artifacts token requires write scope", { + status: 403, + }); + } + return undefined; +} + +function applyBackendState(repository: Repository, state: BackendState): void { + repository.defaultBranch = state.defaultBranch; +} + +function makeRepo( + name: string, + namespace: string, + origin: string, + options: RepositoryOptions +): Repository { + const time = new Date().toISOString(); + return { + id: randomBase36(16), + name, + namespace, + description: options.description, + readOnly: Boolean(options.readOnly), + defaultBranch: options.setDefaultBranch ?? "main", + remote: `${origin}/git/${namespace}/${name}.git`, + createdAt: time, + updatedAt: time, + lastPushAt: undefined, + tokens: [], + }; +} + +function repoMetadata(repository: Repository): RepositoryMetadata { + return { + id: repository.id, + name: repository.name, + remote: repository.remote, + defaultBranch: repository.defaultBranch, + description: repository.description ?? null, + source: + repository.source === undefined + ? null + : repository.source.type === "fork" + ? `artifacts:${repository.namespace}/${repository.source.repository}` + : `git:${repository.source.url}`, + readOnly: repository.readOnly, + createdAt: repository.createdAt, + updatedAt: repository.updatedAt, + lastPushAt: repository.lastPushAt ?? null, + }; +} + +function createRepoResult( + repository: Repository, + token: CreatedToken +): CreateRepositoryResult { + return { + id: repository.id, + name: repository.name, + description: repository.description ?? null, + defaultBranch: repository.defaultBranch, + remote: repository.remote, + token: token.plaintext, + }; +} + +function tokenResult(token: CreatedToken): CreateTokenResult { + return { + id: token.record.id, + plaintext: token.plaintext, + scope: token.record.scope, + expiresAt: token.record.expiresAt, + }; +} + +function getRepo( + repositories: RepositoryMap, + name: string, + expectedId?: string +): Repository { + const repository = repositories[name.toLowerCase()]; + if ( + !repository || + (expectedId !== undefined && repository.id !== expectedId) + ) { + throw new ArtifactsError("NOT_FOUND", `Repository not found: ${name}.`); + } + return repository; +} + +function hasRepository(repositories: RepositoryMap, name: string): boolean { + return Object.hasOwn(repositories, name.toLowerCase()); +} + +function parseImportRequest(options: ImportRepositoryOptions): { + source: ImportRepositoryOptions["source"]; + target: ImportRepositoryOptions["target"]; + sourceUrl: URL; +} { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts import options are required" + ); + } + const { source, target } = options; + validateName(target?.name); + let sourceUrl: URL; + try { + sourceUrl = new URL(source?.url); + } catch { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be a valid URL." + ); + } + if (sourceUrl.protocol !== "https:") { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be an HTTPS URL." + ); + } + return { source, target, sourceUrl }; +} + +function validateName(name: unknown): asserts name is string { + if (typeof name !== "string" || !repoName.test(name)) { + throw new ArtifactsError("INVALID_REPO_NAME", "Invalid repo name."); + } +} + +function validateRepositoryOptions( + options: RepositoryOptions | ForkOptions +): void { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository options must be an object" + ); + } + if ( + options.description !== undefined && + typeof options.description !== "string" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository description must be a string" + ); + } + if (options.readOnly !== undefined && typeof options.readOnly !== "boolean") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository readOnly must be a boolean" + ); + } + if ("setDefaultBranch" in options && options.setDefaultBranch !== undefined) { + validateBranch(options.setDefaultBranch); + } + if ( + "defaultBranchOnly" in options && + options.defaultBranchOnly !== undefined && + typeof options.defaultBranchOnly !== "boolean" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts defaultBranchOnly must be a boolean" + ); + } +} + +function validateBranch(branch: unknown): asserts branch is string { + if ( + typeof branch !== "string" || + branch.length < 1 || + branch.length > 256 || + branch.startsWith("-") || + branch.startsWith(".") || + branch.endsWith("/") || + branch.endsWith(".") || + branch.endsWith(".lock") || + branch === "@" || + branch.includes("..") || + branch.includes("//") || + branch.includes("@{") || + branch.split("/").some((segment) => segment.startsWith(".")) || + [...branch].some((character) => character.charCodeAt(0) <= 32) || + /[~^:?*[\\]/.test(branch) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts default branch is invalid" + ); + } +} + +function validateLogOptions( + options: unknown +): asserts options is RepositoryLogOptions { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts log options must be an object" + ); + } + const { ref, limit, offset } = options as RepositoryLogOptions; + if (ref !== undefined && typeof ref !== "string") { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: must be a string."); + } + if ( + limit !== undefined && + (!Number.isInteger(limit) || limit < 1 || limit > 1000) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 1000." + ); + } + if (offset !== undefined && (!Number.isInteger(offset) || offset < 0)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid offset: must be a non-negative integer." + ); + } +} + +async function createToken( + repository: Repository, + scope: TokenScope = "write", + ttl = 86_400 +): Promise { + if (scope !== "read" && scope !== "write") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts token scope must be read or write" + ); + } + if (!Number.isInteger(ttl) || ttl < 60 || ttl > 31_536_000) { + throw new ArtifactsError( + "INVALID_TTL", + `Invalid TTL ${ttl}: must be between 60 and 31536000 seconds.` + ); + } + const createdAt = new Date(); + const expiresAt = new Date(createdAt.getTime() + ttl * 1000); + const plaintext = `art_v2_x_${randomHex(20)}?expires=${Math.floor(expiresAt.getTime() / 1000)}`; + const record = { + id: randomBase36(16), + tokenHash: await hashToken(plaintext), + scope, + createdAt: createdAt.toISOString(), + expiresAt: expiresAt.toISOString(), + }; + repository.tokens.push(record); + return { record, plaintext }; +} + +function isActive(token: Pick): boolean { + return Date.parse(token.expiresAt) > Date.now(); +} + +function randomHex(bytes: number): string { + return [...crypto.getRandomValues(new Uint8Array(bytes))] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +async function hashToken(value: string): Promise { + const secret = value.split("?", 1)[0] ?? ""; + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(secret) + ); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function constantTimeEqual(left: string, right: string): boolean { + if (left.length !== right.length) { + return false; + } + let difference = 0; + for (let index = 0; index < left.length; index++) { + difference |= left.charCodeAt(index) ^ right.charCodeAt(index); + } + return difference === 0; +} + +function randomBase36(length: number): string { + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"; + return [...crypto.getRandomValues(new Uint8Array(length))] + .map((byte) => alphabet[byte % alphabet.length]) + .join(""); +} + +function isTokenId(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-z]{16}$/.test(value); +} + +function isArtifactToken(value: unknown): value is string { + return ( + typeof value === "string" && + /^art_v(?:1_[0-9a-f]{40}|2_[xeuf]_[0-9a-f]{40})(?:\?expires=\d+)?$/.test( + value + ) + ); +} + +function validateHash(hash: unknown): asserts hash is string { + if (!hash) { + throw new ArtifactsError("INVALID_INPUT", "Invalid hash: hash required."); + } + if (typeof hash !== "string" || !/^[0-9a-f]{40}$/.test(hash)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid hash: expected a 40-character SHA-1 hex hash." + ); + } +} + +function normalizePath(path: string): string | null { + const segments = path + .split("/") + .filter((segment) => segment !== "" && segment !== "."); + if (segments.includes("..") || segments.length === 0) { + return null; + } + return segments.join("/"); +} + +function contentType(path: string, bytes: Uint8Array): string { + const extension = path.toLowerCase().match(/\.[^./]+$/)?.[0]; + const known: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".ico": "image/x-icon", + ".pdf": "application/pdf", + }; + if (extension && known[extension]) { + return known[extension]; + } + return bytes.subarray(0, 8192).includes(0) + ? "application/octet-stream" + : "text/plain;charset=utf-8"; +} + +function decodeBase64(value: string): Uint8Array { + return Uint8Array.from(atob(value), (character) => character.charCodeAt(0)); +} + +function encodeCursor(offset: number): string { + return btoa(JSON.stringify({ offset })); +} + +function decodeCursor(cursor: string | undefined): number { + if (cursor === undefined) { + return 0; + } + try { + const value = JSON.parse(atob(cursor)) as { offset?: unknown }; + return typeof value.offset === "number" && + Number.isInteger(value.offset) && + value.offset >= 0 + ? value.offset + : 0; + } catch { + return 0; + } +} + +async function publicCall(action: () => Promise): Promise { + try { + return await action(); + } catch (error) { + if (isArtifactsError(error)) { + throw error; + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } +} + +function isArtifactsError(error: unknown): error is ArtifactsError { + return ( + error instanceof Error && + error.name === "ArtifactsError" && + "code" in error && + typeof error.code === "string" && + "numericCode" in error && + typeof error.numericCode === "number" + ); +} + +function classifyImportError(message: string): ArtifactsError { + const normalized = message.toLowerCase(); + if ( + normalized.includes("authentication failed") || + normalized.includes("could not read username") || + normalized.includes("http 401") + ) { + return new ArtifactsError( + "REMOTE_AUTH_REQUIRED", + "Authentication is required to access the remote repository." + ); + } + if ( + normalized.includes("remote branch") && + normalized.includes("not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote branch not found."); + } + if ( + normalized.includes("not found") || + normalized.includes("repository not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote repository not found."); + } + if (normalized.includes("does not appear to be a git repository")) { + return new ArtifactsError( + "INVALID_URL", + "The URL does not point to a git repository." + ); + } + return new ArtifactsError( + "UPSTREAM_UNAVAILABLE", + "The upstream service is unavailable. Please retry." + ); +} diff --git a/packages/miniflare/test/config/v4-convert.spec.ts b/packages/miniflare/test/config/v4-convert.spec.ts index d121d3da45d..9516f28f279 100644 --- a/packages/miniflare/test/config/v4-convert.spec.ts +++ b/packages/miniflare/test/config/v4-convert.spec.ts @@ -4,6 +4,27 @@ import { convertV4MiniflareOptions } from "../../src/config/v4-convert"; import type { RemoteProxyConnectionString } from "../../src/plugins/shared"; describe("convertV4MiniflareOptions", () => { + test("Artifacts defaults to local and only an explicit connection is remote", ({ + expect, + }) => { + const remoteProxyConnectionString = new URL( + "http://127.0.0.1:8787" + ) as RemoteProxyConnectionString; + const converted = convertV4MiniflareOptions({ + name: "worker", + script: "export default {};", + artifacts: { + LOCAL: { namespace: "local" }, + REMOTE: { namespace: "remote", remoteProxyConnectionString }, + }, + }); + expect(converted.workers[0].config.env?.LOCAL).toMatchObject({ + dev: { remote: false }, + }); + expect(converted.workers[0].config.env?.REMOTE).toMatchObject({ + dev: { remote: true }, + }); + }); test("preserves opaque K2 stream IDs through v4 parsing and conversion", ({ expect, }) => { diff --git a/packages/miniflare/test/plugins/artifacts/contract.spec.ts b/packages/miniflare/test/plugins/artifacts/contract.spec.ts new file mode 100644 index 00000000000..bb7244101c5 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract.spec.ts @@ -0,0 +1,269 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { singleModuleManifest, useDispose } from "../../test-shared"; +import { compare, normalize } from "./contract/compare"; +import { runApprovedLiveComparison } from "./contract/live"; +import { runScenarios } from "./contract/scenarios"; +import type { + Created, + ArtifactsBinding, + Observation, +} from "./contract/scenarios"; + +const exec = promisify(execFile); +const SCRIPT = `export default { fetch() { return new Response("contract"); } };`; + +async function prepareLocalFixture(created: Created): Promise { + const source = await mkdtemp(path.join(tmpdir(), "artifacts-contract-")); + const environment = { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + GIT_AUTHOR_DATE: "2020-01-02T03:04:05Z", + GIT_COMMITTER_DATE: "2020-01-02T03:04:05Z", + }; + const git = async (...args: string[]) => + exec("git", args, { cwd: source, env: environment }); + try { + await git("init", "--initial-branch=main"); + await mkdir(path.join(source, "folder")); + await writeFile(path.join(source, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(source, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await writeFile(path.join(source, "folder", "nested.txt"), "nested\n"); + await git("add", "."); + await git("commit", "-m", "first"); + await writeFile(path.join(source, "second.txt"), "second\n"); + await git("add", "."); + await git("commit", "-m", "second"); + // Keep the token out of argv, stderr, and test output. The child alone sees the header. + const pushEnv = { + ...environment, + GIT_CONFIG_COUNT: "2", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `Authorization: Bearer ${created.token}`, + }; + await exec("git", ["push", created.remote, "main"], { + cwd: source, + env: pushEnv, + }); + await git("checkout", "-b", "feature"); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git("add", "."); + await git("commit", "-m", "feature"); + await exec("git", ["push", created.remote, "feature"], { + cwd: source, + env: pushEnv, + }); + } finally { + await removeDir(source); + } +} + +function local(): Miniflare { + return new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { + type: "artifacts", + namespace: "contract-local", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); +} + +test("shared Artifacts contract runs against the local RPC binding and cleans up", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const observations = await runScenarios( + REPOS, + "contract-test-", + prepareLocalFixture + ); + const find = (step: string) => + observations.find((item) => item.step === step); + expect(observations.length).toBeGreaterThan(50); + expect(find("file binary")).toMatchObject({ + outcome: "value", + value: { + blob: { type: "application/octet-stream", bytes: [0, 255, 128, 1] }, + }, + }); + expect(find("file text")).toMatchObject({ + outcome: "value", + value: { blob: { type: "text/plain;charset=utf-8" } }, + }); + expect(find("duplicate case")).toMatchObject({ outcome: "error" }); + expect(find("retained after delete")).toMatchObject({ outcome: "error" }); + for (const kind of ["blob", "tree", "commit"]) { + expect(find(`${kind} present`)).toMatchObject({ outcome: "value" }); + expect(find(`${kind} missing`)).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find(`${kind} wrong type`)).toMatchObject({ + outcome: "value", + value: null, + }); + } + expect(find("file missing ref")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("file missing path")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("list next page")).toMatchObject({ outcome: "value" }); + expect(find("token revoke wrong repo")).toMatchObject({ + outcome: "value", + value: false, + }); + expect(find("fork all feature")).toMatchObject({ + outcome: "value", + value: { blob: { bytes: [...new TextEncoder().encode("feature\n")] } }, + }); + expect(find("fork main feature")).toMatchObject({ + outcome: "value", + value: null, + }); + expect((await REPOS.list()).total).toBe(0); +}); + +test("comparator preserves exact errors, Blob data, paths and non-loopback URLs", ({ + expect, +}) => { + const base: Observation[] = [ + { + step: "file", + outcome: "value", + value: { blob: { bytes: [0, 255], type: "text/plain" } }, + }, + { + step: "error", + outcome: "error", + value: { + name: "ArtifactsError", + code: "NOT_FOUND", + numericCode: 10200, + message: "not found", + }, + }, + { + step: "repo", + outcome: "value", + value: { + id: "first", + createdAt: "2020-01-01T00:00:00Z", + remote: "http://127.0.0.1:1234/git/repo.git", + name: "repo", + }, + }, + ]; + const variable = structuredClone(base); + (variable[2].value as Record).id = "second"; + (variable[2].value as Record).createdAt = + "2026-01-01T00:00:00Z"; + (variable[2].value as Record).remote = + "http://127.0.0.1:5678/git/repo.git"; + expect(compare(base, variable)).toEqual([]); + const different = structuredClone(variable); + (different[0].value as { blob: { bytes: number[] } }).blob.bytes[1] = 254; + (different[1].value as Record).numericCode = 10201; + (different[2].value as Record).remote = + "https://example.test/git/repo.git"; + expect( + compare(base, different).map(({ step, path, kind }) => [step, path, kind]) + ).toEqual([ + ["file", "$.blob.bytes[1]", "value"], + ["error", "$.numericCode", "error"], + ["repo", "$.remote", "value"], + ]); + expect( + normalize([ + { + step: "err", + outcome: "error", + value: { message: "id first at 2020-01-01T00:00:00Z" }, + }, + ])[0].value + ).toEqual({ message: "id first at 2020-01-01T00:00:00Z" }); + expect(compare(base, base.slice(1))[0].kind).toBe("missing-step"); + expect(compare(base, [...base, base[0]])[0]).toMatchObject({ + kind: "missing-step", + path: "$[3]", + }); + const reordered = structuredClone(base); + reordered.reverse(); + expect(compare(base, reordered)[0].kind).toBe("missing-step"); + const changed = structuredClone(base); + (changed[0].value as { blob: { type: string } }).blob.type = + "application/octet-stream"; + expect(compare(base, changed)).toMatchObject([ + { path: "$.blob.type", kind: "value" }, + ]); +}); + +test("live comparison refuses unapproved or reused fixtures without calling either binding", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const options = { + namespace: "contract-local", + approvedDisposableNamespace: "not-approved", + fixturePrefix: "contract-test-", + approvedDisposableFixturePrefix: "contract-test-", + confirmation: "I_APPROVE_DISPOSABLE_ARTIFACTS_COMPARISON" as const, + local: REPOS, + live: REPOS, + prepareLocalFixture, + prepareLiveFixture: prepareLocalFixture, + }; + await expect(runApprovedLiveComparison(options)).rejects.toThrow(/approval/); + await expect( + runApprovedLiveComparison({ + ...options, + approvedDisposableNamespace: "contract-local", + }) + ).rejects.toThrow(/distinct/); + expect((await REPOS.list()).total).toBe(0); + await REPOS.create("contract-test-main"); + await expect( + runScenarios(REPOS, "contract-test-", prepareLocalFixture) + ).rejects.toThrow(/not empty/); + expect((await REPOS.list()).total).toBe(1); + await REPOS.delete("contract-test-main"); + await expect( + runApprovedLiveComparison({ + ...options, + approvedDisposableNamespace: "contract-local", + approvedDisposableFixturePrefix: "other-prefix-", + }) + ).rejects.toThrow(/approval/); +}); diff --git a/packages/miniflare/test/plugins/artifacts/contract/README.md b/packages/miniflare/test/plugins/artifacts/contract/README.md new file mode 100644 index 00000000000..54f4a81ecff --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/README.md @@ -0,0 +1,23 @@ +# Artifacts observable contract (draft) + +`../contract.spec.ts` runs `runScenarios()` against Miniflare's real RPC binding, with a deterministic, locally pushed Git fixture. The shared scenarios capture returned objects, errors (`name`, `message`, `code`, `numericCode`), Blob MIME types and bytes, and missing/null outcomes. They cover namespace create/get/list/delete (including case and pagination boundaries), retained handles after delete/recreate, tokens and wrong-repository revocation, Git objects/files/refs, and empty/branch-selective/read-only forks. Git HTTP authorization is covered separately in `../index.spec.ts`; this suite does not expose a public RPC HTTP endpoint. + +**Do not treat local observations as an approved remote specification.** The comparator reports field-level differences without updating either side. It normalizes only identity fields (`id`, `token`, `plaintext`), ISO timestamp fields, and the port of a loopback `remote` URL. In particular, error strings/codes, Blob bytes/types, Git hashes, cursor behavior, object fields, and non-loopback remote URLs remain exact. A non-loopback live remote compared to a loopback local remote will be reported, not silently accepted. Traces include credentials before comparison: do not print, persist, or upload raw observations. The comparison API returns normalized mismatches only. + +## Optional manual comparison (not run by CI) + +There is **no** standalone CLI, deploy script, automatic provisioning, authentication helper, or remote endpoint in this directory. After the namespace owner explicitly approves a **disposable** namespace and a unique fixture prefix, a trusted private test environment may obtain two real binding objects (local and live) and call `runApprovedLiveComparison` from `live.ts` in-process. Pass: + +- `namespace` and the identical `approvedDisposableNamespace` confirmed by the owner (the harness cannot verify binding-to-namespace mapping); +- `fixturePrefix` and the identical `approvedDisposableFixturePrefix` confirmed by the owner: a fresh lowercase prefix ending in `-` (for example, `contract-sample-`), reserved solely for this run. The preflight refuses a namespace containing _any_ repositories, because list counts and cursors are global; +- the literal `confirmation: "I_APPROVE_DISPOSABLE_ARTIFACTS_COMPARISON"`; +- `local`, `live`, and `prepareLocalFixture` / `prepareLiveFixture` callbacks. Each callback receives a fresh repo's `remote` and a short-lived write `token` and must push the same two main commits and one feature commit as the local fixture, using a protected credential channel (not argv, logs, or committed files). Supply bindings through an authorized Worker/test context, **not** an unrestricted public RPC fetch route. + +Never pass a production namespace, existing repository, shared prefix, or resource credential through a command line. The harness deletes only repository names whose create/fork RPC returned success in this run; it never deletes a namespace or provisions/deploys a Worker. A failed RPC may still have created a resource; inspect partial failures manually with owner approval. Concurrent writers to the same namespace are not safe: reserve the namespace for the duration of the run. Cleanup failures throw; inspect and remove only the named fixtures with owner approval. Do not run a live comparison without explicit authorization. The local spec is the only automatically executed path. + +## Unresolved contract decisions + +- Remote ownership/authorization, namespace-to-binding mapping, exclusive namespace reservation and disposal approval require an actual owner; none is implied by this test harness. +- Exact production behavior for missing/wrong-type Git objects, commit-hash refs, path normalization, malformed cursors, token visibility/revocation, and retained handles requires an approved remote run. Current local handle errors can differ between direct binding proxy and an in-Worker call; do not rewrite error observations to hide this. +- A returned production Git remote will not be normalized to the local loopback URL. Its difference needs an explicit owner decision, not a comparator exception. +- Local-only regression assertions are not evidence of live platform parity. No production calls have been made by this workstream. diff --git a/packages/miniflare/test/plugins/artifacts/contract/compare.ts b/packages/miniflare/test/plugins/artifacts/contract/compare.ts new file mode 100644 index 00000000000..71eabe1bf21 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/compare.ts @@ -0,0 +1,141 @@ +import type { Observation } from "./scenarios"; + +export interface Mismatch { + step: string; + path: string; + kind: "missing-step" | "outcome" | "error" | "shape" | "value"; + local: unknown; + live: unknown; +} + +/** Normalize only identity fields, timestamps and loopback origins; never object contents or error text. */ +export function normalize(trace: Observation[]): Observation[] { + const ids = new Map(); + function normalizeValue(value: unknown, key = ""): unknown { + if (Array.isArray(value)) { + return value.map((item) => normalizeValue(item)); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([field, item]) => [ + field, + normalizeValue(item, field), + ]) + ); + } + if (typeof value !== "string") { + return value; + } + if (["id", "token", "plaintext"].includes(key)) { + if (!ids.has(value)) { + ids.set(value, `<${key}:${ids.size + 1}>`); + } + return ids.get(value); + } + if ( + ["createdAt", "updatedAt", "lastPushAt", "expiresAt"].includes(key) && + /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z$/.test(value) + ) { + return "