diff --git a/.changeset/artifacts-opt-in-simulator.md b/.changeset/artifacts-opt-in-simulator.md new file mode 100644 index 00000000000..047636b2940 --- /dev/null +++ b/.changeset/artifacts-opt-in-simulator.md @@ -0,0 +1,5 @@ +--- +"miniflare": minor +--- + +Add an opt-in local Artifacts binding simulator to Miniflare. Set `dev.remote: false` on an Artifacts binding to use persistent local repositories and authenticated Git smart HTTP backed by host Git 2.32 or newer. Omitted `dev.remote` retains the existing remote binding behavior. diff --git a/packages/miniflare/scripts/build.mjs b/packages/miniflare/scripts/build.mjs index 56ba5b35265..516f094f304 100644 --- a/packages/miniflare/scripts/build.mjs +++ b/packages/miniflare/scripts/build.mjs @@ -446,7 +446,13 @@ async function buildPackage() { logLevel: watch ? "info" : "warning", outdir: outPath, outbase: pkgRoot, - entryPoints: [indexPath, ...fixtureBuilds], + entryPoints: [ + indexPath, + // A separate thread keeps native Git available during synchronous Node + // binding-proxy calls, which block Miniflare's main event loop. + path.join(pkgRoot, "src/plugins/artifacts/git-sidecar.ts"), + ...fixtureBuilds, + ], }; if (watch) { diff --git a/packages/miniflare/src/config/v4-convert.ts b/packages/miniflare/src/config/v4-convert.ts index 734cd06a2d4..cc4c19b1d40 100644 --- a/packages/miniflare/src/config/v4-convert.ts +++ b/packages/miniflare/src/config/v4-convert.ts @@ -796,7 +796,11 @@ function addProductBindings( env[name] = { type: "artifacts", namespace: binding.namespace, - dev: { remote: isRemote(binding.remoteProxyConnectionString) }, + // At this stage, only explicit V5 dev.remote: false selects local. + // Do not reinterpret an omitted legacy setting as that opt-in. + ...(binding.remoteProxyConnectionString === undefined + ? {} + : { dev: { remote: isRemote(binding.remoteProxyConnectionString) } }), }; } for (const name of Object.keys(worker.workerLoaders ?? {})) { diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 1284d267928..3f6c996bf0a 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -77,6 +77,7 @@ import { WORKER_BINDING_SERVICE_LOOPBACK, WORKFLOWS_PLUGIN_NAME, } from "./plugins"; +import { ArtifactsController } from "./plugins/artifacts/controller"; import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants"; import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version"; import { closeBrowserProcess } from "./plugins/browser-rendering/process"; @@ -888,6 +889,7 @@ export class Miniflare { #maybeInspectorProxyController?: InspectorProxyController; #previousRuntimeInspectorPort?: number; + #artifactsController = new ArtifactsController(() => this.#log); #containerPrivilegesCache = new ContainerPrivilegesCache(); #hyperdriveProxyController: HyperdriveProxyController = new HyperdriveProxyController(); @@ -2326,6 +2328,7 @@ export class Miniflare { queueConsumers, containerPrivilegesCache: this.#containerPrivilegesCache, hyperdriveProxyController: this.#hyperdriveProxyController, + artifactsController: this.#artifactsController, }; for (const [key, plugin] of this.#mergedPluginEntries) { const pluginServicesExtensions = await plugin.getServices({ @@ -2341,6 +2344,11 @@ export class Miniflare { } else { pluginServices = pluginServicesExtensions.services; extensions.push(...pluginServicesExtensions.extensions); + for (const socket of pluginServicesExtensions.sockets ?? []) { + if (!sockets.some((existing) => existing.name === socket.name)) { + sockets.push(socket); + } + } } for (const service of pluginServices) { @@ -2672,11 +2680,13 @@ export class Miniflare { async #assembleAndUpdateConfig(reusePorts = false) { this.#hyperdriveProxyController.beginUpdate(); + this.#artifactsController.beginUpdate(); try { await this.#assembleAndUpdateConfigInternal(reusePorts); } finally { - // If assembly or runtime startup failed, keep the old listeners and - // close only the new candidates. A successful update already committed. + // A failed update retains listeners used by the previous runtime and + // closes only candidates started during this attempt. + await this.#artifactsController.abortUpdate(); this.#hyperdriveProxyController.abortUpdate(); } } @@ -2800,6 +2810,7 @@ export class Miniflare { } } this.#hyperdriveProxyController.commitUpdate(activeExternalAddresses); + await this.#artifactsController.commitUpdate(); // Note: `updateConfig()` doesn't resolve until ports for all required // sockets have been recorded. At this point, `maybeSocketPorts` contains // all of `requiredSockets` as keys. @@ -3915,6 +3926,14 @@ export class Miniflare { } const runtimeCleanupOutcome = await runtimeDisposeOutcome; + try { + await this.#artifactsController.dispose(); + } catch (error) { + if (!independentCleanupFailed) { + independentCleanupFailed = true; + independentCleanupError = error; + } + } this.#devRegistry.unregisterWorkers(); try { await Promise.all( diff --git a/packages/miniflare/src/plugins/artifacts/README.md b/packages/miniflare/src/plugins/artifacts/README.md new file mode 100644 index 00000000000..1eb553a5d0c --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/README.md @@ -0,0 +1,168 @@ +# Artifacts in Miniflare + +The Artifacts plugin implements the local Artifacts binding in workerd. This +first version requires Git 2.32 or newer installed globally on the host and +available on PATH. Miniflare checks the Git version at local startup and gives +installation or upgrade instructions if needed. A Node worker thread runs the +native Git backend. Set `dev.remote` to `false` explicitly to select local +storage; an omitted setting retains remote binding behavior in this stage. +We plan to open-source a JavaScript Git engine in the future; it is not +included in this version. + +## Local startup + +```mermaid +sequenceDiagram + participant Dev as Developer tool + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Controller as ArtifactsController + participant Sidecar as Node Git sidecar + participant Workerd as workerd + + Dev->>MF: Configure Artifacts binding + MF->>Plugin: getBindings() + Plugin->>Plugin: Select local and validate namespace + Plugin-->>MF: Bind to local namespace entrypoint + MF->>Plugin: getServices() + Plugin->>Plugin: Verify host Git is available on PATH + Plugin->>Controller: get(Git storage path) + Controller->>Sidecar: Start worker thread and private HTTP listener + Sidecar-->>Controller: Address and secret + Controller-->>Plugin: Sidecar endpoint and Git socket port + Plugin-->>MF: Binding Worker, DO storage, backend service, Git socket + MF->>Workerd: Start configured services and socket +``` + +`getBindings()` routes each local binding to a namespace-scoped service. +`getServices()` creates the binding Worker, namespace Durable Object, persistent +metadata disk, internal Git backend service, and Git socket. The controller +reuses a sidecar per Git storage path and closes it when Miniflare is disposed. +The socket used by Git clients and the sidecar's private HTTP listener are +**different endpoints**. If the sidecar exits after startup, the controller +forgets its endpoint and warns; an explicit Miniflare configuration reload +reconnects, but existing workerd connections do not recover automatically. +Restart the dev server or reload its configuration if Git operations stop +working after a sidecar exit. + +## Local binding RPC + +For example, `env.REPOS.create("demo")`: + +```mermaid +sequenceDiagram + participant App as User Worker + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Disk as Local disk + + App->>Entry: env.REPOS.create("demo") + Entry->>DO: RPC create("demo") + DO->>DO: Validate and serialize mutation + DO->>Backend: POST internal create request + Backend->>Sidecar: Inject private backend header + Sidecar->>Disk: git init --bare + Sidecar-->>DO: Git state (via backend service) + DO->>Disk: Persist metadata and token hash + DO-->>Entry: Repository info and initial token + Entry-->>App: Create result +``` + +The Durable Object stores repository metadata and token hashes on local disk; +the sidecar stores the bare Git repositories separately. `git-sidecar.ts` +handles HTTP routing and repository locks, while `git-client.ts` owns +repository-scoped Git commands and result parsing. Read operations such as +`readFile()` also use the binding Worker and native Git backend. Node callers +reach the same binding through Miniflare's Node binding proxy. Namespace and +repository names are hashed into short, namespace-scoped on-disk paths; this +leaves room for Git packfiles and workerd SQLite files on platforms with path +length limits. Creation, fork, and import use private staging directories; +interrupted staging is removed on the next backend startup. If a process dies +while creating a repository, a leftover creation lock blocks reuse until you +confirm no other dev server uses the directory and remove the lock named in +the error. Import URL credentials are passed to Git as an ephemeral HTTPS +authorization header, +not recorded in a repository's Git config. Host Git runs with an isolated HOME +so it cannot read the developer's `.netrc`; SSH/scp-style imports and Git remote +helpers are rejected rather than accessing host SSH keys or executing helpers. +The local Worker binding accepts only explicit HTTPS imports; filesystem paths +are available only to the internal Git client. Repository locks are process-local: +do not run two dev servers against the same Artifacts persistence directory at +the same time. Older draft storage layouts are not migrated automatically: +startup rejects a detected old layout rather than silently creating empty +repositories. Back up any local repositories before resetting the Artifacts +persistence directory. Do not delete the persistence directory without first +checking whether it holds work you need. + +## Git clone and push + +```mermaid +sequenceDiagram + participant Git as Git client + participant Socket as Git socket + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Native as Native Git + + Git->>Socket: Git smart HTTP at /git/:namespace/:repo.git + Socket->>Entry: Route request + Entry->>DO: fetch(request) + DO->>DO: Check endpoint, repository, token, and scope + DO->>Backend: Forward request without Authorization header + Backend->>Sidecar: Inject private backend header + Sidecar->>Native: Stream via git http-backend + Native-->>Sidecar: Discovery or pack response + Sidecar-->>DO: Git response (via backend service) + opt Successful receive-pack push + DO->>Backend: Read updated refs + Backend->>Sidecar: Internal refs request + Sidecar-->>DO: Refs (via backend service) + DO->>DO: Persist updated push metadata + end + DO-->>Entry: Git response + Entry-->>Socket: Git response + Socket-->>Git: Git response +``` + +The binding Worker checks the supported Git endpoints and authenticates +repository tokens before forwarding. The internal service injects the sidecar +secret; the client's `Authorization` header is removed before that hop. For a +successful push, the Durable Object refreshes repository metadata before +returning the response. The sidecar uses native `git http-backend` for Git smart +HTTP rather than reproducing Git's pack protocol. All native Git commands +(including the HTTP backend) override repository-local hook configuration. +A failed deletion leaves an inaccessible, persisted tombstone, so retrying +`delete()` or creating the same name can finish native Git cleanup even after a +process restart. + +## Explicit remote binding + +When `dev.remote` is `true`, this binding uses the remote-bindings proxy instead +of creating local Artifacts storage or a local Git sidecar for that namespace. + +```mermaid +sequenceDiagram + participant App as User Worker + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Proxy as Remote-bindings proxy + participant CF as Cloudflare Artifacts + + MF->>Plugin: getBindings() with remote: true + Plugin-->>MF: Bind to remote proxy service + MF->>Proxy: Start remote proxy Worker + App->>Proxy: env.REPOS method call + Proxy->>CF: Remote binding request + CF-->>Proxy: Result + Proxy-->>App: Result +``` + +A local binding does not automatically synchronize with production Artifacts. +An explicit repository import may contact its specified HTTPS Git remote from +the developer's host, including private-network hosts; it does not follow HTTP +redirects. This is distinct from enabling a remote Artifacts binding. Do not +pass an untrusted URL to `import()` when the host can access sensitive services. diff --git a/packages/miniflare/src/plugins/artifacts/controller.ts b/packages/miniflare/src/plugins/artifacts/controller.ts new file mode 100644 index 00000000000..0b1b75050bc --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/controller.ts @@ -0,0 +1,189 @@ +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import getPort from "get-port"; +import type { Log } from "../../shared"; +import type { GitSidecar } from "./git-sidecar"; + +export interface LocalArtifactsBackend { + sidecar: GitSidecar; + port: number; +} + +// Node binding proxies make synchronous calls. The Git backend must run on +// another thread, otherwise those calls prevent it from answering workerd. +export class ArtifactsController { + #backends = new Map>(); + // Keep the old roots alive until workerd accepts the replacement config. + #activeRoots = new Set(); + #pendingRoots?: Set; + + constructor(private readonly getLog?: () => Pick) {} + + beginUpdate(): void { + if (this.#pendingRoots) { + throw new Error("Artifacts runtime update already in progress"); + } + this.#pendingRoots = new Set(); + } + + async commitUpdate(): Promise { + const roots = this.#pendingRoots; + if (!roots) { + throw new Error("No Artifacts runtime update to commit"); + } + this.#pendingRoots = undefined; + this.#activeRoots = roots; + await this.#closeExcept(roots); + } + + async abortUpdate(): Promise { + if (!this.#pendingRoots) { + return; + } + this.#pendingRoots = undefined; + await this.#closeExcept(this.#activeRoots); + } + + get(root: string): Promise { + root = path.resolve(root); + if (!this.#pendingRoots) { + throw new Error("Artifacts backend requested outside runtime update"); + } + this.#pendingRoots.add(root); + let backend = this.#backends.get(root); + if (!backend) { + backend = this.#start(root, () => { + if (this.#backends.get(root) === backend) { + this.#backends.delete(root); + } + }); + this.#backends.set(root, backend); + void backend.catch(() => { + if (this.#backends.get(root) === backend) { + this.#backends.delete(root); + } + }); + } + return backend; + } + + async #start( + root: string, + onExit: () => void + ): Promise { + const port = await getPort({ host: "127.0.0.1" }); + // __dirname is dist/src in the bundled Miniflare entry point. + const worker = new Worker( + path.join(__dirname, "plugins/artifacts/git-sidecar.js"), + { workerData: { root } } + ); + try { + const endpoint = await waitForSidecarReady(worker); + let stopping = false; + worker.on("error", () => { + this.getLog?.().warn("Local Artifacts Git backend failed"); + }); + worker.once("exit", (code) => { + onExit(); + if (!stopping) { + this.getLog?.().warn( + `Local Artifacts Git backend exited (${code}). Restart the dev server or reload its configuration to reconnect.` + ); + } + }); + return { + port, + sidecar: { + ...endpoint, + async close() { + if (worker.threadId === -1) { + return; + } + stopping = true; + // `events.once()` rejects on a preceding error, even if the thread exits. + const exited = new Promise((resolve) => { + const done = () => { + worker.off("exit", done); + worker.off("error", done); + resolve(); + }; + worker.once("exit", done); + worker.once("error", done); + }); + worker.postMessage("close"); + await exited; + }, + }, + }; + } catch (error) { + await worker.terminate(); + throw error; + } + } + + async #closeExcept(roots: Set): Promise { + const retired = [...this.#backends].filter(([root]) => !roots.has(root)); + for (const [root] of retired) { + this.#backends.delete(root); + } + const results = await Promise.allSettled( + retired.map(([, backend]) => + backend.then( + ({ sidecar }) => sidecar.close(), + () => undefined + ) + ) + ); + for (const result of results) { + if (result.status === "rejected") { + this.getLog?.().warn("Unable to stop a retired Artifacts Git backend"); + } + } + } + + async dispose(): Promise { + this.#pendingRoots = undefined; + this.#activeRoots.clear(); + await this.#closeExcept(this.#activeRoots); + } +} + +type SidecarStartupMessage = + | Pick + | { error: string }; + +function waitForSidecarReady( + worker: Worker +): Promise> { + return new Promise((resolve, reject) => { + function cleanup() { + worker.off("message", ready); + worker.off("error", failed); + worker.off("exit", exited); + } + function ready(value: SidecarStartupMessage) { + cleanup(); + if ("error" in value) { + reject( + new Error( + `Local Artifacts Git backend failed to start: ${value.error}` + ) + ); + } else { + resolve(value); + } + } + function failed(error: Error) { + cleanup(); + reject(error); + } + function exited(code: number) { + failed( + new Error(`Local Artifacts Git backend exited during startup (${code})`) + ); + } + worker.on("message", ready); + worker.on("error", failed); + worker.on("exit", exited); + }); +} diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts index 84e80277047..a87871b001e 100644 --- a/packages/miniflare/src/plugins/artifacts/git-client.ts +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -1,5 +1,5 @@ import { Buffer } from "node:buffer"; -import { spawn } from "node:child_process"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { lstat, mkdtemp, open, rename, unlink } from "node:fs/promises"; import { devNull, tmpdir } from "node:os"; @@ -12,6 +12,7 @@ type GitOptions = { allowFailure?: boolean; timeout?: number; httpAuthorization?: { origin: string; value: string }; + signal?: AbortSignal; }; type GitResult = { stdout: Buffer; stderr: Buffer; status: number }; @@ -60,6 +61,35 @@ export function gitEnvironment( }; } +/** + * Stop a Git invocation's process tree so HTTPS helpers cannot keep its pipes open. + * POSIX invocations own a process group; Windows requires taskkill /T. + */ +export function stopGitProcess(child: ChildProcessWithoutNullStreams): void { + if (process.platform === "win32" && child.pid !== undefined) { + // Git for Windows can spawn git-remote-https; killing only git leaves it alive. + const killer = spawn("taskkill", ["/PID", String(child.pid), "/T", "/F"], { + windowsHide: true, + stdio: "ignore", + }); + killer.once("error", () => child.kill()); + killer.unref(); + } else if (child.pid !== undefined) { + try { + process.kill(-child.pid, "SIGTERM"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") { + child.kill(); + } + } + } else { + child.kill(); + } + child.stdin.destroy(); + child.stdout.destroy(); + child.stderr.destroy(); +} + /** Require Git 2.32's GIT_CONFIG_GLOBAL to isolate native Git subprocesses. */ export function assertSupportedGitVersion(output: string): void { const match = /^git version (\d+)\.(\d+)(?:\.\d+)?(?:[.+ -][^\r\n]*)?$/.exec( @@ -123,18 +153,37 @@ export async function runGit( const child = spawn("git", gitArgumentsWithoutHooks(args), { env, stdio: ["pipe", "pipe", "pipe"], + detached: process.platform !== "win32", + signal: options.signal, }); + if (options.signal?.aborted) { + stopGitProcess(child); + } else { + options.signal?.addEventListener("abort", stop, { once: true }); + } + function stop() { + stopGitProcess(child); + } const stdout: Buffer[] = []; const stderr: Buffer[] = []; - const timeout = setTimeout(() => child.kill(), options.timeout ?? 300_000); + const timeout = setTimeout( + () => stopGitProcess(child), + options.timeout ?? 300_000 + ); timeout.unref(); child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); child.once("error", (error) => { + // Node emits AbortError before other abort listeners can stop Git's + // process group. Keep that listener until the abort finishes. + if ((error as NodeJS.ErrnoException).code !== "ABORT_ERR") { + options.signal?.removeEventListener("abort", stop); + } clearTimeout(timeout); reject(error); }); child.once("close", (status) => { + options.signal?.removeEventListener("abort", stop); clearTimeout(timeout); const result = { stdout: Buffer.concat(stdout), @@ -188,6 +237,15 @@ export interface GitTreeEntry { type GitObjectType = "blob" | "tree" | "commit" | "tag"; +export class LocalRepositoryStorageError extends Error { + constructor( + readonly kind: "creation-lock" | "on-disk-conflict", + message: string + ) { + super(message); + } +} + /** Only one host process may publish a repository at this path at a time. */ export async function withRepositoryCreationLock( repository: string, @@ -206,7 +264,8 @@ export async function withRepositoryCreationLock( lock = await open(lockPath, "wx", 0o600); } catch (error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") { - throw new Error( + throw new LocalRepositoryStorageError( + "creation-lock", `Another local process is creating this Git repository. Stop it and retry; if it crashed, remove "${lockPath}" only after confirming no process uses this storage path.` ); } @@ -222,19 +281,28 @@ export async function withRepositoryCreationLock( /** Repository-scoped Git operations; clone/init are supported before it exists. */ export class GitClient { - constructor(readonly repository: string) {} + constructor( + readonly repository: string, + private readonly signal?: AbortSignal + ) {} + + /** Create a client for another repository with the same cancellation signal. */ + withRepository(repository: string): GitClient { + return new GitClient(repository, this.signal); + } private git(args: string[], options?: GitOptions): Promise { - return runGit(["-C", this.repository, ...args], options); + return runGit(["-C", this.repository, ...args], { + ...options, + signal: this.signal, + }); } async init(defaultBranch: string): Promise { - await runGit([ - "init", - "--bare", - `--initial-branch=${defaultBranch}`, - this.repository, - ]); + await runGit( + ["init", "--bare", `--initial-branch=${defaultBranch}`, this.repository], + { signal: this.signal } + ); } async forkFrom( @@ -245,6 +313,7 @@ export class GitClient { const sourceBranch = `refs/heads/${branch}`; const validBranch = await runGit(["check-ref-format", sourceBranch], { allowFailure: true, + signal: this.signal, }); if (validBranch.status !== 0) { throw new Error("Git fork source branch is invalid"); @@ -261,16 +330,19 @@ export class GitClient { { allowFailure: true } ); if (defaultBranchOnly && sourceHasBranch.status === 0) { - await runGit([ - "clone", - "--bare", - "--local", - "--single-branch", - "--branch", - branch, - source.repository, - this.repository, - ]); + await runGit( + [ + "clone", + "--bare", + "--local", + "--single-branch", + "--branch", + branch, + source.repository, + this.repository, + ], + { signal: this.signal } + ); } else if (defaultBranchOnly) { // Only an unborn HEAD on the requested branch permits an empty fork. // A typo or a missing source must never create a repository instead. @@ -285,13 +357,10 @@ export class GitClient { } await this.init(branch); } else { - await runGit([ - "clone", - "--bare", - "--local", - source.repository, - this.repository, - ]); + await runGit( + ["clone", "--bare", "--local", source.repository, this.repository], + { signal: this.signal } + ); } } @@ -334,10 +403,17 @@ export class GitClient { } args.push(source.url, staged); try { - await runGit(args, { httpAuthorization: source.authorization }); + await runGit(args, { + httpAuthorization: source.authorization, + signal: this.signal, + }); // A bare clone records remote.origin.url in its local Git config. It may // include credentials used only for the import. Never retain the remote. - await new GitClient(staged).git(["remote", "remove", "origin"]); + await new GitClient(staged, this.signal).git([ + "remote", + "remove", + "origin", + ]); await assertImportTargetAvailable(this.repository); await rename(staged, this.repository); } catch (error) { diff --git a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts new file mode 100644 index 00000000000..c0196087542 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts @@ -0,0 +1,737 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { mkdir, mkdtemp, readdir, rename, stat } from "node:fs/promises"; +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import path from "node:path"; +import { Transform } from "node:stream"; +import { parentPort, workerData } from "node:worker_threads"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { z } from "zod"; +import { + assertGitAvailable, + GitClient, + gitArgumentsWithoutHooks, + gitEnvironment, + LocalRepositoryStorageError, + stopGitProcess, + withRepositoryCreationLock, +} from "./git-client"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "./storage"; +import type { RepositoryState } from "./git-client"; + +const adminPath = "/__local_artifacts__"; +const componentPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +type RepositoryLocks = Map>; +type InFlightGitRequest = { + path: string; + generation: string; + abort: () => void; +}; +const decoder = new TextDecoder(); + +if (parentPort) { + const port = parentPort; + void startGitSidecar(workerData.root).then( + (sidecar) => { + port.postMessage({ address: sidecar.address, secret: sidecar.secret }); + port.once("message", () => { + void sidecar.close().then( + () => port.close(), + () => port.close() + ); + }); + }, + (error: unknown) => { + port.postMessage({ + error: + error instanceof Error + ? error.message + : "Unable to start local Artifacts Git backend", + }); + port.close(); + } + ); +} + +export interface GitSidecar { + address: string; + secret: string; + close(): Promise; +} + +const adminBase = z.object({ + namespace: z.string().regex(componentPattern), + name: z.string().regex(componentPattern), +}); +const generatedRepo = adminBase.extend({ generation: z.string().min(1) }); +const AdminRequestSchema = z.discriminatedUnion("action", [ + generatedRepo.extend({ + action: z.literal("create"), + defaultBranch: z.string().optional(), + }), + generatedRepo.extend({ action: z.literal("delete") }), + generatedRepo.extend({ + action: z.literal("fork"), + sourceName: z.string().regex(componentPattern), + sourceBranch: z.string().min(1), + defaultBranchOnly: z.boolean().optional(), + }), + generatedRepo.extend({ + action: z.literal("import"), + sourceUrl: z.string().min(1), + branch: z.string().optional(), + depth: z.number().optional(), + }), + generatedRepo.extend({ action: z.literal("refs") }), + generatedRepo.extend({ + action: z.literal("readBlob"), + hash: z.string().min(1), + }), + generatedRepo.extend({ + action: z.literal("readTree"), + hash: z.string().min(1), + }), + generatedRepo.extend({ + action: z.literal("readCommit"), + hash: z.string().min(1), + }), + generatedRepo.extend({ + action: z.literal("file"), + ref: z.string().min(1), + path: z.string().min(1), + }), + generatedRepo.extend({ + action: z.literal("log"), + ref: z.string().optional(), + limit: z.number().optional(), + offset: z.number().optional(), + }), + generatedRepo.extend({ + action: z.literal("abortPush"), + requestId: z.uuid(), + }), +]); + +type AdminRequest = z.infer; +type GitAction = Exclude; +type CreateRequest = Extract; +type ForkRequest = Extract; +type ImportRequest = Extract; +type NewRepositoryRequest = CreateRequest | ForkRequest | ImportRequest; + +function createGitSecret(): string { + return randomBytes(32).toString("hex"); +} + +export async function startGitSidecar(root: string): Promise { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + await assertGitAvailable(); + await cleanupInterruptedRepositories(root); + const secret = createGitSecret(); + const shutdown = new AbortController(); + const repositoryLocks: RepositoryLocks = new Map(); + const inFlight = new Map(); + const server = createServer((request, response) => { + if (request.headers["x-local-artifacts-backend"] !== secret) { + response.writeHead(403).end("Forbidden"); + return; + } + void handleRequest( + root, + repositoryLocks, + inFlight, + request, + response, + shutdown.signal + ).catch((error: unknown) => { + if (response.headersSent) { + response.destroy(error instanceof Error ? error : undefined); + } else { + const storageError = error instanceof LocalRepositoryStorageError; + response.writeHead(storageError ? 409 : 500, { + "Content-Type": "text/plain", + ...(storageError + ? { "X-Local-Artifacts-Storage-Error": error.kind } + : {}), + }); + response.end( + error instanceof Error ? error.message : "Native Git backend failed" + ); + } + }); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + // The listener outlives this function; ArtifactsController closes it on disposal. + // `using` here would close it before workerd could make any requests. + server.unref(); + const addressInfo = server.address(); + if (!addressInfo || typeof addressInfo === "string") { + throw new Error("Unable to start native Git backend"); + } + const address = `127.0.0.1:${addressInfo.port}`; + return { + address, + secret, + close() { + shutdown.abort(); + server.closeAllConnections(); + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +function parseGitPath(pathname: string): { + namespace: string; + repository: string; + suffix: string; +} | null { + const match = pathname.match(/^\/git\/([^/]+)\/([^/]+\.git)(\/.*)?$/); + if (!match?.[1] || !match[2]) { + return null; + } + const namespace = decodeURIComponent(match[1]); + const repository = decodeURIComponent(match[2].slice(0, -4)); + validateComponent(namespace, "namespace"); + validateComponent(repository, "repository"); + return { namespace, repository, suffix: match[3] ?? "" }; +} + +// Native Git's CGI backend needs streaming Node requests and responses; keep +// packfiles on node:http rather than buffering them through a Worker router. +async function handleRequest( + root: string, + locks: RepositoryLocks, + inFlight: Map, + request: IncomingMessage, + response: ServerResponse, + signal: AbortSignal +): Promise { + const url = new URL(request.url ?? "/", "http://local-artifacts.invalid"); + if (url.pathname === adminPath) { + await handleAdmin(root, locks, inFlight, request, response, signal); + return; + } + const gitPath = parseGitPath(url.pathname); + if (!gitPath) { + response.writeHead(404).end("Not found"); + return; + } + const { namespace, repository, suffix } = gitPath; + const contentLength = Number(request.headers["content-length"] ?? 0); + if (!Number.isFinite(contentLength) || contentLength > 256 * 1024 * 1024) { + response.writeHead(413).end("Request body too large"); + return; + } + const path = repositoryPath(root, namespace, repository); + await withRepositoryLocks(locks, [path], async () => { + const generation = request.headers["x-local-artifacts-generation"]; + const configured = await new GitClient(path, signal).generation(); + if (typeof generation !== "string" || generation !== configured) { + response.writeHead(409).end("Repository generation changed"); + return; + } + const requestId = request.headers["x-local-artifacts-request-id"]; + const trackPush = + request.method === "POST" && + suffix === "/git-receive-pack" && + typeof requestId === "string" && + /^[0-9a-f-]{36}$/.test(requestId); + try { + await runGitHttpBackend( + root, + request, + response, + `/${repositoryDirectory(namespace, repository)}${suffix}`, + url.search.slice(1), + trackPush + ? (abort) => + inFlight.set(requestId, { + path, + generation, + abort, + }) + : undefined, + signal + ); + } finally { + if (trackPush) { + inFlight.delete(requestId); + } + } + }); +} + +async function handleAdmin( + root: string, + locks: RepositoryLocks, + inFlight: Map, + request: IncomingMessage, + response: ServerResponse, + signal: AbortSignal +): Promise { + if (request.method !== "POST") { + response.writeHead(405).end("Method not allowed"); + return; + } + const body = AdminRequestSchema.parse( + JSON.parse(decoder.decode(await readRequest(request))) + ); + const repository = repositoryPath(root, body.namespace, body.name); + if (body.action === "abortPush") { + const active = inFlight.get(body.requestId); + if (active?.path === repository && active.generation === body.generation) { + active.abort(); + } + response.writeHead(200, { "Content-Type": "application/json" }); + response.end("true"); + return; + } + const lockPaths = [repository]; + if (body.action === "fork") { + lockPaths.push(repositoryPath(root, body.namespace, body.sourceName)); + } + await withRepositoryLocks(locks, lockPaths, async () => { + const result = await handleAction( + root, + body, + new GitClient(repository, signal) + ); + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(result)); + }); +} + +async function handleAction( + root: string, + body: GitAction, + git: GitClient +): Promise { + // This check runs under the repository lock: a delayed read must not see + // a different generation after the Worker checked its handle ID. + if ( + body.action !== "create" && + body.action !== "delete" && + body.action !== "fork" && + body.action !== "import" && + (await git.generation()) !== body.generation + ) { + throw new Error("Repository generation changed"); + } + switch (body.action) { + case "create": + return handleCreate(root, body, git); + case "delete": + if (!(await exists(git.repository))) { + return true; + } + // A retried tombstone must never delete a replacement repository. + // An unreadable ownership marker must not clear the tombstone either. + const generation = await git.generation(); + if (generation === null) { + throw new Error("Unable to verify Git repository ownership"); + } + if (generation === body.generation) { + await retireRepository(git.repository); + } + return true; + case "fork": + return handleFork(root, body, git); + case "import": + return handleImport(root, body, git); + case "refs": + return git.state(); + case "readBlob": + return git.readBlob(body.hash); + case "readTree": + return git.readTree(body.hash); + case "readCommit": + return git.readCommit(body.hash); + case "file": + return git.readFile(body.ref, body.path); + case "log": + return git.log(body.ref, body.limit, body.offset); + } +} + +function handleCreate( + root: string, + body: CreateRequest, + git: GitClient +): Promise { + return withNewRepository(root, body, git, (staged) => + staged.init(body.defaultBranch ?? "main") + ); +} + +function handleFork( + root: string, + body: ForkRequest, + git: GitClient +): Promise { + const { sourceName, sourceBranch } = body; + const source = git.withRepository( + repositoryPath(root, body.namespace, sourceName) + ); + return withNewRepository(root, body, git, (staged) => + staged.forkFrom(source, sourceBranch, body.defaultBranchOnly ?? true) + ); +} + +function handleImport( + root: string, + body: ImportRequest, + git: GitClient +): Promise { + const { sourceUrl } = body; + return withNewRepository(root, body, git, (staged) => + staged.importFrom(sourceUrl, body.branch, body.depth) + ); +} + +async function withNewRepository( + root: string, + body: NewRepositoryRequest, + git: GitClient, + initialize: (staged: GitClient) => Promise +): Promise { + await mkdir(root, { recursive: true }); + return withRepositoryCreationLock(git.repository, async () => { + if (await exists(git.repository)) { + // The namespace has no record for this name. A prior deletion may have + // committed metadata before the process stopped removing Git data. + // Never remove a directory that was not created by this simulator. + if ((await git.generation()) === null) { + throw new LocalRepositoryStorageError( + "on-disk-conflict", + `Repository "${body.name}" already exists on disk` + ); + } + await retireRepository(git.repository); + } + // All initialization and cleanup happens under an owned staging directory. + // A failed clone or fork must not remove a repository created concurrently + // at the final path by another host process. + const stagingDirectory = await mkdtemp( + path.join(root, ".artifacts-stage-") + ); + const staged = git.withRepository(path.join(stagingDirectory, "repo.git")); + try { + await initialize(staged); + await staged.configure(body.generation); + const state = await staged.state(); + if (await exists(git.repository)) { + throw new LocalRepositoryStorageError( + "on-disk-conflict", + `Repository "${body.name}" already exists on disk` + ); + } + await rename(staged.repository, git.repository); + return state; + } finally { + await removeDir(stagingDirectory); + } + }); +} + +// Moving the directory out of its active path is atomic on the same volume. +// A crash while removing its contents leaves only an inaccessible trash path; +// the next sidecar startup retries cleanup without blocking name reuse. +async function retireRepository(repository: string): Promise { + const retired = `${repository}.deleting-${randomBytes(16).toString("hex")}`; + try { + await rename(repository, retired); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + await removeDir(retired); +} + +async function cleanupInterruptedRepositories(root: string): Promise { + const retired = /^[0-9a-f]{32}\.git\.deleting-[0-9a-f]{32}$/; + const partialImport = /^\.artifacts-import-[a-zA-Z0-9]{6}$/; + const partialCreation = /^\.artifacts-stage-[a-zA-Z0-9]{6}$/; + // Only staging and retired directories are safe to remove. A creation + // lock may still be held by another process, so never clear it here. + for (const entry of await readdir(root, { withFileTypes: true })) { + if ( + entry.isDirectory() && + (retired.test(entry.name) || + partialImport.test(entry.name) || + partialCreation.test(entry.name)) + ) { + await removeDir(path.join(root, entry.name)); + } + } +} + +function parseGitBackendHeaders(headerBytes: Buffer): { + status: number; + headers: Record; +} { + const lines = decoder.decode(headerBytes).split(/\r?\n/); + let status = 200; + const headers: Record = {}; + for (const line of lines) { + const separator = line.indexOf(":"); + if (separator < 0) { + continue; + } + const name = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + if (name.toLowerCase() === "status") { + status = Number(value.slice(0, 3)); + } else { + headers[name] = value; + } + } + return { status, headers }; +} + +async function runGitHttpBackend( + root: string, + request: IncomingMessage, + response: ServerResponse, + pathInfo: string, + query: string, + registerAbort?: (abort: () => void) => void, + signal?: AbortSignal +): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", gitArgumentsWithoutHooks(["http-backend"]), { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: root, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: pathInfo, + QUERY_STRING: query, + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_CONTENT_ENCODING: String( + request.headers["content-encoding"] ?? "" + ), + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + REMOTE_USER: "local-artifacts", + REMOTE_ADDR: request.socket.remoteAddress ?? "127.0.0.1", + }, + stdio: ["pipe", "pipe", "pipe"], + detached: process.platform !== "win32", + signal, + }); + const stop = () => stopGitProcess(child); + if (signal?.aborted) { + stop(); + } else { + signal?.addEventListener("abort", stop, { once: true }); + } + let header = Buffer.alloc(0); + let headersSent = false; + let settled = false; + let timedOut = false; + let uploadTooLarge = false; + const timeout = setTimeout(() => { + timedOut = true; + stop(); + }, 300_000); + timeout.unref(); + const stderr: Buffer[] = []; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + signal?.removeEventListener("abort", stop); + request.off("aborted", abort); + response.off("close", abort); + if (error) { + reject(error); + } else { + resolve(); + } + }; + const abort = () => { + if (!response.writableEnded) { + stop(); + } + }; + registerAbort?.(() => { + stop(); + response.destroy(); + }); + const write = (chunk: Buffer) => { + if (!response.write(chunk)) { + child.stdout.pause(); + response.once("drain", () => child.stdout.resume()); + } + }; + request.once("aborted", abort); + response.once("close", abort); + child.stdin.on("error", (error) => { + if (!request.destroyed) { + finish(error); + } + }); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.stdout.on("data", (chunk: Buffer) => { + if (headersSent) { + write(chunk); + return; + } + header = Buffer.concat([header, chunk]); + const crlf = header.indexOf("\r\n\r\n"); + const lf = header.indexOf("\n\n"); + const index = crlf >= 0 ? crlf : lf; + if (index < 0) { + if (header.length > 64 * 1024) { + stop(); + } + return; + } + const separatorLength = crlf >= 0 ? 4 : 2; + const { status, headers } = parseGitBackendHeaders( + header.subarray(0, index) + ); + response.writeHead(status, headers); + headersSent = true; + const remaining = header.subarray(index + separatorLength); + if (remaining.length) { + write(remaining); + } + header = Buffer.alloc(0); + }); + child.stdout.on("end", () => { + if (headersSent) { + response.end(); + } + }); + child.once("error", finish); + child.once("close", (status) => { + if (uploadTooLarge) { + finish(); + return; + } + if (timedOut) { + finish(new Error("git http-backend timed out")); + return; + } + if (!headersSent) { + finish( + new Error( + decoder.decode(Buffer.concat(stderr)).trim() || + `git http-backend exited with status ${status}` + ) + ); + } else { + finish(); + } + }); + const upload = limitGitUpload(256 * 1024 * 1024, () => { + uploadTooLarge = true; + if (!response.headersSent) { + response.writeHead(413).end("Request body too large"); + } + stop(); + }); + upload.on("error", () => { + request.unpipe(upload); + child.stdin.destroy(); + }); + request.pipe(upload).pipe(child.stdin); + }); +} + +export function limitGitUpload( + maxBytes: number, + onLimit: () => void +): Transform { + let received = 0; + return new Transform({ + transform(chunk: Buffer, _encoding, callback) { + received += chunk.length; + if (received > maxBytes) { + onLimit(); + callback(new Error("Git upload exceeds the request size limit")); + } else { + callback(null, chunk); + } + }, + }); +} + +function readRequest(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => resolve(Buffer.concat(chunks))); + request.once("error", reject); + }); +} + +function validateComponent(value: string, type: string): void { + if (!componentPattern.test(value)) { + throw new Error(`Invalid ${type}`); + } +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +} + +async function withRepositoryLocks( + repositoryLocks: RepositoryLocks, + paths: string[], + action: () => Promise +): Promise { + const keys = [...new Set(paths)].sort(); + const previous = keys.map( + (key) => repositoryLocks.get(key) ?? Promise.resolve() + ); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + for (const key of keys) { + repositoryLocks.set(key, current); + } + await Promise.all(previous); + try { + return await action(); + } finally { + release(); + for (const key of keys) { + if (repositoryLocks.get(key) === current) { + repositoryLocks.delete(key); + } + } + } +} diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 48242920859..e39ce376c81 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -1,31 +1,87 @@ +import { createHash } from "node:crypto"; +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import SCRIPT from "worker:artifacts/binding"; import { buildRemoteProxyProps, getEnvBindingsOfType, + getPersistPath, getRemoteProxyConnectionString, ProxyNodeBinding, remoteProxyClientWorker, } from "../shared"; -import type { Plugin } from "../shared"; +import { assertGitAvailable } from "./git-client"; +import { assertSupportedGitLayout } from "./storage"; +import type { Service, Socket } from "../../runtime"; +import type { MiniflareBinding, Plugin } from "../shared"; +import type { ArtifactsController } from "./controller"; + +export { ArtifactsController, type LocalArtifactsBackend } from "./controller"; +export type { GitSidecar } from "./git-sidecar"; export const ARTIFACTS_PLUGIN_NAME = "artifacts"; -// One shared remote-proxy service for every artifacts binding; per-binding -// config travels via props. const ARTIFACTS_REMOTE_SERVICE_NAME = `${ARTIFACTS_PLUGIN_NAME}:remote`; +const ENTRYPOINT = "LocalArtifactsNamespace"; +const OBJECT_CLASS = "LocalArtifactsNamespaceObject"; +const LOCAL_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +type ArtifactsBinding = Extract; +type ArtifactsEntry = [name: string, binding: ArtifactsBinding]; + +function isLocalArtifactsBinding(binding: ArtifactsBinding): boolean { + return binding.dev?.remote === false; +} + +function getLocalNamespaceNames(bindings: ArtifactsEntry[]): Set { + return new Set( + bindings + .filter(([, binding]) => isLocalArtifactsBinding(binding)) + .map(([, binding]) => binding.namespace) + ); +} + +function validateLocalNamespace(namespace: string): void { + if (!LOCAL_NAMESPACE_PATTERN.test(namespace)) { + throw new Error(`Invalid local Artifacts namespace: ${namespace}`); + } +} + +function namespaceId(namespace: string): string { + // 128 bits keeps storage paths short enough for workerd's SQLite files on Windows. + return createHash("sha256").update(namespace).digest("hex").slice(0, 32); +} + +function localServiceName(namespace: string): string { + return `${ARTIFACTS_PLUGIN_NAME}-${namespaceId(namespace)}`; +} export const ARTIFACTS_PLUGIN: Plugin = { bindingTypeDescription: "Artifacts", - async getBindings(options) { + getBindings(options) { return getEnvBindingsOfType(options.config, "artifacts").map( - ([name, binding]) => ({ - name, - service: { - name: ARTIFACTS_REMOTE_SERVICE_NAME, - props: buildRemoteProxyProps( - getRemoteProxyConnectionString(binding, options.dev), - name - ), - }, - }) + ([name, binding]) => { + // In this stage, only explicit remote: false opts into local storage. + if (isLocalArtifactsBinding(binding)) { + validateLocalNamespace(binding.namespace); + return { + name, + service: { + name: localServiceName(binding.namespace), + entrypoint: ENTRYPOINT, + }, + }; + } + return { + name, + service: { + name: ARTIFACTS_REMOTE_SERVICE_NAME, + props: buildRemoteProxyProps( + getRemoteProxyConnectionString(binding, options.dev), + name + ), + }, + }; + } ); }, getNodeBindings(options) { @@ -36,16 +92,106 @@ export const ARTIFACTS_PLUGIN: Plugin = { ]) ); }, - async getServices({ options }) { - if (getEnvBindingsOfType(options.config, "artifacts").length === 0) { - return []; - } - - return [ - { + async getServices({ options, sharedOptions, tmpPath, artifactsController }) { + const bindings = getEnvBindingsOfType(options.config, "artifacts"); + const services: Service[] = []; + const sockets: Socket[] = []; + if (bindings.some(([, binding]) => !isLocalArtifactsBinding(binding))) { + services.push({ name: ARTIFACTS_REMOTE_SERVICE_NAME, worker: remoteProxyClientWorker(), - }, - ]; + }); + } + const namespaces = getLocalNamespaceNames(bindings); + if (namespaces.size > 0) { + await assertGitAvailable(); + } + const root = getPersistPath( + ARTIFACTS_PLUGIN_NAME, + tmpPath, + sharedOptions.resourcePersistencePath + ); + + for (const namespace of namespaces) { + const local = await createLocalNamespaceRuntime( + root, + namespace, + artifactsController + ); + sockets.push(local.socket); + services.push(...local.services); + } + return { services, extensions: [], sockets }; }, }; + +async function createLocalNamespaceRuntime( + root: string, + namespace: string, + artifactsController: ArtifactsController +): Promise<{ services: Service[]; socket: Socket }> { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + const persistPath = path.join(root, namespaceId(namespace)); + const metadataPath = path.join(persistPath, "metadata").replaceAll("\\", "/"); + await mkdir(metadataPath, { recursive: true }); + await assertSupportedGitLayout(metadataPath); + const { sidecar, port } = await artifactsController.get( + path.join(persistPath, "git") + ); + const serviceName = localServiceName(namespace); + const storageName = `${serviceName}-storage`; + const backendName = `${serviceName}-backend`; + const socket: Socket = { + name: `${serviceName}-git`, + address: `127.0.0.1:${port}`, + http: {}, + service: { name: serviceName, entrypoint: ENTRYPOINT }, + }; + const storageService: Service = { + name: storageName, + disk: { path: metadataPath, writable: true }, + }; + const gitBackendService: Service = { + name: backendName, + external: { + address: sidecar.address, + http: { + injectRequestHeaders: [ + { name: "X-Local-Artifacts-Backend", value: sidecar.secret }, + ], + }, + }, + }; + const bindingService: Service = { + name: serviceName, + worker: { + compatibilityDate: "2026-09-03", + modules: [{ name: "binding.worker.js", esModule: SCRIPT() }], + bindings: [ + { + name: "config", + json: JSON.stringify({ + namespace, + origin: `http://127.0.0.1:${port}`, + }), + }, + { + name: "localArtifactsNamespace", + durableObjectNamespace: { className: OBJECT_CLASS }, + }, + { name: "gitBackend", service: { name: backendName } }, + ], + // Keys must be unique across namespaces in one workerd, but should + // not repeat the long service prefix in SQLite's directory name. + durableObjectNamespaces: [ + { className: OBJECT_CLASS, uniqueKey: namespaceId(namespace) }, + ], + durableObjectStorage: { localDisk: storageName }, + }, + }; + return { + services: [storageService, gitBackendService, bindingService], + socket, + }; +} diff --git a/packages/miniflare/src/plugins/artifacts/storage.ts b/packages/miniflare/src/plugins/artifacts/storage.ts new file mode 100644 index 00000000000..ac693dc1bcc --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/storage.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { readdir } from "node:fs/promises"; +import path from "node:path"; + +/** + * Keep repository paths short: native Git adds long pack and lock filenames. + * Hash the tuple rather than nesting namespace and repository hashes. Each + * identifier retains 128 bits and no user input becomes a path component. + */ +export function repositoryDirectory(namespace: string, name: string): string { + const id = createHash("sha256") + .update(JSON.stringify([namespace, name])) + .digest("hex") + .slice(0, 32); + return `${id}.git`; +} + +export function repositoryPath( + root: string, + namespace: string, + name: string +): string { + return path.join(root, repositoryDirectory(namespace, name)); +} + +/** Do not quietly abandon repositories written by earlier unreleased layouts. */ +export async function assertSupportedGitLayout(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + if ( + entries.some( + (entry) => + entry.isDirectory() && + (/^[0-9a-f]{64}$/.test(entry.name) || /^artifacts[-:]/.test(entry.name)) + ) + ) { + throw new Error( + "Local Artifacts storage uses an older draft layout. Back up your local repositories, then reset the Artifacts persistence directory before restarting." + ); + } +} diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 9c0658a0c3b..578f1a1deca 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -12,6 +12,7 @@ import type { Extension, Service, ServiceDesignator, + Socket, Worker_Binding, Worker_Module, } from "../../runtime"; @@ -21,6 +22,7 @@ import type { QueueConsumerSchema, QueueProducerSchema, } from "../../workers"; +import type { ArtifactsController } from "../artifacts/controller"; import type { ContainerPrivilegesCache } from "../core/container"; import type { DOContainerOptions } from "../do"; import type { HyperdriveProxyController } from "../hyperdrive/hyperdrive-proxy"; @@ -88,11 +90,14 @@ export interface PluginServicesOptions { devRegistryEnabled: boolean; containerPrivilegesCache: ContainerPrivilegesCache; hyperdriveProxyController: HyperdriveProxyController; + artifactsController: ArtifactsController; } export interface ServicesExtensions { services: Service[]; extensions: Extension[]; + // Internal plugin listeners (e.g. Git smart HTTP), deduplicated by name. + sockets?: Socket[]; } /** diff --git a/packages/miniflare/src/workers/artifacts/binding.worker.ts b/packages/miniflare/src/workers/artifacts/binding.worker.ts new file mode 100644 index 00000000000..97aeb8c36e4 --- /dev/null +++ b/packages/miniflare/src/workers/artifacts/binding.worker.ts @@ -0,0 +1,1385 @@ +import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; + +const repoName = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +type TokenScope = "read" | "write"; + +type RepositorySource = + | { type: "fork"; repository: string } + | { type: "remote"; url: string }; + +interface RepositoryOptions { + description?: string | undefined; + readOnly?: boolean; + setDefaultBranch?: string; +} + +interface ForkOptions { + description?: string; + readOnly?: boolean; + defaultBranchOnly?: boolean; +} + +interface ImportRepositoryOptions { + source: { url: string; branch?: string; depth?: number }; + target: { name: string; opts?: { description?: string; readOnly?: boolean } }; +} + +interface RepositoryToken { + id: string; + tokenHash: string; + scope: TokenScope; + createdAt: string; + expiresAt: string; + revokedAt?: string; +} + +interface CreatedToken { + record: RepositoryToken; + plaintext: string; +} + +interface Repository { + id: string; + name: string; + namespace: string; + description?: string | undefined; + readOnly: boolean; + defaultBranch: string; + remote: string; + source?: RepositorySource | undefined; + createdAt: string; + updatedAt: string; + lastPushAt?: string | undefined; + // A failed native deletion stays retryable without exposing the repository. + deleting?: boolean; + tokens: RepositoryToken[]; +} + +interface RepositoryMetadata { + id: string; + name: string; + remote: string; + defaultBranch: string; + description: string | null; + source: string | null; + readOnly: boolean; + createdAt: string; + updatedAt: string; + lastPushAt: string | null; +} + +interface CreateRepositoryResult { + id: string; + name: string; + description: string | null; + defaultBranch: string; + remote: string; + token: string; +} + +interface CreateTokenResult { + id: string; + plaintext: string; + scope: TokenScope; + expiresAt: string; +} +type TokenInfo = Omit & { + state: "active" | "expired" | "revoked"; +}; + +interface RepositoryListOptions { + cursor?: string; + limit?: number; +} + +type RepositoryListResult = { + repos: Array< + RepositoryMetadata & { status: "ready"; jurisdiction: "unrestricted" } + >; + total: number; + cursor?: string; +}; + +interface RepositoryLogOptions { + ref?: string; + limit?: number; + offset?: number; +} + +interface BackendState { + defaultBranch: string; + refs: Record; +} + +type BackendRequest = + | { action: "create"; defaultBranch: string } + | { action: "delete" } + | { + action: "fork"; + sourceName: string; + sourceBranch: string; + defaultBranchOnly?: boolean; + } + | { action: "import"; sourceUrl: string; branch?: string; depth?: number } + | { action: "refs" } + | { action: "abortPush"; requestId: string } + | { action: "readBlob"; hash: string } + | { action: "readTree"; hash: string } + | { action: "readCommit"; hash: string } + | { action: "file"; ref: string; path: string } + | { action: "log"; ref?: string; limit?: number; offset?: number }; + +interface TreeEntry { + name: string; + mode: string; + hash: string; + type: "tree" | "blob" | "symlink" | "gitlink" | "exec"; +} + +interface CommitMetadata { + hash: string; + treeHash: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + parents: string[]; + authoredAt: number; + committedAt: number; +} + +interface LocalArtifactsRepo { + info(): Promise; + createToken(scope?: TokenScope, ttl?: number): Promise; + listTokens(): Promise<{ total: number; tokens: TokenInfo[] }>; + revokeToken(tokenOrId: string): Promise; + readBlob(hash: string): Promise; + readTree(hash: string): Promise; + readCommit(hash: string): Promise; + readFile(args: { ref: string; path: string }): Promise; + log(options?: RepositoryLogOptions): Promise; + fork(name: string, options?: ForkOptions): Promise; +} + +interface BackendBlob { + data: string; +} + +type ArtifactsErrorCode = + | "ALREADY_EXISTS" + | "NOT_FOUND" + | "INVALID_INPUT" + | "INVALID_REPO_NAME" + | "INVALID_TTL" + | "INVALID_URL" + | "REMOTE_AUTH_REQUIRED" + | "MEMORY_LIMIT" + | "UPSTREAM_UNAVAILABLE" + | "INTERNAL_ERROR"; + +const numericCodes: Record = { + INVALID_INPUT: 10100, + INVALID_REPO_NAME: 10101, + INVALID_TTL: 10103, + INVALID_URL: 10104, + REMOTE_AUTH_REQUIRED: 10106, + NOT_FOUND: 10200, + ALREADY_EXISTS: 10201, + INTERNAL_ERROR: 10400, + UPSTREAM_UNAVAILABLE: 10401, + MEMORY_LIMIT: 10402, +}; + +class ArtifactsError extends Error { + readonly code: ArtifactsErrorCode; + readonly numericCode: number; + + constructor(code: ArtifactsErrorCode, message: string) { + super(message); + this.name = "ArtifactsError"; + this.code = code; + this.numericCode = numericCodes[code]; + } +} + +interface LocalArtifactsEnvironment { + config: { namespace: string; origin: string }; + localArtifactsNamespace: DurableObjectNamespace; + gitBackend: Fetcher; +} + +type RepositoryMap = Record; + +export class LocalArtifactsNamespace extends WorkerEntrypoint { + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return publicCall(() => this.#namespace().create(name, options)); + } + + async get(name: string): Promise { + const repository = await publicCall(() => this.#namespace().get(name)); + return repository as unknown as LocalArtifactsRepo; + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return publicCall(() => this.#namespace().import(options)); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + return publicCall(() => this.#namespace().list(options)); + } + + async delete(name: string): Promise { + return publicCall(() => this.#namespace().delete(name)); + } + + override async fetch(request: Request): Promise { + return this.#namespace().fetch(request); + } + + #namespace(): DurableObjectStub { + const { localArtifactsNamespace, config } = this.env; + return localArtifactsNamespace.get( + localArtifactsNamespace.idFromName(config.namespace) + ); + } +} + +export class LocalArtifactsNamespaceObject extends DurableObject { + // The resolved tail lets the first mutation run immediately. exclusive() + // serializes subsequent metadata and backend mutations across awaits. + private operations = Promise.resolve(); + + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + await this.finishDeletion(repositories, name); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + + const repository = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + try { + const state = await this.backend(repository, { + action: "create", + defaultBranch: repository.defaultBranch, + }); + applyBackendState(repository, state); + repositories[name.toLowerCase()] = repository; + const token = await createToken(repository, "write"); + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist Artifacts repository" + ); + } + return createRepoResult(repository, token); + } catch (error) { + if (error instanceof ArtifactsError) { + throw error; + } + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to create Artifacts repository" + ); + } + }); + } + + async get(name: string): Promise { + validateName(name); + const repository = await this.getRepo(name); + return createRepoHandle(this, repository.name, repository.id); + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return this.exclusive(async () => { + const { source, target, sourceUrl } = parseImportRequest(options); + const repositories = await this.repos(); + await this.finishDeletion(repositories, target.name); + if (hasRepository(repositories, target.name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${target.name}` + ); + } + validateRepositoryOptions(target.opts ?? {}); + const repository = makeRepo( + target.name, + this.env.config.namespace, + this.env.config.origin, + target.opts ?? {} + ); + // The clone URL may contain one-time credentials. Keep them out of + // repository metadata returned by info() and persisted in storage. + const publicSourceUrl = new URL(sourceUrl); + publicSourceUrl.username = ""; + publicSourceUrl.password = ""; + publicSourceUrl.search = ""; + publicSourceUrl.hash = ""; + repository.source = { type: "remote", url: publicSourceUrl.href }; + const state = await this.backend(repository, { + action: "import", + sourceUrl: sourceUrl.href, + ...(source.branch === undefined ? {} : { branch: source.branch }), + ...(source.depth === undefined ? {} : { depth: source.depth }), + }); + applyBackendState(repository, state); + repository.updatedAt = new Date().toISOString(); + const token = await createToken(repository, "write"); + repositories[repository.name.toLowerCase()] = repository; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist imported Artifacts repository" + ); + } + return createRepoResult(repository, token); + }); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + const limit = options.limit ?? 50; + if (!Number.isInteger(limit) || limit < 1 || limit > 200) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 200." + ); + } + const offset = decodeCursor(options.cursor); + const repositories = Object.values(await this.repos()) + .filter((repository) => !repository.deleting) + .sort((left, right) => right.createdAt.localeCompare(left.createdAt)); + const page = repositories + .slice(offset, offset + limit) + .map((repository) => ({ + ...repoMetadata(repository), + status: "ready" as const, + // Local storage has no placement jurisdiction; this is the public default. + jurisdiction: "unrestricted" as const, + })); + const cursor = + offset + page.length < repositories.length + ? encodeCursor(offset + page.length) + : undefined; + return { + repos: page, + total: repositories.length, + ...(cursor === undefined ? {} : { cursor }), + }; + } + + async delete(name: string): Promise { + return this.exclusive(async () => { + validateName(name); + const repositories = await this.repos(); + const key = name.toLowerCase(); + if (!Object.hasOwn(repositories, key)) { + return false; + } + const repository = repositories[key]; + // Persist a tombstone before Git cleanup. A storage failure leaves the + // live repository intact; a backend failure remains retryable by delete + // or the next creation of this name, including after a process restart. + if (!repository.deleting) { + repository.deleting = true; + await this.saveRepos(repositories); + } + await this.finishDeletion(repositories, name); + return true; + }); + } + + async createToken( + name: string, + scope: TokenScope = "write", + ttl?: number, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const token = await createToken(repository, scope, ttl); + await this.saveRepos(repositories); + return tokenResult(token); + }); + } + + async listTokens( + name: string, + expectedId?: string + ): Promise<{ total: number; tokens: TokenInfo[] }> { + const repository = await this.getRepo(name, expectedId); + const active = repository.tokens.filter( + (token) => !token.revokedAt && isActive(token) + ); + const tokens = active + .toReversed() + .slice(0, 30) + .map(({ tokenHash: _tokenHash, revokedAt: _revokedAt, ...token }) => ({ + ...token, + state: "active" as const, + })); + return { total: active.length, tokens }; + } + + async revokeToken( + name: string, + tokenOrId: string, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + if (!isTokenId(tokenOrId) && !isArtifactToken(tokenOrId)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid tokenOrId: must be a token id or artifact token." + ); + } + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const tokenHash = isArtifactToken(tokenOrId) + ? await hashToken(tokenOrId) + : undefined; + const token = repository.tokens.find( + (candidate) => + candidate.id === tokenOrId || + (tokenHash !== undefined && + constantTimeEqual(candidate.tokenHash, tokenHash)) + ); + if (!token || token.revokedAt) { + return false; + } + token.revokedAt = new Date().toISOString(); + await this.saveRepos(repositories); + return true; + }); + } + + async fork( + sourceName: string, + name: string, + options: ForkOptions = {}, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + const source = getRepo(repositories, sourceName, expectedId); + await this.finishDeletion(repositories, name); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + const fork = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + let state: BackendState; + try { + state = await this.backend(fork, { + action: "fork", + sourceName: source.name, + sourceBranch: source.defaultBranch, + ...(options.defaultBranchOnly === undefined + ? {} + : { defaultBranchOnly: options.defaultBranchOnly }), + }); + } catch { + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to fork Artifacts repository" + ); + } + applyBackendState(fork, state); + fork.source = { type: "fork", repository: source.name }; + const token = await createToken(fork, "write"); + repositories[name.toLowerCase()] = fork; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(fork); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist forked Artifacts repository" + ); + } + return createRepoResult(fork, token); + }); + } + + override async fetch(request: Request): Promise { + return handleGitRequest( + this, + this.env.config.namespace, + request, + (promise) => this.ctx.waitUntil(promise) + ); + } + + async repos(): Promise { + const repositories = + (await this.ctx.storage.get("repos")) ?? + (Object.create(null) as RepositoryMap); + // The local Git listener can move after a process restart. + for (const repository of Object.values(repositories)) { + repository.remote = `${this.env.config.origin}/git/${repository.namespace}/${repository.name}.git`; + } + return repositories; + } + + saveRepos(repositories: RepositoryMap): Promise { + for (const repository of Object.values(repositories)) { + repository.tokens = repository.tokens.filter( + (token) => !token.revokedAt && isActive(token) + ); + } + return this.ctx.storage.put("repos", repositories); + } + + private async finishDeletion( + repositories: RepositoryMap, + name: string + ): Promise { + const key = name.toLowerCase(); + if (!Object.hasOwn(repositories, key)) { + return; + } + const repository = repositories[key]; + if (!repository.deleting) { + return; + } + await this.backend(repository, { action: "delete" }); + delete repositories[key]; + await this.saveRepos(repositories); + } + + async getRepo(name: string, expectedId?: string): Promise { + return getRepo(await this.repos(), name, expectedId); + } + + async backend( + repository: Repository, + request: BackendRequest + ): Promise { + const response = await this.env.gitBackend.fetch( + "http://local-artifacts-backend/__local_artifacts__", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ...request, + namespace: repository.namespace, + name: repository.name, + generation: repository.id, + }), + } + ); + if (!response.ok) { + const message = await response.text(); + const storageError = response.headers.get( + "X-Local-Artifacts-Storage-Error" + ); + if ( + response.status === 409 && + (storageError === "creation-lock" || + storageError === "on-disk-conflict") && + (request.action === "create" || + request.action === "fork" || + request.action === "import") + ) { + // Only the sidecar's typed local-storage failures carry safe diagnostics. + throw new ArtifactsError("INTERNAL_ERROR", message); + } + if (request.action === "import") { + throw classifyImportError(message); + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } + return response.json(); + } + + private async deleteBackend(repository: Repository): Promise { + try { + await this.backend(repository, { action: "delete" }); + } catch { + // Preserve the original persistence error; native delete is idempotent on retry. + } + } + + gitFetch( + repository: Repository, + request: Request, + requestId?: string + ): Promise { + const headers = new Headers(request.headers); + headers.delete("Authorization"); + headers.delete("X-Local-Artifacts-Request-Id"); + headers.set("X-Local-Artifacts-Generation", repository.id); + if (requestId) { + headers.set("X-Local-Artifacts-Request-Id", requestId); + } + const url = new URL(request.url); + url.pathname = url.pathname.replace( + /^\/git\/[^/]+\/[^/]+\.git/, + `/git/${repository.namespace}/${repository.name}.git` + ); + return this.env.gitBackend.fetch( + new Request(url, new Request(request, { headers })) + ); + } + + async syncAfterPush( + name: string, + id: string, + previousRefs: Record + ): Promise { + await this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name, id); + const state = await this.backend(repository, { + action: "refs", + }); + if (JSON.stringify(state.refs) !== JSON.stringify(previousRefs)) { + repository.updatedAt = new Date().toISOString(); + repository.lastPushAt = repository.updatedAt; + } + await this.saveRepos(repositories); + }); + } + + private async exclusive(action: () => Promise): Promise { + const previous = this.operations; + let release!: () => void; + this.operations = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await action(); + } finally { + release(); + } + } +} + +function createRepoHandle( + namespace: LocalArtifactsNamespaceObject, + name: string, + id: string +): LocalArtifactsRepo { + const read = (request: BackendRequest): Promise => + publicCall(async () => + namespace.backend(await namespace.getRepo(name, id), request) + ); + const readFile = async (args: { + ref: string; + path: string; + }): Promise => { + if (!args?.ref) { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: ref required."); + } + if (!args?.path) { + throw new ArtifactsError("INVALID_INPUT", "Invalid path: path required."); + } + const path = normalizePath(args.path); + if (path === null) { + return null; + } + const result = await read({ + action: "file", + ref: args.ref, + path, + }); + if (result === null) { + return null; + } + const bytes = decodeBase64(result.data); + return new Blob([bytes], { type: contentType(args.path, bytes) }); + }; + + return { + info: () => + publicCall(async () => repoMetadata(await namespace.getRepo(name, id))), + createToken: (scope: TokenScope = "write", ttl?: number) => + publicCall(() => namespace.createToken(name, scope, ttl, id)), + listTokens: () => publicCall(() => namespace.listTokens(name, id)), + revokeToken: (tokenOrId: string) => + publicCall(() => namespace.revokeToken(name, tokenOrId, id)), + readBlob: async (hash: string) => { + validateHash(hash); + const result = await read({ + action: "readBlob", + hash, + }); + return result === null ? null : new Blob([decodeBase64(result.data)]); + }, + readTree: async (hash: string) => { + validateHash(hash); + return await read({ action: "readTree", hash }); + }, + readCommit: async (hash: string) => { + validateHash(hash); + return await read({ action: "readCommit", hash }); + }, + readFile, + log: (options: RepositoryLogOptions = {}) => { + validateLogOptions(options); + return read({ + action: "log", + ...(options.ref === undefined ? {} : { ref: options.ref }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.offset === undefined ? {} : { offset: options.offset }), + }); + }, + fork: (targetName: string, options: ForkOptions = {}) => + publicCall(() => namespace.fork(name, targetName, options, id)), + }; +} + +function parseGitRoute( + request: Request, + namespace: string +): { name: string; write: boolean; push: boolean } | null { + const url = new URL(request.url); + const match = url.pathname.match(/^\/git\/([^/]+)\/([^/]+)\.git(?:\/(.*))?$/); + if (!match || match[1] !== namespace || !match[2]) { + return null; + } + const path = match[3] ?? ""; + const service = url.searchParams.get("service"); + const discovery = + request.method === "GET" && + path === "info/refs" && + (service === "git-upload-pack" || service === "git-receive-pack"); + const pack = + request.method === "POST" && + (path === "git-upload-pack" || path === "git-receive-pack"); + if (!discovery && !pack) { + return null; + } + return { + name: match[2], + write: path === "git-receive-pack" || service === "git-receive-pack", + push: request.method === "POST" && path === "git-receive-pack", + }; +} + +async function handleGitRequest( + state: LocalArtifactsNamespaceObject, + namespace: string, + request: Request, + keepAlive: (promise: Promise) => void +): Promise { + const route = parseGitRoute(request, namespace); + if (!route) { + return new Response("Not found", { status: 404 }); + } + let repository: Repository; + try { + repository = await state.getRepo(route.name); + } catch (error) { + if (error instanceof ArtifactsError && error.code === "NOT_FOUND") { + return new Response("Not found", { status: 404 }); + } + throw error; + } + const authorizationError = await authorize( + repository, + request, + route.write ? "write" : "read" + ); + if (authorizationError) { + return authorizationError; + } + if (route.write && repository.readOnly) { + return new Response("Artifacts repository is read-only", { status: 403 }); + } + + const previousRefs = route.push + ? (await state.backend(repository, { action: "refs" })).refs + : undefined; + const requestId = route.push ? crypto.randomUUID() : undefined; + const response = await state.gitFetch(repository, request, requestId); + if (!previousRefs || !requestId) { + return response; + } + + if (!response.ok) { + return response; + } + const syncRefs = async () => { + try { + await state.syncAfterPush(repository.name, repository.id, previousRefs); + } catch { + // Git has already accepted the push; metadata bookkeeping must not fail the client request. + } + }; + if (!response.body) { + await syncRefs(); + return response; + } + // A canceled response may leave the sidecar blocked on its output socket. + // Stop only this push, then synchronize after its repository lock releases. + const reader = response.body.getReader(); + let canceled = false; + let synchronization: Promise | undefined; + const syncOnce = () => (synchronization ??= syncRefs()); + const body = new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + await syncOnce(); + if (!canceled) { + controller.close(); + } + } else if (!canceled) { + controller.enqueue(value); + } + } catch (error) { + keepAlive(syncOnce()); + if (!canceled) { + controller.error(error); + } + } + }, + cancel() { + canceled = true; + keepAlive( + (async () => { + try { + await state.backend(repository, { action: "abortPush", requestId }); + } catch { + // The sidecar may already have finished or stopped. + } + void reader.cancel().catch(() => {}); + await syncOnce(); + })() + ); + }, + }); + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function authorize( + repository: Repository, + request: Request, + scope: TokenScope +): Promise { + const header = request.headers.get("Authorization") ?? ""; + let value = ""; + if (header.startsWith("Bearer ")) { + value = header.slice(7); + } else if (header.startsWith("Basic ")) { + try { + value = atob(header.slice(6)).split(":").slice(1).join(":"); + } catch { + value = ""; + } + } + const valueHash = await hashToken(value); + const token = repository.tokens.find( + (candidate) => + !candidate.revokedAt && + constantTimeEqual(candidate.tokenHash, valueHash) && + isActive(candidate) + ); + if (!token) { + return new Response("Invalid Artifacts token", { + status: 401, + headers: { "WWW-Authenticate": 'Basic realm="Artifacts"' }, + }); + } + if (scope === "write" && token.scope !== "write") { + return new Response("Artifacts token requires write scope", { + status: 403, + }); + } + return undefined; +} + +function applyBackendState(repository: Repository, state: BackendState): void { + repository.defaultBranch = state.defaultBranch; +} + +function makeRepo( + name: string, + namespace: string, + origin: string, + options: RepositoryOptions +): Repository { + const time = new Date().toISOString(); + return { + id: randomBase36(16), + name, + namespace, + description: options.description, + readOnly: Boolean(options.readOnly), + defaultBranch: options.setDefaultBranch ?? "main", + remote: `${origin}/git/${namespace}/${name}.git`, + createdAt: time, + updatedAt: time, + lastPushAt: undefined, + tokens: [], + }; +} + +function repoMetadata(repository: Repository): RepositoryMetadata { + return { + id: repository.id, + name: repository.name, + remote: repository.remote, + defaultBranch: repository.defaultBranch, + description: repository.description ?? null, + source: + repository.source === undefined + ? null + : repository.source.type === "fork" + ? `artifacts:${repository.namespace}/${repository.source.repository}` + : `git:${repository.source.url}`, + readOnly: repository.readOnly, + createdAt: repository.createdAt, + updatedAt: repository.updatedAt, + lastPushAt: repository.lastPushAt ?? null, + }; +} + +function createRepoResult( + repository: Repository, + token: CreatedToken +): CreateRepositoryResult { + return { + id: repository.id, + name: repository.name, + description: repository.description ?? null, + defaultBranch: repository.defaultBranch, + remote: repository.remote, + token: token.plaintext, + }; +} + +function tokenResult(token: CreatedToken): CreateTokenResult { + return { + id: token.record.id, + plaintext: token.plaintext, + scope: token.record.scope, + expiresAt: token.record.expiresAt, + }; +} + +function getRepo( + repositories: RepositoryMap, + name: string, + expectedId?: string +): Repository { + const key = name.toLowerCase(); + if (!Object.hasOwn(repositories, key)) { + throw new ArtifactsError("NOT_FOUND", `Repository not found: ${name}.`); + } + const repository = repositories[key]; + if ( + repository.deleting || + (expectedId !== undefined && repository.id !== expectedId) + ) { + throw new ArtifactsError("NOT_FOUND", `Repository not found: ${name}.`); + } + return repository; +} + +function hasRepository(repositories: RepositoryMap, name: string): boolean { + return Object.hasOwn(repositories, name.toLowerCase()); +} + +function parseImportRequest(options: ImportRepositoryOptions): { + source: ImportRepositoryOptions["source"]; + target: ImportRepositoryOptions["target"]; + sourceUrl: URL; +} { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts import options are required" + ); + } + const { source, target } = options; + validateName(target?.name); + let sourceUrl: URL; + try { + sourceUrl = new URL(source?.url); + } catch { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be a valid URL." + ); + } + if (sourceUrl.protocol !== "https:") { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be an HTTPS URL." + ); + } + if ( + source.depth !== undefined && + (!Number.isSafeInteger(source.depth) || source.depth < 1) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.depth: must be a positive integer." + ); + } + return { source, target, sourceUrl }; +} + +function validateName(name: unknown): asserts name is string { + if (typeof name !== "string" || !repoName.test(name)) { + throw new ArtifactsError("INVALID_REPO_NAME", "Invalid repo name."); + } +} + +function validateRepositoryOptions( + options: RepositoryOptions | ForkOptions +): void { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository options must be an object" + ); + } + if ( + options.description !== undefined && + typeof options.description !== "string" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository description must be a string" + ); + } + if (options.readOnly !== undefined && typeof options.readOnly !== "boolean") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository readOnly must be a boolean" + ); + } + if ("setDefaultBranch" in options && options.setDefaultBranch !== undefined) { + validateBranch(options.setDefaultBranch); + } + if ( + "defaultBranchOnly" in options && + options.defaultBranchOnly !== undefined && + typeof options.defaultBranchOnly !== "boolean" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts defaultBranchOnly must be a boolean" + ); + } +} + +function validateBranch(branch: unknown): asserts branch is string { + if ( + typeof branch !== "string" || + branch.length < 1 || + branch.length > 256 || + branch.startsWith("-") || + branch.startsWith(".") || + branch.endsWith("/") || + branch.endsWith(".") || + branch.endsWith(".lock") || + branch === "@" || + branch.includes("..") || + branch.includes("//") || + branch.includes("@{") || + branch.split("/").some((segment) => segment.startsWith(".")) || + [...branch].some((character) => character.charCodeAt(0) <= 32) || + /[~^:?*[\\]/.test(branch) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts default branch is invalid" + ); + } +} + +function validateLogOptions( + options: unknown +): asserts options is RepositoryLogOptions { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts log options must be an object" + ); + } + const { ref, limit, offset } = options as RepositoryLogOptions; + if (ref !== undefined && typeof ref !== "string") { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: must be a string."); + } + if ( + limit !== undefined && + (!Number.isInteger(limit) || limit < 1 || limit > 1000) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 1000." + ); + } + if (offset !== undefined && (!Number.isInteger(offset) || offset < 0)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid offset: must be a non-negative integer." + ); + } +} + +async function createToken( + repository: Repository, + scope: TokenScope = "write", + ttl = 86_400 +): Promise { + if (scope !== "read" && scope !== "write") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts token scope must be read or write" + ); + } + if (!Number.isInteger(ttl) || ttl < 60 || ttl > 31_536_000) { + throw new ArtifactsError( + "INVALID_TTL", + `Invalid TTL ${ttl}: must be between 60 and 31536000 seconds.` + ); + } + const createdAt = new Date(); + const expiresAt = new Date(createdAt.getTime() + ttl * 1000); + const plaintext = `art_v2_x_${randomHex(20)}?expires=${Math.floor(expiresAt.getTime() / 1000)}`; + const record = { + id: randomBase36(16), + tokenHash: await hashToken(plaintext), + scope, + createdAt: createdAt.toISOString(), + expiresAt: expiresAt.toISOString(), + }; + repository.tokens.push(record); + return { record, plaintext }; +} + +function isActive(token: Pick): boolean { + return Date.parse(token.expiresAt) > Date.now(); +} + +function randomHex(bytes: number): string { + return [...crypto.getRandomValues(new Uint8Array(bytes))] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +async function hashToken(value: string): Promise { + const secret = value.split("?", 1)[0] ?? ""; + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(secret) + ); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function constantTimeEqual(left: string, right: string): boolean { + if (left.length !== right.length) { + return false; + } + let difference = 0; + for (let index = 0; index < left.length; index++) { + difference |= left.charCodeAt(index) ^ right.charCodeAt(index); + } + return difference === 0; +} + +function randomBase36(length: number): string { + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"; + return [...crypto.getRandomValues(new Uint8Array(length))] + .map((byte) => alphabet[byte % alphabet.length]) + .join(""); +} + +function isTokenId(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-z]{16}$/.test(value); +} + +function isArtifactToken(value: unknown): value is string { + return ( + typeof value === "string" && + /^art_v(?:1_[0-9a-f]{40}|2_[xeuf]_[0-9a-f]{40})(?:\?expires=\d+)?$/.test( + value + ) + ); +} + +function validateHash(hash: unknown): asserts hash is string { + if (!hash) { + throw new ArtifactsError("INVALID_INPUT", "Invalid hash: hash required."); + } + if (typeof hash !== "string" || !/^[0-9a-f]{40}$/.test(hash)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid hash: expected a 40-character SHA-1 hex hash." + ); + } +} + +function normalizePath(path: string): string | null { + const segments = path + .split("/") + .filter((segment) => segment !== "" && segment !== "."); + if (segments.includes("..") || segments.length === 0) { + return null; + } + return segments.join("/"); +} + +function contentType(path: string, bytes: Uint8Array): string { + const extension = path.toLowerCase().match(/\.[^./]+$/)?.[0]; + const known: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".ico": "image/x-icon", + ".pdf": "application/pdf", + }; + if (extension && known[extension]) { + return known[extension]; + } + const sample = bytes.subarray(0, 8192); + if (sample.includes(0)) { + return "application/octet-stream"; + } + try { + // A sample may stop midway through a valid multibyte character. + new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(sample, { + stream: sample.length < bytes.length, + }); + return "text/plain;charset=utf-8"; + } catch { + return "application/octet-stream"; + } +} + +function decodeBase64(value: string): Uint8Array { + return Uint8Array.from(atob(value), (character) => character.charCodeAt(0)); +} + +function encodeCursor(offset: number): string { + return btoa(JSON.stringify({ offset })); +} + +function decodeCursor(cursor: string | undefined): number { + if (cursor === undefined) { + return 0; + } + try { + const value = JSON.parse(atob(cursor)) as { offset?: unknown }; + return typeof value.offset === "number" && + Number.isInteger(value.offset) && + value.offset >= 0 + ? value.offset + : 0; + } catch { + return 0; + } +} + +async function publicCall(action: () => Promise): Promise { + try { + return await action(); + } catch (error) { + if (isArtifactsError(error)) { + throw error; + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } +} + +function isArtifactsError(error: unknown): error is ArtifactsError { + return ( + error instanceof Error && + error.name === "ArtifactsError" && + "code" in error && + typeof error.code === "string" && + "numericCode" in error && + typeof error.numericCode === "number" + ); +} + +function classifyImportError(message: string): ArtifactsError { + const normalized = message.toLowerCase(); + if ( + normalized.includes("authentication failed") || + normalized.includes("could not read username") || + normalized.includes("http 401") + ) { + return new ArtifactsError( + "REMOTE_AUTH_REQUIRED", + "Authentication is required to access the remote repository." + ); + } + if ( + normalized.includes("remote branch") && + normalized.includes("not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote branch not found."); + } + if ( + normalized.includes("not found") || + normalized.includes("repository not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote repository not found."); + } + if (normalized.includes("does not appear to be a git repository")) { + return new ArtifactsError( + "INVALID_URL", + "The URL does not point to a git repository." + ); + } + return new ArtifactsError( + "UPSTREAM_UNAVAILABLE", + "The upstream service is unavailable. Please retry." + ); +} diff --git a/packages/miniflare/test/config/v4-convert.spec.ts b/packages/miniflare/test/config/v4-convert.spec.ts index d121d3da45d..66059b90269 100644 --- a/packages/miniflare/test/config/v4-convert.spec.ts +++ b/packages/miniflare/test/config/v4-convert.spec.ts @@ -4,6 +4,28 @@ import { convertV4MiniflareOptions } from "../../src/config/v4-convert"; import type { RemoteProxyConnectionString } from "../../src/plugins/shared"; describe("convertV4MiniflareOptions", () => { + test("Artifacts keeps legacy remote defaults until the local-default switch", ({ + expect, + }) => { + const remoteProxyConnectionString = new URL( + "http://127.0.0.1:8787" + ) as RemoteProxyConnectionString; + const converted = convertV4MiniflareOptions({ + name: "worker", + script: "export default {};", + artifacts: { + LOCAL: { namespace: "local" }, + REMOTE: { namespace: "remote", remoteProxyConnectionString }, + }, + }); + expect(converted.workers[0].config.env?.LOCAL).toEqual({ + type: "artifacts", + namespace: "local", + }); + expect(converted.workers[0].config.env?.REMOTE).toMatchObject({ + dev: { remote: true }, + }); + }); test("preserves opaque K2 stream IDs through v4 parsing and conversion", ({ expect, }) => { diff --git a/packages/miniflare/test/plugins/artifacts/contract.spec.ts b/packages/miniflare/test/plugins/artifacts/contract.spec.ts new file mode 100644 index 00000000000..9694a524959 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract.spec.ts @@ -0,0 +1,163 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { singleModuleManifest, useDispose } from "../../test-shared"; +import { runScenarios } from "./contract/scenarios"; +import type { Created, ArtifactsBinding } from "./contract/scenarios"; + +const exec = promisify(execFile); +const SCRIPT = `export default { fetch() { return new Response("contract"); } };`; + +async function prepareLocalFixture(created: Created): Promise { + const source = await mkdtemp(path.join(tmpdir(), "artifacts-contract-")); + const environment = { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + GIT_AUTHOR_DATE: "2020-01-02T03:04:05Z", + GIT_COMMITTER_DATE: "2020-01-02T03:04:05Z", + }; + const git = async (...args: string[]) => + exec("git", args, { cwd: source, env: environment }); + try { + await git("init", "--initial-branch=main"); + await mkdir(path.join(source, "folder")); + await writeFile(path.join(source, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(source, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await writeFile(path.join(source, "folder", "nested.txt"), "nested\n"); + await git("add", "."); + await git("commit", "-m", "first"); + await writeFile(path.join(source, "second.txt"), "second\n"); + await git("add", "."); + await git("commit", "-m", "second"); + // Keep the token out of argv, stderr, and test output. The child alone sees the header. + const pushEnv = { + ...environment, + GIT_CONFIG_COUNT: "2", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `Authorization: Bearer ${created.token}`, + }; + await exec("git", ["push", created.remote, "main"], { + cwd: source, + env: pushEnv, + }); + await git("checkout", "-b", "feature"); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git("add", "."); + await git("commit", "-m", "feature"); + await exec("git", ["push", created.remote, "feature"], { + cwd: source, + env: pushEnv, + }); + } finally { + await removeDir(source); + } +} + +function local(): Miniflare { + return new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { + type: "artifacts", + namespace: "contract-local", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); +} + +test("shared Artifacts contract runs against the local RPC binding and cleans up", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const observations = await runScenarios( + REPOS, + "contract-test-", + prepareLocalFixture + ); + const find = (step: string) => + observations.find((item) => item.step === step); + expect(observations.length).toBeGreaterThan(50); + expect(find("file binary")).toMatchObject({ + outcome: "value", + value: { + blob: { type: "application/octet-stream", bytes: [0, 255, 128, 1] }, + }, + }); + expect(find("file text")).toMatchObject({ + outcome: "value", + value: { blob: { type: "text/plain;charset=utf-8" } }, + }); + expect(find("duplicate case")).toMatchObject({ outcome: "error" }); + expect(find("retained after delete")).toMatchObject({ outcome: "error" }); + for (const kind of ["blob", "tree", "commit"]) { + expect(find(`${kind} present`)).toMatchObject({ outcome: "value" }); + expect(find(`${kind} missing`)).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find(`${kind} wrong type`)).toMatchObject({ + outcome: "value", + value: null, + }); + } + expect(find("file missing ref")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("file missing path")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("list next page")).toMatchObject({ outcome: "value" }); + expect(find("token revoke wrong repo")).toMatchObject({ + outcome: "value", + value: false, + }); + expect(find("fork all feature")).toMatchObject({ + outcome: "value", + value: { blob: { bytes: [...new TextEncoder().encode("feature\n")] } }, + }); + expect(find("fork main feature")).toMatchObject({ + outcome: "value", + value: null, + }); + expect((await REPOS.list()).total).toBe(0); +}, 90_000); + +test("local contract scenarios refuse a nonempty namespace", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + await REPOS.create("contract-test-main"); + await expect( + runScenarios(REPOS, "contract-test-", prepareLocalFixture) + ).rejects.toThrow(/not empty/); + expect((await REPOS.list()).total).toBe(1); + await REPOS.delete("contract-test-main"); +}); diff --git a/packages/miniflare/test/plugins/artifacts/contract/README.md b/packages/miniflare/test/plugins/artifacts/contract/README.md new file mode 100644 index 00000000000..0037e5f370f --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/README.md @@ -0,0 +1,5 @@ +# Artifacts local contract tests + +`../contract.spec.ts` runs shared scenarios against Miniflare's real RPC binding with a deterministic, locally pushed Git fixture. The scenarios assert returned objects, error fields, Blob MIME types and bytes, missing/null outcomes, repository lifecycle, tokens, Git objects and files, and forks. Git HTTP authorization is tested separately in `../index.spec.ts`. The scenarios refuse a nonempty namespace and clean up repositories they create. + +These tests exercise only local behavior. They do not contact deployed Artifacts or establish parity with it. Comparing against the deployed service requires an approved, exclusively reserved disposable namespace; never run these scenarios against shared or production data. diff --git a/packages/miniflare/test/plugins/artifacts/contract/scenarios.ts b/packages/miniflare/test/plugins/artifacts/contract/scenarios.ts new file mode 100644 index 00000000000..d9a3ea549ff --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/scenarios.ts @@ -0,0 +1,331 @@ +/* Shared observable scenarios. No HTTP RPC shim: calls use the real Worker binding. */ +export interface Repository { + info(): Promise; + createToken( + scope?: "read" | "write", + ttl?: number + ): Promise<{ id: string; plaintext: string }>; + listTokens(): Promise; + revokeToken(tokenOrId: string): Promise; + readBlob(hash: string): Promise; + readTree(hash: string): Promise | null>; + readCommit(hash: string): Promise<{ treeHash: string } | null>; + readFile(args: { ref: string; path: string }): Promise; + log(options?: { + ref?: string; + limit?: number; + offset?: number; + }): Promise>; + fork( + name: string, + options?: { + readOnly?: boolean; + defaultBranchOnly?: boolean; + description?: string; + } + ): Promise; +} + +export interface Created { + id: string; + name: string; + remote: string; + token: string; +} + +export interface ArtifactsBinding { + create( + name: string, + options?: { readOnly?: boolean; description?: string } + ): Promise; + get(name: string): Promise; + list(options?: { limit?: number; cursor?: string }): Promise<{ + repos: Array<{ name: string }>; + total: number; + cursor?: string; + }>; + delete(name: string): Promise; +} + +export type Observation = { + step: string; + outcome: "value" | "error"; + value: unknown; +}; +export type PrepareFixture = (created: Created) => Promise; + +/** Execute a bounded, namespace-scoped set of RPC calls and capture their exact outcomes. */ +export async function runScenarios( + binding: ArtifactsBinding, + prefix: string, + prepareFixture: PrepareFixture +): Promise { + if (!/^[a-z][a-z0-9-]{7,40}-$/.test(prefix)) { + throw new Error( + "Fixture prefix must be lowercase, 8–41 characters, and end with '-'" + ); + } + const names = [ + "main", + "second", + "third", + "fork", + "all", + "readonly", + "reused", + ].map((suffix) => prefix + suffix); + // The namespace must be empty: global list totals/cursors are otherwise affected + // by repositories outside this fixture, and cleanup must never touch them. + const seenCursors = new Set(); + let cursor: string | undefined; + do { + const page = await binding.list({ + limit: 200, + ...(cursor ? { cursor } : {}), + }); + if (page.repos.length || page.total !== 0) { + throw new Error("Disposable fixture namespace is not empty"); + } + cursor = page.cursor; + if (cursor && seenCursors.has(cursor)) { + throw new Error("Fixture preflight pagination repeated a cursor"); + } + if (cursor) { + seenCursors.add(cursor); + } + } while (cursor); + const [main, second, third, fork, all, readonly, reused] = names; + const observations: Observation[] = []; + const owned = new Set(); + const recordCreation = async ( + step: string, + name: string, + call: () => Promise + ): Promise => { + await record(step, async () => { + const created = await call(); + owned.add(name); + return created; + }); + }; + const record = async ( + step: string, + call: () => Promise + ): Promise => { + try { + observations.push({ + step, + outcome: "value", + value: await snapshot(await call()), + }); + } catch (error) { + observations.push({ + step, + outcome: "error", + value: errorSnapshot(error), + }); + } + }; + // Only successfully created fixtures belong to this run. Failed RPC calls + // may have unknown effects; never assume a failed create grants ownership. + let scenarioFailure: { error: unknown } | undefined; + try { + await record("empty list", () => binding.list({ limit: 1 })); + await record("invalid name", async () => + (await binding.get(prefix + "invalid/name")).info() + ); + await record("missing get info", async () => + (await binding.get(prefix + "absent")).info() + ); + await record("missing delete", () => binding.delete(prefix + "absent")); + await recordCreation("create", main, () => + binding.create(main, { description: "contract fixture" }) + ); + await recordCreation("duplicate case", main.toUpperCase(), () => + binding.create(main.toUpperCase()) + ); + await record("case get", async () => + (await binding.get(main.toUpperCase())).info() + ); + await recordCreation("create second", second, () => binding.create(second)); + await recordCreation("create third", third, () => binding.create(third)); + await record("list first page", () => binding.list({ limit: 2 })); + const page = await binding.list({ limit: 2 }); + await record("list next page", () => + binding.list({ limit: 2, cursor: page.cursor }) + ); + await record("list limit zero", () => binding.list({ limit: 0 })); + await record("list limit high", () => binding.list({ limit: 201 })); + await record("list malformed cursor", () => + binding.list({ limit: 1, cursor: "not-base64!" }) + ); + const retained = await binding.get(second); + await record("delete retained", () => binding.delete(second)); + await record("retained after delete", () => retained.info()); + await recordCreation("recreate retained", second, () => + binding.create(second) + ); + await record("retained after recreate", () => retained.info()); + await record("delete case", () => binding.delete(third.toUpperCase())); + await record("delete twice", () => binding.delete(third)); + const repo = await binding.get(main); + await record("token list initial", () => repo.listTokens()); + let readToken: { id: string; plaintext: string } | undefined; + await record( + "token create read", + async () => (readToken = await repo.createToken("read", 60)) + ); + await record("token list read", () => repo.listTokens()); + await record("token ttl low", () => repo.createToken("write", 59)); + await record("token ttl high", () => repo.createToken("write", 31_536_001)); + if (!readToken) { + throw new Error("Could not create fixture read token"); + } + const tokenId = readToken.id; + await record("token revoke wrong repo", async () => + (await binding.get(second)).revokeToken(tokenId) + ); + await record("token revoke", () => repo.revokeToken(tokenId)); + await record("token revoke twice", () => repo.revokeToken(tokenId)); + await record("token list revoked", () => repo.listTokens()); + await record("token invalid id", () => repo.revokeToken("bad")); + await recordCreation("fork empty", fork, async () => + (await binding.get(second)).fork(fork) + ); + await recordCreation("fork duplicate case", fork.toUpperCase(), async () => + (await binding.get(second)).fork(fork.toUpperCase()) + ); + await recordCreation("fork read only", readonly, async () => + (await binding.get(second)).fork(readonly, { readOnly: true }) + ); + await recordCreation("fork missing source", reused, async () => + (await binding.get(prefix + "absent")).fork(reused) + ); + const created = (await repo.info()) as Created; + // The initial create token is only returned by create, not info; prepare via a fresh token. + const write = await repo.createToken("write"); + await prepareFixture({ ...created, token: write.plaintext }); + const log = await repo.log({ ref: "main" }); + await record("log main", () => repo.log({ ref: "main" })); + await record("log page", () => + repo.log({ ref: "main", limit: 1, offset: 1 }) + ); + await record("log invalid limit", () => repo.log({ limit: 0 })); + await record("log missing ref", () => repo.log({ ref: "missing" })); + const hash = log[0]?.hash; + if (!hash) { + throw new Error("Fixture push did not produce a main commit"); + } + const commit = await repo.readCommit(hash); + if (!commit) { + throw new Error("Fixture main commit is missing"); + } + const tree = await repo.readTree(commit.treeHash); + if (!tree) { + throw new Error("Fixture main tree is missing"); + } + const blob = tree.find((entry) => entry.name === "hello.txt")?.hash; + if (!blob) { + throw new Error("Fixture hello.txt blob is missing"); + } + const missingHash = "0".repeat(40); + for (const [kind, read, wrong] of [ + ["blob", (h: string) => repo.readBlob(h), commit.treeHash], + ["tree", (h: string) => repo.readTree(h), blob], + ["commit", (h: string) => repo.readCommit(h), blob], + ] as const) { + await record(`${kind} present`, () => + read(kind === "blob" ? blob : kind === "tree" ? commit.treeHash : hash) + ); + await record(`${kind} missing`, () => read(missingHash)); + await record(`${kind} invalid hash`, () => read("bad")); + await record(`${kind} wrong type`, () => read(wrong)); + } + for (const [label, ref, path] of [ + ["text", "main", "hello.txt"], + ["binary", "main", "bytes.bin"], + ["nested", "main", "folder/nested.txt"], + ["directory", "main", "folder"], + ["missing path", "main", "absent"], + ["traversal", "main", "../hello.txt"], + ["missing ref", "missing", "hello.txt"], + ["commit ref", hash, "hello.txt"], + ] as const) { + await record(`file ${label}`, () => repo.readFile({ ref, path })); + } + await record("file invalid path", () => + repo.readFile({ ref: "main", path: "" }) + ); + await recordCreation("fork main only", reused, () => + repo.fork(reused, { defaultBranchOnly: true }) + ); + await recordCreation("fork all refs", all, () => + repo.fork(all, { defaultBranchOnly: false, description: "all refs" }) + ); + await record("fork main feature", async () => + (await binding.get(reused)).readFile({ + ref: "feature", + path: "feature.txt", + }) + ); + await record("fork all feature", async () => + (await binding.get(all)).readFile({ ref: "feature", path: "feature.txt" }) + ); + } catch (error) { + scenarioFailure = { error }; + } + const cleanupFailures: string[] = []; + for (const name of owned) { + try { + await binding.delete(name); + } catch { + cleanupFailures.push(name); + } + } + if (cleanupFailures.length) { + throw new Error( + `Fixture cleanup failed for: ${cleanupFailures.join(", ")}`, + { cause: scenarioFailure?.error } + ); + } + if (scenarioFailure) { + throw scenarioFailure.error; + } + return observations; +} + +function errorSnapshot(error: unknown): unknown { + if (error === null || typeof error !== "object") { + return { thrown: String(error) }; + } + return Object.fromEntries( + ["name", "message", "code", "numericCode"] + .filter((key) => key in error) + .map((key) => [key, (error as Record)[key]]) + ); +} + +async function snapshot(value: unknown): Promise { + if (value instanceof Blob) { + return { + blob: { + type: value.type, + bytes: [...new Uint8Array(await value.arrayBuffer())], + }, + }; + } + if (Array.isArray(value)) { + return Promise.all(value.map(snapshot)); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + await Promise.all( + Object.entries(value).map(async ([key, item]) => [ + key, + await snapshot(item), + ]) + ) + ); + } + return value === undefined ? { undefined: true } : value; +} diff --git a/packages/miniflare/test/plugins/artifacts/git-client.spec.ts b/packages/miniflare/test/plugins/artifacts/git-client.spec.ts index fd71803c756..2b587bacb73 100644 --- a/packages/miniflare/test/plugins/artifacts/git-client.spec.ts +++ b/packages/miniflare/test/plugins/artifacts/git-client.spec.ts @@ -1,4 +1,5 @@ import { execFile } from "node:child_process"; +import { getEventListeners } from "node:events"; import { chmod, mkdir, @@ -38,6 +39,30 @@ function git(args: string[], cwd?: string, extraEnv: NodeJS.ProcessEnv = {}) { }); } +test("failed Git spawns release the sidecar shutdown listener", async ({ + expect, +}) => { + const emptyPath = await useTmp(); + const previousPath = process.env.PATH; + const shutdown = new AbortController(); + process.env.PATH = emptyPath; + try { + await expect( + runGit(["--version"], { signal: shutdown.signal }) + ).rejects.toMatchObject({ + code: "ENOENT", + }); + await new Promise((resolve) => setImmediate(resolve)); + expect( + getEventListeners(shutdown.signal, "abort").filter( + (listener) => listener.name === "stop" + ) + ).toHaveLength(0); + } finally { + process.env.PATH = previousPath; + } +}); + test("Git subprocesses do not inherit Cloudflare or host credentials", ({ expect, }) => { diff --git a/packages/miniflare/test/plugins/artifacts/https-import.spec.ts b/packages/miniflare/test/plugins/artifacts/https-import.spec.ts new file mode 100644 index 00000000000..9f6357dcdfe --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/https-import.spec.ts @@ -0,0 +1,569 @@ +import { execFile, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:https"; +import path from "node:path"; +import { promisify } from "node:util"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { repositoryPath } from "../../../src/plugins/artifacts/storage"; +import { singleModuleManifest, useTmp } from "../../test-shared"; + +const exec = promisify(execFile); +const WORKER = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name ? await env.REPOS.get(name) : env.REPOS; + const result = await target[method](...args); + if (result instanceof Blob) { + return Response.json({ bytes: [...new Uint8Array(await result.arrayBuffer())] }); + } + return Response.json(result); + } catch (error) { + return Response.json({ error: error.message, code: error.code }, { status: 400 }); + } + } +}; +`; + +type ImportOptions = { + source: { url: string; branch?: string; depth?: number }; + target: { name: string }; +}; + +async function git(args: string[], cwd?: string): Promise { + const { stdout } = await exec("git", args, { + cwd, + env: { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }); + return stdout.trim(); +} + +async function makeSource(directory: string): Promise<{ + first: string; + second: string; + feature: string; +}> { + const source = path.join(directory, "source"); + await git(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "first\n"); + await git(["add", "README"], source); + await git(["commit", "-m", "first"], source); + const first = await git(["rev-parse", "HEAD"], source); + await writeFile(path.join(source, "README"), "second\n"); + await git(["commit", "-am", "second"], source); + const second = await git(["rev-parse", "HEAD"], source); + await git(["checkout", "-b", "feature"], source); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git(["add", "feature.txt"], source); + await git(["commit", "-m", "feature"], source); + const feature = await git(["rev-parse", "HEAD"], source); + const bare = path.join(directory, "source.git"); + await git(["clone", "--bare", source, bare]); + await git(["symbolic-ref", "HEAD", "refs/heads/main"], bare); + return { first, second, feature }; +} + +/** Serve native Git's smart HTTP CGI over a local, self-signed HTTPS origin. */ +async function startHttpsGit(directory: string) { + const keyPath = path.join(directory, "key.pem"); + const certPath = path.join(directory, "cert.pem"); + await exec("openssl", [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + "-keyout", + keyPath, + "-out", + certPath, + ]); + const server = createServer( + { key: await readFile(keyPath), cert: await readFile(certPath) }, + (request, response) => { + const auth = `Basic ${Buffer.from("reader:correct-password").toString("base64")}`; + if (request.headers.authorization !== auth) { + response + .writeHead(401, { "WWW-Authenticate": 'Basic realm="fixture"' }) + .end(); + return; + } + if (request.url?.startsWith("/redirected.git")) { + redirectedRequests++; + response.writeHead(500).end("Unexpected redirected request"); + return; + } + if (redirectTarget && request.url?.startsWith("/source.git/info/refs")) { + response.writeHead(302, { Location: redirectTarget }).end(); + return; + } + if (stallInfoRefs && request.url?.startsWith("/source.git/info/refs")) { + stalledRequests++; + return; + } + if ( + interruptUpload && + request.url?.startsWith("/source.git/git-upload-pack") + ) { + interrupted++; + response.writeHead(200, { + "Content-Type": "application/x-git-upload-pack-result", + }); + response.write("0008NAK\n"); + response.destroy(); + return; + } + const url = new URL(request.url ?? "/", "https://localhost"); + const child = spawn("git", ["http-backend"], { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: directory, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: url.pathname, + QUERY_STRING: url.search.slice(1), + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + }, + stdio: ["pipe", "pipe", "pipe"], + }); + const output: Buffer[] = []; + const errors: Buffer[] = []; + request.pipe(child.stdin); + child.stdout.on("data", (chunk: Buffer) => output.push(chunk)); + child.stderr.on("data", (chunk: Buffer) => errors.push(chunk)); + response.once("close", () => child.kill()); + child.once("error", () => response.destroy()); + child.once("close", (status) => { + if (response.destroyed) { + return; + } + const bytes = Buffer.concat(output); + const separator = bytes.indexOf("\r\n\r\n") >= 0 ? "\r\n\r\n" : "\n\n"; + const end = bytes.indexOf(separator); + if (end < 0) { + response.writeHead(500).end(Buffer.concat(errors)); + return; + } + const headers = bytes.subarray(0, end).toString().split(/\r?\n/); + const statusLine = headers.find((line) => line.startsWith("Status:")); + const statusCode = statusLine + ? Number(statusLine.match(/^Status: (\d{3})/)?.[1]) + : 200; + for (const line of headers) { + const colon = line.indexOf(":"); + if (colon > 0 && !line.startsWith("Status:")) { + response.setHeader( + line.slice(0, colon), + line.slice(colon + 1).trim() + ); + } + } + response.writeHead(status === 0 ? statusCode : 500); + response.end(bytes.subarray(end + separator.length)); + }); + } + ); + let interruptUpload = false; + let interrupted = 0; + let stallInfoRefs = false; + let stalledRequests = 0; + let redirectTarget: string | undefined; + let redirectedRequests = 0; + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("HTTPS fixture did not bind a port"); + } + return { + certPath, + url: `https://localhost:${address.port}/source.git`, + get interrupted() { + return interrupted; + }, + get stalledRequests() { + return stalledRequests; + }, + set stallInfoRefs(value: boolean) { + stallInfoRefs = value; + }, + get redirectedRequests() { + return redirectedRequests; + }, + set redirectTarget(value: string | undefined) { + redirectTarget = value; + }, + set interruptUpload(value: boolean) { + interruptUpload = value; + }, + async close() { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +async function call( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise<{ ok: boolean; body: unknown }> { + const response = await mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); + return { ok: response.ok, body: await response.json() }; +} + +function importOptions( + url: string, + name: string, + branch?: string, + depth?: number +): ImportOptions { + return { + source: { url, ...(branch ? { branch } : {}), ...(depth ? { depth } : {}) }, + target: { name }, + }; +} + +async function withFixture( + callback: (context: { + mf: Miniflare; + fixture: Awaited>; + commits: Awaited>; + directory: string; + }) => Promise, + trust = true +) { + const directory = await useTmp(); + const commits = await makeSource(directory); + const fixture = await startHttpsGit(directory); + const previousCA = process.env.GIT_SSL_CAINFO; + // The controller's sidecar snapshots this environment at startup. Git gets + // this CA only via GIT_SSL_CAINFO; neither Node nor workerd trust is changed. + if (trust) { + process.env.GIT_SSL_CAINFO = fixture.certPath; + } else { + delete process.env.GIT_SSL_CAINFO; + } + let mf: Miniflare | undefined; + try { + mf = new Miniflare({ + cf: false, + resourcePersistencePath: path.join(directory, "persist"), + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { + type: "artifacts", + namespace: "https-import", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(WORKER), + }, + }, + ], + }); + await mf.ready; + restoreCA(previousCA); + await callback({ mf, fixture, commits, directory }); + } finally { + restoreCA(previousCA); + try { + await mf?.dispose(); + } finally { + await fixture.close(); + } + } +} + +function restoreCA(value: string | undefined): void { + if (value === undefined) { + delete process.env.GIT_SSL_CAINFO; + } else { + process.env.GIT_SSL_CAINFO = value; + } +} + +test("HTTPS import: credentials never follow a Git redirect", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + fixture.redirectTarget = fixture.url.replace( + "/source.git", + "/redirected.git" + ); + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const result = await call(mf, "import", [importOptions(url, "redirect")]); + expect(result.ok).toBe(false); + expect(fixture.redirectedRequests).toBe(0); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + }); +}); + +test("HTTPS import: full import exposes refs, file, log and Git clone through the public binding", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, commits, directory }) => { + // Repository-local settings at the HTTPS source are not transferred by + // a bare clone, even when the source itself uses native Git. + await git( + ["config", "core.hooksPath", path.join(directory, "source-hooks")], + path.join(directory, "source.git") + ); + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const imported = await call(mf, "import", [importOptions(url, "full")]); + expect(imported.ok, JSON.stringify(imported.body)).toBe(true); + expect(imported.body).toMatchObject({ + name: "full", + defaultBranch: "main", + }); + const namespaceId = createHash("sha256") + .update("https-import") + .digest("hex") + .slice(0, 32); + const config = await readFile( + path.join( + repositoryPath( + path.join(directory, "persist", "artifacts", namespaceId, "git"), + "https-import", + "full" + ), + "config" + ), + "utf8" + ); + expect(config).not.toContain("correct-password"); + expect(config).not.toContain("source-hooks"); + expect(config).not.toContain('remote "origin"'); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "full")) + .body + ).toEqual({ bytes: [...Buffer.from("second\n")] }); + expect((await call(mf, "log", [], "full")).body).toMatchObject([ + { hash: commits.second }, + { hash: commits.first }, + ]); + expect( + ( + await call( + mf, + "readFile", + [{ ref: "feature", path: "feature.txt" }], + "full" + ) + ).body + ).toEqual({ bytes: [...Buffer.from("feature\n")] }); + const info = (await call(mf, "info", [], "full")).body as { + remote: string; + source: string; + }; + expect(info.source).toBe(`git:${fixture.url}`); + expect(JSON.stringify(info)).not.toContain("correct-password"); + const token = (imported.body as { token: string }).token; + await exec( + "git", + [ + "-c", + `http.extraHeader=Authorization: Bearer ${token}`, + "clone", + info.remote, + path.join(directory, "clone"), + ], + { env: gitEnvironment() } + ); + expect( + await readFile(path.join(directory, "clone", "README"), "utf8") + ).toBe("second\n"); + }); +}); + +test("HTTPS import: depth and branch constrain the imported history and refs", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, commits }) => { + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const result = await call(mf, "import", [ + importOptions(url, "shallow", "feature", 1), + ]); + expect(result.ok, JSON.stringify(result.body)).toBe(true); + expect(result.body).toMatchObject({ defaultBranch: "feature" }); + expect((await call(mf, "log", [], "shallow")).body).toMatchObject([ + { hash: commits.feature }, + ]); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "shallow")) + .body + ).toBeNull(); + }); +}); + +test("HTTPS import: disposal stops a Git clone stalled by the origin", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + fixture.stallInfoRefs = true; + const pending = call(mf, "import", [ + importOptions( + fixture.url.replace("https://", "https://reader:correct-password@"), + "stalled" + ), + ]).catch(() => undefined); + for ( + let attempt = 0; + attempt < 100 && fixture.stalledRequests === 0; + attempt++ + ) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + expect(fixture.stalledRequests).toBeGreaterThan(0); + await expect(mf.dispose()).resolves.toBeUndefined(); + await pending; + }); +}, 15_000); + +test("HTTPS import: Basic URL credentials, missing repos/branches, duplicate targets and retries", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const invalid = fixture.url.replace( + "https://", + "https://reader:wrong-password@" + ); + for (const [url, name] of [ + [fixture.url, "no-auth"], + [invalid, "wrong-auth"], + ]) { + const failure = await call(mf, "import", [importOptions(url, name)]); + expect(failure.ok).toBe(false); + expect(failure.body).toMatchObject({ code: "REMOTE_AUTH_REQUIRED" }); + expect(JSON.stringify(failure.body)).not.toContain("wrong-password"); + } + const missing = await call(mf, "import", [ + importOptions(valid.replace("source.git", "missing.git"), "absent"), + ]); + expect(missing.ok).toBe(false); + expect(missing.body).toMatchObject({ code: "NOT_FOUND" }); + const branch = await call(mf, "import", [ + importOptions(valid, "absent-branch", "nonexistent"), + ]); + expect(branch.ok).toBe(false); + expect(branch.body).toMatchObject({ code: "NOT_FOUND" }); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + expect( + (await call(mf, "import", [importOptions(valid, "absent")])).ok + ).toBe(true); + const duplicate = await call(mf, "import", [ + importOptions(valid, "ABSENT"), + ]); + expect(duplicate.ok).toBe(false); + expect(duplicate.body).toMatchObject({ code: "ALREADY_EXISTS" }); + }); +}); + +test("HTTPS import: untrusted TLS fails closed without a CA", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const failed = await call(mf, "import", [ + importOptions(valid, "tls-failed"), + ]); + expect(failed.ok).toBe(false); + expect(failed.body).toMatchObject({ code: "UPSTREAM_UNAVAILABLE" }); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + }, false); +}); + +test("HTTPS import: interrupted upload cleans partial target and retry succeeds", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, directory }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + fixture.interruptUpload = true; + const failed = await call(mf, "import", [importOptions(valid, "retry")]); + expect(failed.ok).toBe(false); + expect(failed.body).toMatchObject({ code: "UPSTREAM_UNAVAILABLE" }); + expect(fixture.interrupted).toBeGreaterThan(0); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + const [namespaceDirectory] = await readdir( + path.join(directory, "persist", "artifacts") + ); + expect( + await readdir( + path.join(directory, "persist", "artifacts", namespaceDirectory, "git") + ) + ).toEqual([]); + fixture.interruptUpload = false; + expect((await call(mf, "import", [importOptions(valid, "retry")])).ok).toBe( + true + ); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "retry")) + .body + ).toEqual({ bytes: [...Buffer.from("second\n")] }); + }); +}); + +test("Git child only inherits explicit CA trust, never host TLS bypass or credentials", ({ + expect, +}) => { + const env = gitEnvironment({ + PATH: "/usr/bin", + GIT_SSL_CAINFO: "/tmp/explicit-ca.pem", + GIT_SSL_NO_VERIFY: "true", + GIT_ASKPASS: "/tmp/askpass", + GIT_CONFIG_PARAMETERS: "'http.extraHeader=Authorization: secret'", + }); + expect(env.GIT_SSL_CAINFO).toBe("/tmp/explicit-ca.pem"); + expect(env.GIT_SSL_NO_VERIFY).toBeUndefined(); + expect(env.GIT_ASKPASS).toBeUndefined(); + expect(env.GIT_CONFIG_PARAMETERS).toBeUndefined(); +}); diff --git a/packages/miniflare/test/plugins/artifacts/index.spec.ts b/packages/miniflare/test/plugins/artifacts/index.spec.ts new file mode 100644 index 00000000000..7828b5659d1 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/index.spec.ts @@ -0,0 +1,1639 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + chmod, + mkdir, + readFile, + rename, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { Readable, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { ArtifactsController, Miniflare } from "miniflare"; +import { test } from "vitest"; +import { + GitClient, + gitEnvironment, + withRepositoryCreationLock, +} from "../../../src/plugins/artifacts/git-client"; +import { + limitGitUpload, + startGitSidecar, + type GitSidecar, +} from "../../../src/plugins/artifacts/git-sidecar"; +import { repositoryPath } from "../../../src/plugins/artifacts/storage"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; +import type { MiniflareOptions } from "miniflare"; + +const exec = promisify(execFile); +const SCRIPT = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name === undefined ? env.REPOS : await env.REPOS.get(name); + if (method === "keys") return Response.json(Object.keys(target)); + const result = await target[method](...args); + if (result instanceof Blob) { + return Response.json({ bytes: [...new Uint8Array(await result.arrayBuffer())], type: result.type }); + } + return Response.json(result); + } catch (error) { + return Response.json({ error: error.message, name: error.name, code: error.code, numericCode: error.numericCode }, { status: 400 }); + } + } +}; +`; + +function options( + namespace = "test", + binding = "REPOS", + remote: boolean | null = false +): MiniflareOptions { + return { + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + [binding]: { + type: "artifacts", + namespace, + ...(remote === null ? {} : { dev: { remote } }), + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }; +} + +function rpcRequest( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + return mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); +} + +async function rpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +) { + const response = await rpcRequest(mf, method, args, name); + const result = (await response.json()) as any; + if (!response.ok) { + throw new Error(result.error); + } + return result; +} + +async function rpcFailure( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +) { + const response = await rpcRequest(mf, method, args, name); + return { + status: response.status, + ...((await response.json()) as { + error: string; + name: string; + code: string; + numericCode: number; + }), + }; +} + +function git(args: string[], cwd?: string, extraEnv: NodeJS.ProcessEnv = {}) { + return exec("git", args, { + cwd, + env: { + ...gitEnvironment(), + ...extraEnv, + GIT_AUTHOR_NAME: "Miniflare", + GIT_AUTHOR_EMAIL: "miniflare@example.com", + GIT_COMMITTER_NAME: "Miniflare", + GIT_COMMITTER_EMAIL: "miniflare@example.com", + }, + }); +} + +async function pushFixture(mf: Miniflare, directory: string) { + const created = await rpc(mf, "create", ["Repo"]); + await git(["init", "--initial-branch=main", directory]); + await writeFile(path.join(directory, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(directory, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await writeFile( + path.join(directory, "payload.dat"), + new Uint8Array([255, 254, 128]) + ); + await git(["add", "."], directory); + await git(["commit", "-m", "fixture"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "main", + ], + directory + ); + return created; +} + +async function createSidecarRepository( + sidecar: GitSidecar, + generation: string +): Promise { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { + "X-Local-Artifacts-Backend": sidecar.secret, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + action: "create", + namespace: "test", + name: "repo", + generation, + }), + } + ); + if (!response.ok) { + throw new Error(`Sidecar create failed: ${await response.text()}`); + } + await response.body?.cancel(); +} + +test("artifacts: sidecar parses admin actions before modifying repositories", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + const admin = (body: unknown) => + fetch(`http://${sidecar.address}/__local_artifacts__`, { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify(body), + }); + const common = { namespace: "test", name: "repo" }; + try { + for (const body of [ + { ...common, action: "unknown" }, + { ...common, action: "create" }, + { ...common, action: "create", generation: "gen", namespace: "../bad" }, + { ...common, action: "fork", generation: "gen", sourceName: 42 }, + { ...common, action: "import", generation: "gen", sourceUrl: 42 }, + { ...common, action: "delete", generation: null }, + { ...common, action: "abortPush", generation: "gen", requestId: "bad" }, + { ...common, action: "readBlob", hash: null }, + { ...common, action: "file", ref: "main", path: [] }, + ]) { + const response = await admin(body); + expect(response.status, JSON.stringify(body)).toBe(500); + await response.body?.cancel(); + } + const created = await admin({ + ...common, + action: "create", + generation: "gen", + }); + expect(created.status).toBe(200); + await created.body?.cancel(); + const refs = await admin({ ...common, action: "refs", generation: "gen" }); + expect(refs.status).toBe(200); + expect(await refs.json()).toMatchObject({ refs: {} }); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: real RPC exposes current methods, not metadata or legacy methods", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["Repo"]); + const keys = await rpc(mf, "keys", [], "Repo"); + expect(keys).toContain("readFile"); + expect(keys).not.toContain("id"); + expect(keys).not.toContain("name"); + expect(keys).not.toContain("file"); + expect(keys).not.toContain("raw"); + expect( + await rpc(mf, "readFile", [{ ref: "HEAD", path: "missing" }], "Repo") + ).toBeNull(); +}); + +test("artifacts: interrupted deletion leaves a replaceable orphaned generation", async ({ + expect, +}) => { + const root = await useTmp(); + const sidecar = await startGitSidecar(root); + try { + await createSidecarRepository(sidecar, "original"); + const client = new GitClient(repositoryPath(root, "test", "repo")); + expect(await client.generation()).toBe("original"); + // The namespace has committed the deletion, but the process exited + // before the native repository could be removed. + await createSidecarRepository(sidecar, "replacement"); + expect(await client.generation()).toBe("replacement"); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: restart finishes cleanup after an interrupted Git deletion", async ({ + expect, +}) => { + const root = await useTmp(); + const active = repositoryPath(root, "test", "repo"); + const retired = `${active}.deleting-${"a".repeat(32)}`; + const first = await startGitSidecar(root); + try { + await createSidecarRepository(first, "original"); + // Simulate termination after metadata removal and the atomic rename, + // but before the retired directory's contents have been removed. + await rename(active, retired); + } finally { + await first.close(); + } + const restarted = await startGitSidecar(root); + try { + await expect(stat(retired)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(stat(active)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await restarted.close(); + } +}); + +test("artifacts: stale sidecar reads cannot observe a replacement generation", async ({ + expect, +}) => { + const workspace = await useTmp(); + const root = path.join(workspace, "repos"); + const sidecar = await startGitSidecar(root); + try { + await createSidecarRepository(sidecar, "original"); + const source = path.join(workspace, "source"); + await git(["init", "--initial-branch=main", source]); + await writeFile( + path.join(source, "secret.txt"), + "replacement private contents\n" + ); + await git(["add", "secret.txt"], source); + await git(["commit", "-m", "replacement"], source); + const admin = (action: Record) => + fetch(`http://${sidecar.address}/__local_artifacts__`, { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ namespace: "test", name: "repo", ...action }), + }); + const deleted = await admin({ action: "delete", generation: "original" }); + expect(deleted.status).toBe(200); + await deleted.body?.cancel(); + await createSidecarRepository(sidecar, "replacement"); + await git(["push", repositoryPath(root, "test", "repo"), "main"], source); + + // Simulate an old handle's read reaching the sidecar after + // delete/recreate. Its generation must be checked under the lock. + const oldRead = await admin({ + action: "file", + generation: "original", + ref: "main", + path: "secret.txt", + }); + expect(oldRead.status).not.toBe(200); + expect(await oldRead.text()).not.toContain("replacement private contents"); + for (const action of [ + { action: "refs" }, + { action: "readBlob", hash: "HEAD" }, + { action: "readTree", hash: "HEAD" }, + { action: "readCommit", hash: "HEAD" }, + { action: "log", ref: "main" }, + ]) { + const stale = await admin({ ...action, generation: "original" }); + expect(stale.status).not.toBe(200); + await stale.body?.cancel(); + const unpinned = await admin(action); + expect(unpinned.status).toBe(500); + await unpinned.body?.cancel(); + } + const currentRead = await admin({ + action: "file", + generation: "replacement", + ref: "main", + path: "secret.txt", + }); + expect(currentRead.status).toBe(200); + const currentBlob = (await currentRead.json()) as { data: string }; + expect(Buffer.from(currentBlob.data, "base64").toString()).toBe( + "replacement private contents\n" + ); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: local storage conflicts expose safe recovery guidance", async ({ + expect, +}) => { + const persistence = await useTmp(); + const mf = new Miniflare({ + ...options(), + resourcePersistencePath: persistence, + }); + useDispose(mf); + await mf.ready; + const namespaceId = createHash("sha256") + .update("test") + .digest("hex") + .slice(0, 32); + const root = path.join(persistence, "artifacts", namespaceId, "git"); + const locked = repositoryPath(root, "test", "locked"); + await withRepositoryCreationLock(locked, async () => { + const created = await rpcFailure(mf, "create", ["locked"]); + expect(created.code).toBe("INTERNAL_ERROR"); + expect(created.error).toContain("Another local process is creating"); + expect(created.error).toContain(".artifacts-create-"); + const imported = await rpcFailure(mf, "import", [ + { + source: { url: "https://example.invalid/repo.git" }, + target: { name: "locked" }, + }, + ]); + expect(imported.code).toBe("INTERNAL_ERROR"); + expect(imported.error).toContain("Another local process is creating"); + }); + const foreign = repositoryPath(root, "test", "foreign"); + await mkdir(foreign, { recursive: true }); + const created = await rpcFailure(mf, "create", ["foreign"]); + expect(created.code).toBe("INTERNAL_ERROR"); + expect(created.error).toContain( + 'Repository "foreign" already exists on disk' + ); +}); + +test("artifacts: relative and absolute storage paths share one sidecar", async ({ + expect, +}) => { + const controller = new ArtifactsController(); + const absolute = path.join(await useTmp(), "repos"); + const relative = path.relative(process.cwd(), absolute); + try { + controller.beginUpdate(); + const first = await controller.get(relative); + const same = await controller.get(absolute); + expect(same).toBe(first); + await controller.commitUpdate(); + controller.beginUpdate(); + expect(await controller.get(absolute)).toBe(first); + await controller.commitUpdate(); + } finally { + await controller.dispose(); + } +}); + +test("artifacts: config update closes retired sidecars but abort preserves active ones", async ({ + expect, +}) => { + const directory = await useTmp(); + const controller = new ArtifactsController(); + const reachable = (address: string) => fetch(`http://${address}/`); + try { + controller.beginUpdate(); + const first = await controller.get(path.join(directory, "first")); + await controller.commitUpdate(); + controller.beginUpdate(); + const candidate = await controller.get(path.join(directory, "candidate")); + await controller.abortUpdate(); + expect((await reachable(first.sidecar.address)).status).toBe(403); + await expect(reachable(candidate.sidecar.address)).rejects.toThrow(); + controller.beginUpdate(); + const next = await controller.get(path.join(directory, "next")); + await controller.commitUpdate(); + await expect(reachable(first.sidecar.address)).rejects.toThrow(); + expect((await reachable(next.sidecar.address)).status).toBe(403); + } finally { + await controller.dispose(); + } +}); + +test.skipIf(process.platform === "win32")( + "artifacts: disposing a sidecar stops a stalled Git import", + async ({ expect }) => { + const root = await useTmp(); + const { stdout: gitPath } = await exec("which", ["git"]); + const wrapper = path.join(root, "git"); + const marker = path.join(root, "clone-started"); + await writeFile( + wrapper, + `#!/bin/sh +case " $* " in + *" clone "*) + printf started > '${marker}' + exec sleep 120 ;; +esac +exec '${gitPath.trim().replaceAll("'", "'\\''")}' "$@" +` + ); + await chmod(wrapper, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = `${root}:${previousPath}`; + const controller = new ArtifactsController(); + try { + controller.beginUpdate(); + const { sidecar } = await controller.get(path.join(root, "repos")); + await controller.commitUpdate(); + const pending = fetch(`http://${sidecar.address}/__local_artifacts__`, { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "import", + namespace: "test", + name: "stalled", + generation: "stalled-generation", + sourceUrl: "https://example.invalid/source.git", + }), + signal: AbortSignal.timeout(10_000), + }).catch(() => undefined); + let started = false; + for (let attempt = 0; attempt < 50; attempt++) { + started = await stat(marker).then( + () => true, + () => false + ); + if (started) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + expect(started).toBe(true); + await expect(controller.dispose()).resolves.toBeUndefined(); + await pending; + } finally { + process.env.PATH = previousPath; + await controller.dispose(); + } + }, + 12_000 +); + +test("artifacts: sidecar startup failures reject without crashing Miniflare", async ({ + expect, +}) => { + const root = await useTmp(); + const file = path.join(root, "not-a-directory"); + await writeFile(file, "not a directory"); + const controller = new ArtifactsController(); + controller.beginUpdate(); + try { + await expect(controller.get(file)).rejects.toThrow( + /Local Artifacts Git backend failed to start/ + ); + await controller.abortUpdate(); + } finally { + await controller.dispose(); + } +}); + +test("artifacts: a stopped backend is replaced on the next config update", async ({ + expect, +}) => { + const controller = new ArtifactsController(); + const root = path.join(await useTmp(), "repos"); + try { + controller.beginUpdate(); + const first = await controller.get(root); + await controller.commitUpdate(); + await first.sidecar.close(); + controller.beginUpdate(); + const replacement = await controller.get(root); + expect(replacement.sidecar.secret).not.toBe(first.sidecar.secret); + await controller.commitUpdate(); + } finally { + await controller.dispose(); + } +}); + +test("artifacts: omitted remote preserves remote routing without host Git", async ({ + expect, +}) => { + const root = await useTmp(); + const previousPath = process.env.PATH; + process.env.PATH = root; + try { + const mf = new Miniflare(options("test", "REPOS", null)); + useDispose(mf); + await expect(mf.ready).resolves.toBeDefined(); + } finally { + process.env.PATH = previousPath; + } +}); + +test("artifacts: rejects invalid local namespace before starting services", async ({ + expect, +}) => { + const mf = new Miniflare(options("../invalid")); + await expect(mf.ready).rejects.toThrow( + "Invalid local Artifacts namespace: ../invalid" + ); + // dispose() preserves a startup failure after cleaning up the instance. + await expect(mf.dispose()).rejects.toThrow( + "Invalid local Artifacts namespace: ../invalid" + ); +}); + +test("artifacts: names are case-insensitive for create, get, fork and delete", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["Repo"]); + expect((await rpc(mf, "info", [], "rEpO")).id).toBe(created.id); + await expect(rpc(mf, "create", ["repo"])).rejects.toThrow(/already exists/i); + await expect(rpc(mf, "fork", ["REPO"], "repo")).rejects.toThrow( + /already exists/i + ); + expect(await rpc(mf, "delete", ["REPO"])).toBe(true); + expect(await rpc(mf, "delete", ["repo"])).toBe(false); +}); + +test("artifacts: retained handles cannot access a replacement repository", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { + create(name: string): Promise<{ id: string }>; + get(name: string): Promise<{ + info(): Promise<{ id: string }>; + createToken(): Promise; + }>; + delete(name: string): Promise; + }; + }>(); + const original = await REPOS.create("repo"); + const retained = await REPOS.get("repo"); + await REPOS.delete("repo"); + const replacement = await REPOS.create("repo"); + expect(replacement.id).not.toBe(original.id); + await expect(async () => retained.info()).rejects.toThrow(/not found/i); + await expect(async () => retained.createToken()).rejects.toThrow( + /not found/i + ); + expect((await (await REPOS.get("repo")).info()).id).toBe(replacement.id); +}); + +test("artifacts: restored repository maps ignore inherited names and pin handles", async ({ + expect, +}) => { + const resourcePersistencePath = await useTmp(); + const first = new Miniflare({ ...options(), resourcePersistencePath }); + useDispose(first); + await rpc(first, "create", ["other"]); + await first.dispose(); + const second = new Miniflare({ ...options(), resourcePersistencePath }); + useDispose(second); + for (const name of ["constructor", "toString"]) { + await expect(rpc(second, "get", [name])).rejects.toThrow(/not found/i); + expect(await rpc(second, "delete", [name])).toBe(false); + } + const { REPOS } = await second.getBindings<{ + REPOS: { + get(name: string): Promise<{ info(): Promise<{ id: string }> }>; + }; + }>(); + const created = await rpc(second, "create", ["constructor"]); + const retained = await REPOS.get("constructor"); + expect((await retained.info()).id).toBe(created.id); + expect(await rpc(second, "delete", ["constructor"])).toBe(true); + const replacement = await rpc(second, "create", ["constructor"]); + expect(replacement.id).not.toBe(created.id); + await expect(async () => retained.info()).rejects.toThrow(/not found/i); + expect((await (await REPOS.get("constructor")).info()).id).toBe( + replacement.id + ); +}); + +test("artifacts: supports Node binding proxies", async ({ expect }) => { + const mf = new Miniflare(options()); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { + create(name: string): Promise<{ name: string }>; + list(): Promise<{ total: number }>; + }; + }>(); + expect(await REPOS.create("node-repo")).toMatchObject({ name: "node-repo" }); + expect(await REPOS.list()).toMatchObject({ total: 1 }); +}); + +test("artifacts: aliases share a namespace while distinct namespaces are isolated", async ({ + expect, +}) => { + const mf = new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { + type: "artifacts", + namespace: "shared", + dev: { remote: false }, + }, + ALIAS: { + type: "artifacts", + namespace: "shared", + dev: { remote: false }, + }, + OTHER: { + type: "artifacts", + namespace: "other", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); + useDispose(mf); + const bindings = await mf.getBindings<{ + REPOS: { create(name: string): Promise }; + ALIAS: { list(): Promise<{ total: number }> }; + OTHER: { list(): Promise<{ total: number }> }; + }>(); + await bindings.REPOS.create("shared-repo"); + expect((await bindings.ALIAS.list()).total).toBe(1); + expect((await bindings.OTHER.list()).total).toBe(0); +}); + +test("artifacts: two Workers can share a namespace through different bindings", async ({ + expect, +}) => { + const worker = (name: string, binding: string) => ({ + config: { + name, + compatibilityDate: "2026-09-03", + env: { + [binding]: { + type: "artifacts" as const, + namespace: "shared", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }); + const mf = new Miniflare({ + cf: false, + workers: [worker("first", "REPOS"), worker("second", "ALIAS")], + }); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { create(name: string): Promise<{ name: string }> }; + }>("first"); + const { ALIAS } = await mf.getBindings<{ + ALIAS: { list(): Promise<{ total: number }> }; + }>("second"); + await REPOS.create("shared-repo"); + expect((await ALIAS.list()).total).toBe(1); +}); + +test("artifacts: native Git reads commits, trees, blobs and files", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const source = path.join(await useTmp(), "source"); + await pushFixture(mf, source); + const { stdout } = await git(["rev-parse", "HEAD"], source); + const hash = stdout.trim(); + const commit = await rpc(mf, "readCommit", [hash], "Repo"); + expect(commit.hash).toBe(hash); + expect((await rpc(mf, "log", [], "Repo"))[0].hash).toBe(hash); + const tree = await rpc(mf, "readTree", [commit.treeHash], "Repo"); + const entry = tree.find(({ name }: { name: string }) => name === "hello.txt"); + expect(entry).toBeDefined(); + expect((await rpc(mf, "readBlob", [entry.hash], "Repo")).bytes).toEqual([ + ...new TextEncoder().encode("hello artifacts\n"), + ]); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "hello.txt" }], "Repo") + ).toEqual({ + bytes: [...new TextEncoder().encode("hello artifacts\n")], + type: "text/plain;charset=utf-8", + }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "bytes.bin" }], "Repo") + ).toEqual({ bytes: [0, 255, 128, 1], type: "application/octet-stream" }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "payload.dat" }], "Repo") + ).toEqual({ bytes: [255, 254, 128], type: "application/octet-stream" }); +}); + +test("artifacts: native Git push/clone and token revocation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = await useTmp(); + const created = await pushFixture(mf, path.join(directory, "source")); + expect(new URL(created.remote).hostname).toBe("127.0.0.1"); + const token = await rpc(mf, "createToken", ["read"], "Repo"); + await git([ + "-c", + `http.extraHeader=Authorization: Bearer ${token.plaintext}`, + "clone", + created.remote.replace("Repo.git", "repo.git"), + path.join(directory, "clone"), + ]); + expect( + await readFile(path.join(directory, "clone", "hello.txt"), "utf8") + ).toBe("hello artifacts\n"); + const advertise = `${created.remote}/info/refs?service=git-upload-pack`; + const discovery = await fetch(advertise, { + headers: { + Authorization: `Bearer ${token.plaintext}`, + "Git-Protocol": "version=2", + }, + }); + expect(discovery.status).toBe(200); + expect(discovery.headers.get("content-type")).toContain( + "git-upload-pack-advertisement" + ); + await discovery.body?.cancel(); + const unsupported = await fetch(`${created.remote}/HEAD`, { + headers: { Authorization: `Bearer ${created.token}` }, + }); + expect(unsupported.status).toBe(404); + await unsupported.body?.cancel(); + const unauthenticated = await fetch(advertise); + expect(unauthenticated.status).toBe(401); + await unauthenticated.body?.cancel(); + await rpc(mf, "revokeToken", [token.id], "Repo"); + const revoked = await fetch(advertise, { + headers: { Authorization: `Bearer ${token.plaintext}` }, + }); + expect(revoked.status).toBe(401); + await revoked.body?.cancel(); +}); + +test.skipIf(process.platform === "win32")( + "artifacts: large push replies drain before ref synchronization", + async ({ expect }) => { + const root = await useTmp(); + const { stdout: gitPath } = await exec("which", ["git"]); + const wrapper = path.join(root, "git"); + await writeFile( + wrapper, + `#!/bin/sh +case " $* " in + *" http-backend "*) + printf 'Content-Type: application/x-git-receive-pack-result\\r\\n\\r\\n' + dd if=/dev/zero bs=1048576 count=2 2>/dev/null + exit 0 ;; +esac +exec '${gitPath.trim().replaceAll("'", "'\\''")}' "$@" +` + ); + await chmod(wrapper, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = `${root}:${previousPath}`; + try { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["repo"]); + const response = await fetch(`${created.remote}/git-receive-pack`, { + method: "POST", + headers: { + Authorization: `Bearer ${created.token}`, + "Content-Type": "application/x-git-receive-pack-request", + }, + body: "", + signal: AbortSignal.timeout(15_000), + }); + expect(response.status).toBe(200); + expect((await response.arrayBuffer()).byteLength).toBe(2 * 1024 * 1024); + } finally { + process.env.PATH = previousPath; + } + }, + 20_000 +); + +test.skipIf(process.platform === "win32")( + "artifacts: canceled push reply synchronizes committed refs", + async ({ expect }) => { + const root = await useTmp(); + const { stdout: gitPath } = await exec("which", ["git"]); + const gitBinary = gitPath.trim().replaceAll("'", "'\\''"); + const wrapper = path.join(root, "git"); + await writeFile( + wrapper, + `#!/bin/sh +case " $* " in + *" http-backend "*) + if [ "$REQUEST_METHOD" = POST ]; then + repo="$GIT_PROJECT_ROOT\${PATH_INFO%/git-receive-pack}" + tree=$('${gitBinary}' -C "$repo" hash-object -t tree -w --stdin '${path.join(root, "ref-marker")}' + printf 'Content-Type: application/x-git-receive-pack-result\\r\\n\\r\\n' + dd if=/dev/zero bs=1048576 count=32 2>/dev/null + exit 0 + fi ;; +esac +exec '${gitBinary}' "$@" +` + ); + await chmod(wrapper, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = `${root}:${previousPath}`; + try { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["repo"]); + expect((await rpc(mf, "info", [], "repo")).lastPushAt).toBeNull(); + const response = await fetch(`${created.remote}/git-receive-pack`, { + method: "POST", + headers: { + Authorization: `Bearer ${created.token}`, + "Content-Type": "application/x-git-receive-pack-request", + }, + body: "", + signal: AbortSignal.timeout(15_000), + }); + expect(response.status).toBe(200); + if (!response.body) { + throw new Error("Git response has no body"); + } + const reader = response.body.getReader(); + expect((await reader.read()).done).toBe(false); + await reader.cancel(); + const [repo, commit] = ( + await readFile(path.join(root, "ref-marker"), "utf8") + ) + .trim() + .split(String.fromCharCode(10)); + const { stdout: actualRef } = await exec(gitPath.trim(), [ + "-C", + repo, + "rev-parse", + "refs/heads/main", + ]); + expect(actualRef.trim()).toBe(commit); + let lastPushAt: string | null = null; + for (let attempt = 0; attempt < 50; attempt++) { + lastPushAt = (await rpc(mf, "info", [], "repo")).lastPushAt; + if (lastPushAt) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + expect(lastPushAt).toEqual(expect.any(String)); + } finally { + process.env.PATH = previousPath; + } + }, + 20_000 +); + +test("artifacts: read-only and read-scoped tokens reject Git pushes", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = await useTmp(); + const writable = await pushFixture(mf, path.join(directory, "source")); + const readToken = await rpc(mf, "createToken", ["read"], "Repo"); + const withReadToken = await fetch( + `${writable.remote}/info/refs?service=git-receive-pack`, + { + headers: { Authorization: `Bearer ${readToken.plaintext}` }, + } + ); + expect(withReadToken.status).toBe(403); + await withReadToken.body?.cancel(); + const readOnly = await rpc( + mf, + "fork", + ["read-only", { readOnly: true }], + "Repo" + ); + const writeAttempt = await fetch( + `${readOnly.remote}/info/refs?service=git-receive-pack`, + { + headers: { Authorization: `Bearer ${readOnly.token}` }, + } + ); + expect(writeAttempt.status).toBe(403); + await writeAttempt.body?.cancel(); + const readAttempt = await fetch( + `${readOnly.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${readOnly.token}` } } + ); + expect(readAttempt.status).toBe(200); + await readAttempt.body?.cancel(); +}); + +test("artifacts: metadata and Git data survive restart and binding rename", async ({ + expect, +}) => { + const root = await useTmp(); + const first = new Miniflare({ ...options(), resourcePersistencePath: root }); + useDispose(first); + const created = await pushFixture(first, path.join(await useTmp(), "source")); + await first.dispose(); + const second = new Miniflare({ + ...options("test", "RENAMED"), + resourcePersistencePath: root, + }); + useDispose(second); + const { RENAMED } = await second.getBindings<{ + RENAMED: { + get(name: string): Promise<{ + info(): Promise<{ id: string; remote: string }>; + readFile(args: { ref: string; path: string }): Promise; + }>; + }; + }>(); + const repo = await RENAMED.get("repo"); + const info = await repo.info(); + expect(info.id).toBe(created.id); + expect( + await (await repo.readFile({ ref: "main", path: "hello.txt" })).text() + ).toBe("hello artifacts\n"); + const response = await fetch( + `${info.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${created.token}` } } + ); + expect(response.status).toBe(200); + await response.body?.cancel(); +}); + +test("artifacts: namespaces and instances are isolated; reload preserves state", async ({ + expect, +}) => { + const first = new Miniflare(options()); + const second = new Miniflare(options()); + useDispose(first); + useDispose(second); + await rpc(first, "create", ["repo"]); + expect((await rpc(second, "list")).total).toBe(0); + await first.setOptions(options()); + expect((await rpc(first, "list")).total).toBe(1); + await first.setOptions(options("another")); + expect((await rpc(first, "list")).total).toBe(0); +}); + +test("artifacts: lists pages and deletes a repository and its handle", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + for (const name of ["one", "two", "three"]) { + await rpc(mf, "create", [name]); + } + const first = await rpc(mf, "list", [{ limit: 2 }]); + expect(first).toMatchObject({ total: 3 }); + expect(first.repos).toHaveLength(2); + expect(first.repos[0]).toMatchObject({ + status: "ready", + jurisdiction: "unrestricted", + }); + expect(first.cursor).toEqual(expect.any(String)); + const second = await rpc(mf, "list", [{ limit: 2, cursor: first.cursor }]); + expect(second).toMatchObject({ total: 3 }); + expect(second.repos).toHaveLength(1); + expect(second.cursor).toBeUndefined(); + expect( + [...first.repos, ...second.repos].map((repo) => repo.name).sort() + ).toEqual(["one", "three", "two"]); + const { REPOS } = await mf.getBindings<{ + REPOS: { + get(name: string): Promise<{ info(): Promise<{ name: string }> }>; + }; + }>(); + const handle = await REPOS.get("ONE"); + expect((await handle.info()).name).toBe("one"); + expect(await rpc(mf, "delete", ["one"])).toBe(true); + await expect(async () => handle.info()).rejects.toThrow(/not found/i); + expect(await rpcFailure(mf, "get", ["one"])).toMatchObject({ + code: "NOT_FOUND", + }); + expect((await rpc(mf, "list")).total).toBe(2); +}); + +test("artifacts: token metadata, revocation and validation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["repo"]); + const readToken = await rpc(mf, "createToken", ["read", 60], "repo"); + expect(readToken).toMatchObject({ + scope: "read", + expiresAt: expect.any(String), + }); + const listed = await rpc(mf, "listTokens", [], "repo"); + expect(listed.total).toBe(2); + expect(JSON.stringify(listed)).not.toContain(created.token); + expect(JSON.stringify(listed)).not.toContain(readToken.plaintext); + expect( + listed.tokens.every((token: { state: string }) => token.state === "active") + ).toBe(true); + expect(await rpc(mf, "revokeToken", [readToken.plaintext], "repo")).toBe( + true + ); + expect(await rpc(mf, "revokeToken", [readToken.id], "repo")).toBe(false); + expect((await rpc(mf, "listTokens", [], "repo")).total).toBe(1); + expect( + await rpcFailure(mf, "createToken", ["write", 59], "repo") + ).toMatchObject({ + status: 400, + name: "ArtifactsError", + code: "INVALID_TTL", + }); + expect( + await rpcFailure(mf, "createToken", ["write", 31_536_001], "repo") + ).toMatchObject({ + code: "INVALID_TTL", + }); + expect(await rpcFailure(mf, "revokeToken", ["bad"], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); +}); + +test("artifacts: rejects invalid inputs and returns null for missing objects", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + expect(await rpcFailure(mf, "create", ["invalid/name"])).toMatchObject({ + code: "INVALID_REPO_NAME", + numericCode: 10101, + }); + await rpc(mf, "create", ["repo"]); + expect(await rpcFailure(mf, "list", [{ limit: 0 }])).toMatchObject({ + code: "INVALID_INPUT", + }); + expect(await rpcFailure(mf, "readBlob", ["bad"], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); + const absentHash = "0".repeat(40); + expect(await rpc(mf, "readBlob", [absentHash], "repo")).toBeNull(); + expect(await rpc(mf, "readTree", [absentHash], "repo")).toBeNull(); + expect(await rpc(mf, "readCommit", [absentHash], "repo")).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "missing" }], "repo") + ).toBeNull(); + expect( + await rpcFailure(mf, "readFile", [{ ref: "main" }], "repo") + ).toMatchObject({ + code: "INVALID_INPUT", + }); + expect(await rpcFailure(mf, "log", [{ limit: 0 }], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); + expect( + await rpcFailure(mf, "import", [ + { + source: { url: "http://example.test/repo.git" }, + target: { name: "imported" }, + }, + ]) + ).toMatchObject({ + code: "INVALID_INPUT", + }); + expect( + await rpcFailure(mf, "import", [ + { source: { url: "not a url" }, target: { name: "imported" } }, + ]) + ).toMatchObject({ + code: "INVALID_INPUT", + }); + for (const depth of [0, -1, 1.5, "1", null]) { + expect( + await rpcFailure(mf, "import", [ + { + source: { url: "https://example.test/repo.git", depth }, + target: { name: "imported" }, + }, + ]) + ).toMatchObject({ code: "INVALID_INPUT" }); + } + expect((await rpc(mf, "list")).total).toBe(1); +}); + +test("artifacts: Git credentials are scoped to the repository and operation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const first = await rpc(mf, "create", ["first"]); + const second = await rpc(mf, "create", ["second"]); + const readToken = await rpc(mf, "createToken", ["read"], "first"); + const firstRead = `${first.remote}/info/refs?service=git-upload-pack`; + const firstWrite = `${first.remote}/info/refs?service=git-receive-pack`; + const secondRead = `${second.remote}/info/refs?service=git-upload-pack`; + const basic = `Basic ${Buffer.from(`git:${readToken.plaintext}`).toString("base64")}`; + const read = await fetch(firstRead, { headers: { Authorization: basic } }); + expect(read.status).toBe(200); + await read.body?.cancel(); + const write = await fetch(firstWrite, { headers: { Authorization: basic } }); + expect(write.status).toBe(403); + await write.body?.cancel(); + const wrongRepo = await fetch(secondRead, { + headers: { Authorization: `Bearer ${first.token}` }, + }); + expect(wrongRepo.status).toBe(401); + await wrongRepo.body?.cancel(); +}); + +test("artifacts: forks an empty repo and selects which branches to copy", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["empty"]); + const emptyFork = await rpc(mf, "fork", ["empty-fork"], "empty"); + expect(emptyFork.defaultBranch).toBe("main"); + + const directory = path.join(await useTmp(), "source"); + const created = await pushFixture(mf, directory); + await git(["checkout", "-b", "feature"], directory); + await writeFile(path.join(directory, "feature.txt"), "feature branch\n"); + await git(["add", "."], directory); + await git(["commit", "-m", "feature"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "feature", + ], + directory + ); + const mainOnly = await rpc(mf, "fork", ["main-only"], "Repo"); + const all = await rpc( + mf, + "fork", + ["all", { defaultBranchOnly: false, description: "all refs" }], + "Repo" + ); + expect(mainOnly.defaultBranch).toBe("main"); + expect((await rpc(mf, "info", [], "all")).description).toBe("all refs"); + expect( + await rpc(mf, "readFile", [{ ref: "feature", path: "feature.txt" }], "all") + ).toMatchObject({ bytes: [...new TextEncoder().encode("feature branch\n")] }); + expect( + await rpc( + mf, + "readFile", + [{ ref: "feature", path: "feature.txt" }], + "main-only" + ) + ).toBeNull(); + expect(all.token).toEqual(expect.any(String)); +}); + +test("artifacts: log pages commits and files resolve only valid paths", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = path.join(await useTmp(), "source"); + const created = await pushFixture(mf, directory); + const firstHash = (await git(["rev-parse", "HEAD"], directory)).stdout.trim(); + await writeFile(path.join(directory, "second.txt"), "second\n"); + await mkdir(path.join(directory, "folder")); + await writeFile(path.join(directory, "folder", "nested.txt"), "nested\n"); + await git(["add", "."], directory); + await git(["commit", "-m", "second"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "main", + ], + directory + ); + const secondHash = ( + await git(["rev-parse", "HEAD"], directory) + ).stdout.trim(); + expect( + (await rpc(mf, "log", [{ limit: 1 }], "Repo")).map( + (commit: { hash: string }) => commit.hash + ) + ).toEqual([secondHash]); + expect( + (await rpc(mf, "log", [{ limit: 1, offset: 1 }], "Repo")).map( + (commit: { hash: string }) => commit.hash + ) + ).toEqual([firstHash]); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "../hello.txt" }], "Repo") + ).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "folder" }], "Repo") + ).toBeNull(); + expect( + await rpc( + mf, + "readFile", + [{ ref: "main", path: "folder/nested.txt" }], + "Repo" + ) + ).toMatchObject({ bytes: [...new TextEncoder().encode("nested\n")] }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "missing" }], "Repo") + ).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "missing", path: "hello.txt" }], "Repo") + ).toBeNull(); +}); + +test("artifacts: an explicit reset removes both metadata and Git repositories", async ({ + expect, +}) => { + const persistence = await useTmp(); + const first = new Miniflare({ + ...options(), + resourcePersistencePath: persistence, + }); + useDispose(first); + const created = await pushFixture(first, path.join(await useTmp(), "source")); + await first.dispose(); + await removeDir(path.join(persistence, "artifacts")); + const second = new Miniflare({ + ...options(), + resourcePersistencePath: persistence, + }); + useDispose(second); + expect((await rpc(second, "list")).total).toBe(0); + const fresh = await rpc(second, "create", ["Repo"]); + expect(fresh.id).not.toBe(created.id); + const oldToken = await fetch( + `${fresh.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${created.token}` } } + ); + expect(oldToken.status).toBe(401); + await oldToken.body?.cancel(); +}); + +test("artifacts: missing Git gives an actionable error during local startup", async ({ + expect, +}) => { + const root = await useTmp(); + const previousPath = process.env.PATH; + process.env.PATH = root; + try { + const mf = new Miniflare(options()); + await expect(mf.ready).rejects.toThrow( + /requires Git 2\.32 or newer on PATH.*Install Git.*restart/i + ); + // dispose() preserves the startup error after cleaning up the instance. + await expect(mf.dispose()).rejects.toThrow( + /requires Git 2\.32 or newer on PATH/ + ); + await expect(startGitSidecar(path.join(root, "repos"))).rejects.toThrow( + /requires Git 2\.32 or newer on PATH/ + ); + } finally { + process.env.PATH = previousPath; + } +}); + +test.skipIf(process.platform === "win32")( + "artifacts: Git older than 2.32 fails during local startup with upgrade instructions", + async ({ expect }) => { + const root = await useTmp(); + const fakeGit = path.join(root, "git"); + await writeFile(fakeGit, "#!/bin/sh\nprintf 'git version 2.31.99\\n'\n"); + await chmod(fakeGit, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = root; + try { + const mf = new Miniflare(options()); + await expect(mf.ready).rejects.toThrow( + /requires Git 2\.32 or newer.*Found git version 2\.31\.99.*Upgrade Git.*restart/i + ); + await expect(mf.dispose()).rejects.toThrow(/requires Git 2\.32 or newer/); + await expect(startGitSidecar(path.join(root, "repos"))).rejects.toThrow( + /requires Git 2\.32 or newer.*Upgrade Git/ + ); + } finally { + process.env.PATH = previousPath; + } + } +); + +test("artifacts: sidecar creates a repository without host Git config", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "create", + namespace: "test", + name: "repo", + generation: "test-generation", + defaultBranch: "main", + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + expect(JSON.parse(body)).toEqual({ defaultBranch: "main", refs: {} }); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: an unreadable ownership marker cannot complete deletion", async ({ + expect, +}) => { + const root = path.join(await useTmp(), "repos"); + const sidecar = await startGitSidecar(root); + try { + await createSidecarRepository(sidecar, "original"); + const repository = repositoryPath(root, "test", "repo"); + await git(["config", "--unset", "local-artifacts.generation"], repository); + const remove = () => + fetch(`http://${sidecar.address}/__local_artifacts__`, { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "delete", + namespace: "test", + name: "repo", + generation: "original", + }), + }); + const failed = await remove(); + expect(failed.status).toBe(500); + await failed.body?.cancel(); + await expect(stat(repository)).resolves.toBeDefined(); + await git(["config", "local-artifacts.generation", "original"], repository); + const retried = await remove(); + expect(retried.status).toBe(200); + await retried.body?.cancel(); + await expect(stat(repository)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: chunked Git uploads enforce the streamed byte limit", async ({ + expect, +}) => { + const received: Buffer[] = []; + let rejected = 0; + const sink = new Writable({ + write(chunk: Buffer, _encoding, callback) { + received.push(chunk); + callback(); + }, + }); + await expect( + pipeline( + Readable.from([Buffer.alloc(4), Buffer.alloc(7)]), + limitGitUpload(10, () => rejected++), + sink + ) + ).rejects.toThrow(/request size limit/); + expect(rejected).toBe(1); + expect(Buffer.concat(received)).toHaveLength(4); +}); + +test("artifacts: stale deletion cannot remove a replacement generation", async ({ + expect, +}) => { + const root = path.join(await useTmp(), "repos"); + const sidecar = await startGitSidecar(root); + try { + const deleteGeneration = async (generation: string) => { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "delete", + namespace: "test", + name: "repo", + generation, + }), + } + ); + expect(response.status).toBe(200); + await response.body?.cancel(); + }; + await createSidecarRepository(sidecar, "original-generation"); + await deleteGeneration("original-generation"); + await createSidecarRepository(sidecar, "replacement-generation"); + await deleteGeneration("original-generation"); + expect( + await new GitClient(repositoryPath(root, "test", "repo")).generation() + ).toBe("replacement-generation"); + } finally { + await sidecar.close(); + } +}); + +test.skipIf(process.platform === "win32")( + "artifacts: smart HTTP pushes cannot run repository-local hooks", + async ({ expect }) => { + const root = path.join(await useTmp(), "repos"); + const sidecar = await startGitSidecar(root); + try { + await createSidecarRepository(sidecar, "hook-generation"); + const repository = repositoryPath(root, "test", "repo"); + const hooks = path.join(await useTmp(), "hooks"); + const marker = path.join(await useTmp(), "hook-ran"); + await mkdir(hooks); + const hook = path.join(hooks, "pre-receive"); + await writeFile(hook, `#!/bin/sh\nprintf ran > '${marker}'\n`); + await chmod(hook, 0o755); + await git(["-C", repository, "config", "core.hooksPath", hooks]); + + const source = path.join(await useTmp(), "source"); + await git(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "fixture\n"); + await git(["add", "README"], source); + await git(["commit", "-m", "fixture"], source); + await git( + ["push", `http://${sidecar.address}/git/test/repo.git`, "main"], + source, + { + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `X-Local-Artifacts-Backend: ${sidecar.secret}`, + GIT_CONFIG_KEY_2: "http.extraHeader", + GIT_CONFIG_VALUE_2: "X-Local-Artifacts-Generation: hook-generation", + } + ); + await expect(stat(marker)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await sidecar.close(); + } + } +); + +test("artifacts: shallow imports stay shallow when cloned with Git protocol v2", async ({ + expect, +}) => { + const workspace = await useTmp(); + const source = path.join(workspace, "source"); + await git(["init", "--initial-branch=main", source]); + for (const index of [1, 2, 3]) { + await writeFile(path.join(source, "README"), `version ${index}\n`); + await git(["add", "README"], source); + await git(["commit", "-m", `version ${index}`], source); + } + + const sidecar = await startGitSidecar(path.join(workspace, "repos")); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "import", + namespace: "test", + name: "repo", + generation: "shallow-generation", + sourceUrl: pathToFileURL(source).href, + branch: "main", + depth: 1, + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + + const gitHeaders = { + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `X-Local-Artifacts-Backend: ${sidecar.secret}`, + GIT_CONFIG_KEY_2: "http.extraHeader", + GIT_CONFIG_VALUE_2: "X-Local-Artifacts-Generation: shallow-generation", + }; + const clone = path.join(workspace, "clone"); + const remote = `http://${sidecar.address}/git/test/repo.git`; + await git( + ["-c", "protocol.version=2", "clone", remote, clone], + undefined, + gitHeaders + ); + expect( + (await git(["rev-parse", "--is-shallow-repository"], clone)).stdout.trim() + ).toBe("true"); + expect( + (await git(["rev-list", "--count", "HEAD"], clone)).stdout.trim() + ).toBe("1"); + await git( + ["-c", "protocol.version=2", "fetch", "origin"], + clone, + gitHeaders + ); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: multi-branch clones support compressed negotiation", async ({ + expect, +}) => { + const workspace = await useTmp(); + const source = path.join(workspace, "source"); + await git(["init", "--initial-branch=main", source]); + for (let index = 0; index < 40; index++) { + await git(["switch", "--orphan", `branch-${index}`], source); + await writeFile(path.join(source, "fixture.txt"), `branch ${index}\n`); + await git(["add", "fixture.txt"], source); + await git(["commit", "-m", `branch ${index}`], source); + } + const sidecar = await startGitSidecar(path.join(workspace, "repos")); + try { + await createSidecarRepository(sidecar, "many-branches"); + const headers = { + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `X-Local-Artifacts-Backend: ${sidecar.secret}`, + GIT_CONFIG_KEY_2: "http.extraHeader", + GIT_CONFIG_VALUE_2: "X-Local-Artifacts-Generation: many-branches", + }; + const remote = `http://${sidecar.address}/git/test/repo.git`; + await git(["push", "--all", remote], source, headers); + for (const protocol of ["0", "2"]) { + const clone = path.join(workspace, `clone-${protocol}`); + await git( + ["-c", `protocol.version=${protocol}`, "clone", remote, clone], + undefined, + headers + ); + const refs = await git( + ["for-each-ref", "--format=%(refname)", "refs/remotes/origin"], + clone + ); + expect(refs.stdout.trim().split("\n")).toHaveLength(40); + } + } finally { + await sidecar.close(); + } +}, 60_000); + +test("artifacts: disposing the sidecar closes its private listener", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + const address = `http://${sidecar.address}/missing`; + try { + const response = await fetch(address, { + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + }); + expect(response.status).toBe(404); + await response.body?.cancel(); + } finally { + await sidecar.close(); + } + await expect(fetch(address)).rejects.toThrow(); +}); diff --git a/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts b/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts new file mode 100644 index 00000000000..22eeab6119d --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts @@ -0,0 +1,636 @@ +import { execFile, spawn } from "node:child_process"; +import { once } from "node:events"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { transform } from "esbuild"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; +import type { MiniflareOptions } from "miniflare"; + +const namespace = "lifecycle"; +const workerSource = path.resolve( + __dirname, + "../../../src/workers/artifacts/binding.worker.ts" +); +const clientScript = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name === undefined ? env.REPOS : await env.REPOS.get(name); + return Response.json(await target[method](...args)); + } catch (error) { + return Response.json({ code: error.code, message: error.message }, { status: 400 }); + } + } +};`; + +function options(persistencePath?: string): MiniflareOptions { + return { + cf: false, + ...(persistencePath === undefined + ? {} + : { resourcePersistencePath: persistencePath }), + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { type: "artifacts", namespace, dev: { remote: false } }, + }, + manifest: singleModuleManifest(clientScript), + }, + }, + ], + }; +} + +type RpcResult = Record; + +async function rpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + const response = await mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); + const result = (await response.json()) as RpcResult; + if (!response.ok) { + throw new Error(String(result.message)); + } + return result; +} + +// Run the actual binding worker and its DO inside workerd. Only this test +// source has a controlled clock and a one-shot metadata write fault; neither +// hook exists in the shipped worker. +async function clockedWorker(): Promise { + let source = await readFile(workerSource, "utf8"); + const replacements = [ + ["const createdAt = new Date();", "const createdAt = new Date(testClock);"], + [ + "return Date.parse(token.expiresAt) > Date.now();", + "return Date.parse(token.expiresAt) > testClock;", + ], + [ + "\t// The resolved tail lets the first mutation run immediately.", + `\tprivate failNextSave = false; + failSaveOnce(): void { this.failNextSave = true; } + setTestClock(time: number): void { testClock = time; } + async storedTokenIds(name: string): Promise { + const repositories = await this.ctx.storage.get("repos"); + return repositories?.[name]?.tokens.map((token) => token.id) ?? []; + } + // The resolved tail lets the first mutation run immediately.`, + ], + [ + 'return this.ctx.storage.put("repos", repositories);', + `if (this.failNextSave) { + this.failNextSave = false; + throw new Error("Injected metadata write failure"); + } + return this.ctx.storage.put("repos", repositories);`, + ], + ]; + for (const [original, replacement] of replacements) { + if (source.split(original).length !== 2) { + throw new Error(`Artifacts clock test source changed: ${original}`); + } + source = source.replace(original, replacement); + } + source = `let testClock = 0;\n${source}\n${clockedFetch}`; + return (await transform(source, { loader: "ts", format: "esm" })).code; +} + +const clockedFetch = ` +export default { + async fetch(request, env) { + const { action, name, method, args = [], time, token, scope } = await request.json(); + const namespace = env.localArtifactsNamespace; + const state = namespace.get(namespace.idFromName("lifecycle")); + if (action === "clock") { + await state.setTestClock(time); + return Response.json(true); + } + if (action === "failSave") { + await state.failSaveOnce(); + return Response.json(true); + } + if (action === "storedTokens") { + return Response.json(await state.storedTokenIds(name)); + } + if (action === "backendStatus" || action === "failDelete") { + return env.gitBackend.fetch("http://backend/__local_artifacts__", { + method: "POST", body: JSON.stringify({ action: action === "failDelete" ? "failDelete" : "status" }) + }); + } + if (action === "git") { + return state.fetch(new Request( + "http://localhost/git/lifecycle/repo.git/info/refs?service=git-" + scope + "-pack", + { headers: { Authorization: "Bearer " + token } } + )); + } + try { + const target = name === undefined ? state : await state.get(name); + return Response.json(await target[method](...args)); + } catch (error) { + return Response.json({ code: error.code, message: error.message }, { status: 400 }); + } + } +};`; + +function clockOptions( + script: string, + persistencePath?: string +): MiniflareOptions { + return { + cf: false, + ...(persistencePath === undefined + ? {} + : { resourcePersistencePath: persistencePath }), + workers: [ + { + config: { + name: "clocked", + compatibilityDate: "2026-09-03", + manifest: singleModuleManifest(script), + env: { + config: { + type: "json", + value: { namespace, origin: "http://localhost" }, + }, + localArtifactsNamespace: { + type: "durable-object", + worker: "clocked", + exportName: "LocalArtifactsNamespaceObject", + }, + gitBackend: { type: "worker", worker: "backend" }, + }, + exports: { + LocalArtifactsNamespaceObject: { + type: "durable-object", + storage: "sqlite", + }, + }, + }, + }, + { + config: { + name: "backend", + compatibilityDate: "2026-09-03", + manifest: singleModuleManifest(`let deleteCount = 0; + let failDelete = false; + export default { + async fetch(request) { + if (new URL(request.url).pathname !== "/__local_artifacts__") + return new Response("Git advertisement", { status: 200 }); + const { action } = await request.json(); + if (action === "status") return Response.json({ deleteCount }); + if (action === "failDelete") { failDelete = true; return Response.json(true); } + if (action === "delete") { + if (failDelete) { + failDelete = false; + return new Response("Injected Git deletion failure", { status: 500 }); + } + deleteCount++; + } + if (action === "create" || action === "fork") + return Response.json({ defaultBranch: "main", refs: {} }); + return Response.json({ refs: {} }); + } + }`), + }, + }, + ], + }; +} + +async function clockRequest( + mf: Miniflare, + request: Record +): Promise { + return mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify(request), + }); +} + +async function setClock(mf: Miniflare, time: number): Promise { + const response = await clockRequest(mf, { action: "clock", time }); + await response.body?.cancel(); +} + +async function clockRpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + const response = await clockRequest(mf, { method, args, name }); + const result = (await response.json()) as RpcResult; + if (!response.ok) { + throw new Error(JSON.stringify(result)); + } + return result; +} + +async function gitStatus( + mf: Miniflare, + token: string, + scope: "upload" | "receive" +): Promise { + const response = await clockRequest(mf, { + action: "git", + token, + scope, + }); + await response.body?.cancel(); + return response.status; +} + +const start = Date.parse("2026-01-01T00:00:00.000Z"); + +test("artifacts: a failed metadata write cannot delete Git contents", async ({ + expect, +}) => { + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + const created = await clockRpc(mf, "create", ["repo"]); + await (await clockRequest(mf, { action: "failSave" })).body?.cancel(); + const failed = await clockRequest(mf, { method: "delete", args: ["repo"] }); + expect(failed.status).toBe(400); + await failed.body?.cancel(); + expect((await clockRpc(mf, "info", [], "repo")).id).toBe(created.id); + expect( + await (await clockRequest(mf, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 0 }); + expect(await clockRpc(mf, "delete", ["repo"])).toBe(true); + expect( + await (await clockRequest(mf, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 1 }); +}); + +test("artifacts: failed Git deletion stays retryable across a restart", async ({ + expect, +}) => { + const script = await clockedWorker(); + const root = await useTmp(); + const first = new Miniflare(clockOptions(script, root)); + useDispose(first); + const created = await clockRpc(first, "create", ["repo"]); + await (await clockRequest(first, { action: "failDelete" })).body?.cancel(); + const failed = await clockRequest(first, { + method: "delete", + args: ["repo"], + }); + expect(failed.status).toBe(400); + await failed.body?.cancel(); + expect((await clockRpc(first, "list")).total).toBe(0); + const inaccessible = await clockRequest(first, { + method: "get", + args: ["repo"], + }); + expect(inaccessible.status).toBe(400); + await inaccessible.body?.cancel(); + expect( + await (await clockRequest(first, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 0 }); + await first.dispose(); + + const restarted = new Miniflare(clockOptions(script, root)); + useDispose(restarted); + expect(await clockRpc(restarted, "delete", ["repo"])).toBe(true); + expect( + await (await clockRequest(restarted, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 1 }); + const replacement = await clockRpc(restarted, "create", ["repo"]); + expect(replacement.id).not.toBe(created.id); +}); + +test("artifacts: token expiry is exclusive at the exact workerd clock boundary and persists", async ({ + expect, +}) => { + const script = await clockedWorker(); + const root = await useTmp(); + const first = new Miniflare(clockOptions(script, root)); + useDispose(first); + await setClock(first, start); + await clockRpc(first, "create", ["repo"]); + const read = await clockRpc(first, "createToken", ["read", 60], "repo"); + const write = await clockRpc(first, "createToken", ["write", 60], "repo"); + const expiration = start + 60_000; + expect(read.expiresAt).toBe(new Date(expiration).toISOString()); + expect(String(read.plaintext)).toContain(`?expires=${expiration / 1000}`); + expect(write.expiresAt).toBe(read.expiresAt); + const beforeExpiry = await clockRpc(first, "listTokens", [], "repo"); + expect(beforeExpiry.tokens).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: read.id, + scope: "read", + state: "active", + expiresAt: read.expiresAt, + }), + expect.objectContaining({ + id: write.id, + scope: "write", + state: "active", + expiresAt: write.expiresAt, + }), + ]) + ); + expect(JSON.stringify(beforeExpiry)).not.toContain(String(read.plaintext)); + await setClock(first, expiration - 1); + expect((await clockRpc(first, "listTokens", [], "repo")).total).toBe(3); + expect(await gitStatus(first, String(read.plaintext), "upload")).toBe(200); + expect(await gitStatus(first, String(read.plaintext), "receive")).toBe(403); + expect(await gitStatus(first, String(write.plaintext), "receive")).toBe(200); + expect(await gitStatus(first, String(write.plaintext), "upload")).toBe(200); + + await setClock(first, expiration); + expect((await clockRpc(first, "listTokens", [], "repo")).total).toBe(1); + expect(await gitStatus(first, String(read.plaintext), "upload")).toBe(401); + expect(await gitStatus(first, String(write.plaintext), "receive")).toBe(401); + expect(await gitStatus(first, String(read.plaintext), "receive")).toBe(401); + await first.dispose(); + + const restarted = new Miniflare(clockOptions(script, root)); + useDispose(restarted); + await setClock(restarted, expiration + 1); + expect((await clockRpc(restarted, "listTokens", [], "repo")).total).toBe(1); + expect(await gitStatus(restarted, String(write.plaintext), "upload")).toBe( + 401 + ); + expect(await gitStatus(restarted, String(read.plaintext), "upload")).toBe( + 401 + ); + const replacement = await clockRpc( + restarted, + "createToken", + ["read", 60], + "repo" + ); + expect( + await gitStatus(restarted, String(replacement.plaintext), "upload") + ).toBe(200); + expect( + await clockRpc(restarted, "revokeToken", [replacement.id], "repo") + ).toBe(true); + expect( + await gitStatus(restarted, String(replacement.plaintext), "upload") + ).toBe(401); +}); + +test("artifacts: saving repositories prunes revoked and expired token records", async ({ + expect, +}) => { + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + await setClock(mf, start); + await clockRpc(mf, "create", ["repo"]); + const expired = await clockRpc(mf, "createToken", ["read", 60], "repo"); + const revoked = await clockRpc(mf, "createToken", ["read", 600], "repo"); + await setClock(mf, start + 60_000); + const active = await clockRpc(mf, "createToken", ["read", 600], "repo"); + const stored = async () => + (await ( + await clockRequest(mf, { action: "storedTokens", name: "repo" }) + ).json()) as string[]; + expect(await stored()).toHaveLength(3); + expect(await stored()).not.toContain(expired.id); + expect(await stored()).toContain(revoked.id); + expect(await clockRpc(mf, "revokeToken", [revoked.id], "repo")).toBe(true); + expect(await stored()).not.toContain(revoked.id); + expect(await stored()).toContain(active.id); +}); + +test("artifacts: concurrent same-name creates and forks commit one winner each", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["source"]); + const creates = await Promise.allSettled([ + rpc(mf, "create", ["Target"]), + rpc(mf, "create", ["target"]), + ]); + expect(creates.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + const forks = await Promise.allSettled([ + rpc(mf, "fork", ["Child"], "source"), + rpc(mf, "fork", ["child"], "source"), + ]); + expect(forks.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + expect((await rpc(mf, "list")).total).toBe(3); + const child = await rpc(mf, "info", [], "child"); + expect(child.source).toBe(`artifacts:${namespace}/source`); +}); + +test("artifacts: a delayed push cannot update a replacement repository", async ({ + expect, +}) => { + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + const original = await clockRpc(mf, "create", ["repo"]); + await clockRpc(mf, "delete", ["repo"]); + const replacement = await clockRpc(mf, "create", ["repo"]); + expect(replacement.id).not.toBe(original.id); + const before = await clockRpc(mf, "info", [], "repo"); + await expect( + clockRpc(mf, "syncAfterPush", ["repo", original.id, {}]) + ).rejects.toThrow(/NOT_FOUND/); + const after = await clockRpc(mf, "info", [], "repo"); + expect(after.updatedAt).toBe(before.updatedAt); + expect(after.lastPushAt).toBe(before.lastPushAt); +}); + +test("artifacts: import and fork metadata remain distinct across concurrent creations", async ({ + expect, +}) => { + // The backend is a local stub: this exercises namespace state transitions, + // not network cloning (which is covered by the native Git tests). + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + const results = await Promise.allSettled([ + clockRpc(mf, "import", [ + { + source: { + url: "https://example.com/source.git", + branch: "release", + depth: 1, + }, + target: { name: "Imported", opts: { description: "remote copy" } }, + }, + ]), + clockRpc(mf, "import", [ + { + source: { url: "https://example.com/other.git" }, + target: { name: "imported" }, + }, + ]), + ]); + expect(results.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + const imported = await clockRpc(mf, "info", [], "imported"); + expect(imported.source).toMatch( + /^git:https:\/\/example\.com\/(source|other)\.git$/ + ); + const fork = await clockRpc( + mf, + "fork", + ["forked", { readOnly: true }], + "imported" + ); + expect(fork.name).toBe("forked"); + expect(await clockRpc(mf, "info", [], "forked")).toMatchObject({ + readOnly: true, + source: `artifacts:${namespace}/${imported.name}`, + }); + expect((await clockRpc(mf, "list")).total).toBe(2); +}); + +test("artifacts: concurrent independent mutations preserve all repositories and tokens", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await Promise.all( + ["one", "two", "three"].map((name) => rpc(mf, "create", [name])) + ); + await Promise.all( + ["one", "two", "three"].map(async (name) => { + await Promise.all([ + rpc(mf, "createToken", ["read", 60], name), + rpc(mf, "fork", [`${name}-fork`], name), + ]); + }) + ); + for (const name of ["one", "two", "three"]) { + expect((await rpc(mf, "listTokens", [], name)).total).toBe(2); + expect((await rpc(mf, "info", [], `${name}-fork`)).source).toBe( + `artifacts:${namespace}/${name}` + ); + } + expect((await rpc(mf, "list")).total).toBe(6); +}); + +async function killChild(child: ReturnType): Promise { + if (child.exitCode !== null || child.pid === undefined) { + return; + } + const closed = once(child, "close"); + if (process.platform === "win32") { + // Killing only Node would orphan workerd and leave its listener and + // SQLite files open. taskkill /T stops the entire child process tree. + await promisify(execFile)("taskkill", [ + "/PID", + String(child.pid), + "/T", + "/F", + ]); + } else { + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") { + throw error; + } + } + } + await closed; +} + +// The child deliberately never calls dispose(): the parent kills it after +// create() has committed, then starts a new listener against the same root. +test("artifacts: abrupt child-process exit leaves committed repositories restartable", async ({ + expect, +}) => { + const root = await useTmp(); + const marker = path.join(root, "ready"); + const childScript = path.join(root, "child.cjs"); + const modulePath = path.resolve(__dirname, "../../../dist/src/index.js"); + await writeFile( + childScript, + ` +const { Miniflare } = require(${JSON.stringify(modulePath)}); +const { writeFileSync } = require("node:fs"); +const mf = new Miniflare({ + cf: false, + resourcePersistencePath: ${JSON.stringify(root)}, + workers: [{ config: { + name: "", compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts", namespace: "lifecycle", dev: { remote: false } } }, + manifest: { mainModule: "index.mjs", modulesRoot: ${JSON.stringify(root)}, modules: { "index.mjs": { type: "esm", contents: "export default { fetch() { return new Response('ok') } }" } } } + }}] +}); +(async () => { + const { REPOS } = await mf.getBindings(); + const created = await REPOS.create("survivor"); + writeFileSync(${JSON.stringify(marker)}, JSON.stringify(created)); + process.stdout.write("READY\\n"); +})().catch(error => { process.stderr.write(String(error)); process.exitCode = 1; }); +` + ); + const child = spawn(process.execPath, [childScript], { + stdio: ["ignore", "pipe", "pipe"], + // workerd is a subprocess: kill the process group to avoid orphaning it. + detached: process.platform !== "win32", + }); + let listener = ""; + let token = ""; + let stderr = ""; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + try { + if (child.stdout === null) { + throw new Error("Child stdout pipe unavailable"); + } + await Promise.race([ + once(child.stdout, "data", { signal: AbortSignal.timeout(15_000) }), + once(child, "exit").then(() => { + throw new Error(`Child exited before ready: ${stderr}`); + }), + ]); + const created = JSON.parse(await readFile(marker, "utf8")) as { + remote: string; + token: string; + }; + expect(created.remote).toMatch(/^http:\/\/127\.0\.0\.1:/); + listener = `${created.remote}/info/refs?service=git-upload-pack`; + token = created.token; + } finally { + await killChild(child); + } + await expect( + fetch(listener, { signal: AbortSignal.timeout(2_000) }) + ).rejects.toThrow(); + const restarted = new Miniflare(options(root)); + useDispose(restarted); + const info = await rpc(restarted, "info", [], "survivor"); + expect(info.name).toBe("survivor"); + const advertised = await fetch( + `${info.remote}/info/refs?service=git-upload-pack`, + { + headers: { Authorization: `Bearer ${token}` }, + } + ); + expect(advertised.status).toBe(200); + await advertised.body?.cancel(); + const created = await rpc(restarted, "create", ["after-restart"]); + expect(created.name).toBe("after-restart"); + expect((await rpc(restarted, "list")).total).toBe(2); +}); diff --git a/packages/miniflare/test/plugins/artifacts/storage.spec.ts b/packages/miniflare/test/plugins/artifacts/storage.spec.ts new file mode 100644 index 00000000000..33407aeb46a --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/storage.spec.ts @@ -0,0 +1,190 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { GitClient, runGit } from "../../../src/plugins/artifacts/git-client"; +import { startGitSidecar } from "../../../src/plugins/artifacts/git-sidecar"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "../../../src/plugins/artifacts/storage"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; + +const namespace = "namespace-with-a-long-name-for-platform-testing"; + +test("artifacts: repository paths are compact, deterministic and namespace-scoped", ({ + expect, +}) => { + const directory = repositoryDirectory(namespace, "repo"); + expect(directory).toMatch(/^[0-9a-f]{32}\.git$/); + expect(repositoryDirectory(namespace, "repo")).toBe(directory); + expect(repositoryDirectory("other", "repo")).not.toBe(directory); + expect(repositoryDirectory(namespace, "other")).not.toBe(directory); + expect(repositoryDirectory("ab", "c")).not.toBe( + repositoryDirectory("a", "bc") + ); + expect(repositoryDirectory("../namespace", "../repo")).toMatch( + /^[0-9a-f]{32}\.git$/ + ); +}); + +test("artifacts: sidecar rejects legacy storage without removing it", async ({ + expect, +}) => { + const root = await useTmp(); + const legacy = path.join(root, "a".repeat(64)); + await mkdir(legacy); + await writeFile(path.join(legacy, "sentinel"), "preserve me"); + await expect(startGitSidecar(root)).rejects.toThrow( + /older draft layout.*Back up/ + ); + expect(await readFile(path.join(legacy, "sentinel"), "utf8")).toBe( + "preserve me" + ); +}); + +test("artifacts: restart removes interrupted creation and import staging", async ({ + expect, +}) => { + const root = await useTmp(); + const creating = await mkdtemp(path.join(root, ".artifacts-stage-")); + const importing = await mkdtemp(path.join(root, ".artifacts-import-")); + await writeFile(path.join(creating, "partial"), "private data"); + await writeFile(path.join(importing, "partial"), "private data"); + const unrelated = path.join(root, "unrelated"); + await mkdir(unrelated); + await writeFile(path.join(unrelated, "keep"), "keep me"); + + const sidecar = await startGitSidecar(root); + try { + expect(await readdir(root)).toEqual(["unrelated"]); + expect(await readFile(path.join(unrelated, "keep"), "utf8")).toBe( + "keep me" + ); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: legacy metadata layout is detected without resetting it", async ({ + expect, +}) => { + const root = await useTmp(); + const id = createHash("sha256").update(namespace).digest("hex").slice(0, 32); + const metadata = path.join(root, "artifacts", id, "metadata"); + const legacy = path.join(metadata, `artifacts-${id}`); + await mkdir(legacy, { recursive: true }); + await writeFile(path.join(legacy, "sentinel"), "preserve me"); + await expect(assertSupportedGitLayout(metadata)).rejects.toThrow( + /older draft layout.*Back up/ + ); + expect(await readFile(path.join(legacy, "sentinel"), "utf8")).toBe( + "preserve me" + ); +}); + +test("artifacts: namespace metadata and Git data survive restart under paths with spaces", async ({ + expect, +}) => { + const root = path.join(await useTmp(), "development project"); + const config = { + name: "test", + compatibilityDate: "2026-09-03", + env: { + REPOS: { type: "artifacts" as const, namespace, dev: { remote: false } }, + }, + manifest: singleModuleManifest( + `export default { fetch() { return new Response("ok"); } };` + ), + }; + type Bindings = { + REPOS: { + create( + name: string + ): Promise<{ id: string; remote: string; token: string }>; + get(name: string): Promise<{ info(): Promise<{ id: string }> }>; + }; + }; + const options = { + cf: false as const, + resourcePersistencePath: root, + workers: [{ config }], + }; + const first = new Miniflare(options); + useDispose(first); + const { REPOS } = await first.getBindings(); + const created = await REPOS.create("repo"); + const id = created.id; + await first.dispose(); + + const second = new Miniflare(options); + useDispose(second); + const restarted = await second.getBindings(); + expect(await (await restarted.REPOS.get("repo")).info()).toMatchObject({ + id, + }); + const namespaces = await readdir(path.join(root, "artifacts")); + expect(namespaces).toHaveLength(1); + const persisted = path.join(root, "artifacts", namespaces[0]); + expect(await readdir(path.join(persisted, "git"))).toEqual([ + repositoryDirectory(namespace, "repo"), + ]); + expect( + (await readdir(path.join(persisted, "metadata"))).length + ).toBeGreaterThan(0); +}); + +test("artifacts: long Git storage roots can initialize and repack repositories", async ({ + expect, +}) => { + const workspace = await useTmp(); + const root = path.join(workspace, "project with spaces " + "x".repeat(32)); + const sidecar = await startGitSidecar(root); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "create", + namespace, + name: "repo", + generation: "test-generation", + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + const repository = repositoryPath(root, namespace, "repo"); + const source = path.join(workspace, "source"); + await runGit(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "long-path regression\n"); + await runGit(["-C", source, "add", "README"]); + await runGit([ + "-C", + source, + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-m", + "fixture", + ]); + await runGit(["-C", source, "push", repository, "main"]); + await runGit(["-C", repository, "repack", "-ad"]); + expect( + (await readdir(path.join(repository, "objects", "pack"))).some((file) => + file.endsWith(".pack") + ) + ).toBe(true); + expect(await new GitClient(repository).generation()).toBe( + "test-generation" + ); + } finally { + await sidecar.close(); + } +});