diff --git a/.changeset/artifacts-opt-in-simulator.md b/.changeset/artifacts-opt-in-simulator.md new file mode 100644 index 00000000000..047636b2940 --- /dev/null +++ b/.changeset/artifacts-opt-in-simulator.md @@ -0,0 +1,5 @@ +--- +"miniflare": minor +--- + +Add an opt-in local Artifacts binding simulator to Miniflare. Set `dev.remote: false` on an Artifacts binding to use persistent local repositories and authenticated Git smart HTTP backed by host Git 2.32 or newer. Omitted `dev.remote` retains the existing remote binding behavior. diff --git a/.changeset/local-artifacts-binding.md b/.changeset/local-artifacts-binding.md new file mode 100644 index 00000000000..fc0176047ab --- /dev/null +++ b/.changeset/local-artifacts-binding.md @@ -0,0 +1,9 @@ +--- +"miniflare": minor +"wrangler": minor +"@cloudflare/workers-utils": minor +--- + +Enable local Artifacts by default across Wrangler, Vite, and Workers Vitest + +Artifacts bindings now use the Miniflare simulator and persistent local repositories by default in development. Set `remote: true` on the binding to opt into the deployed resource. Local development does not require Cloudflare credentials, but does require Git 2.32 or newer on the host. diff --git a/.changeset/local-artifacts-event-ingress.md b/.changeset/local-artifacts-event-ingress.md new file mode 100644 index 00000000000..ab655a20f23 --- /dev/null +++ b/.changeset/local-artifacts-event-ingress.md @@ -0,0 +1,7 @@ +--- +"miniflare": minor +--- + +Add a private, experimental Artifacts event producer in the local Queue plugin + +Local repository and token lifecycle operations now send typed version-one Artifacts events over a private JSRPC binding. The receiver does not retain or deliver events until subscription support is added; Git transport and namespace events are not yet produced. diff --git a/packages/miniflare/scripts/build.mjs b/packages/miniflare/scripts/build.mjs index 56ba5b35265..516f094f304 100644 --- a/packages/miniflare/scripts/build.mjs +++ b/packages/miniflare/scripts/build.mjs @@ -446,7 +446,13 @@ async function buildPackage() { logLevel: watch ? "info" : "warning", outdir: outPath, outbase: pkgRoot, - entryPoints: [indexPath, ...fixtureBuilds], + entryPoints: [ + indexPath, + // A separate thread keeps native Git available during synchronous Node + // binding-proxy calls, which block Miniflare's main event loop. + path.join(pkgRoot, "src/plugins/artifacts/git-sidecar.ts"), + ...fixtureBuilds, + ], }; if (watch) { diff --git a/packages/miniflare/src/config/v4-convert.ts b/packages/miniflare/src/config/v4-convert.ts index 734cd06a2d4..154a3d47981 100644 --- a/packages/miniflare/src/config/v4-convert.ts +++ b/packages/miniflare/src/config/v4-convert.ts @@ -796,7 +796,11 @@ function addProductBindings( env[name] = { type: "artifacts", namespace: binding.namespace, - dev: { remote: isRemote(binding.remoteProxyConnectionString) }, + // Preserve omitted legacy settings so the Artifacts plugin applies + // the same local default as a V5 binding. + ...(binding.remoteProxyConnectionString === undefined + ? {} + : { dev: { remote: isRemote(binding.remoteProxyConnectionString) } }), }; } for (const name of Object.keys(worker.workerLoaders ?? {})) { diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 1284d267928..ef661dfc487 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -77,6 +77,7 @@ import { WORKER_BINDING_SERVICE_LOOPBACK, WORKFLOWS_PLUGIN_NAME, } from "./plugins"; +import { ArtifactsController } from "./plugins/artifacts/controller"; import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants"; import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version"; import { closeBrowserProcess } from "./plugins/browser-rendering/process"; @@ -888,6 +889,7 @@ export class Miniflare { #maybeInspectorProxyController?: InspectorProxyController; #previousRuntimeInspectorPort?: number; + #artifactsController = new ArtifactsController(); #containerPrivilegesCache = new ContainerPrivilegesCache(); #hyperdriveProxyController: HyperdriveProxyController = new HyperdriveProxyController(); @@ -2326,6 +2328,7 @@ export class Miniflare { queueConsumers, containerPrivilegesCache: this.#containerPrivilegesCache, hyperdriveProxyController: this.#hyperdriveProxyController, + artifactsController: this.#artifactsController, }; for (const [key, plugin] of this.#mergedPluginEntries) { const pluginServicesExtensions = await plugin.getServices({ @@ -2341,6 +2344,11 @@ export class Miniflare { } else { pluginServices = pluginServicesExtensions.services; extensions.push(...pluginServicesExtensions.extensions); + for (const socket of pluginServicesExtensions.sockets ?? []) { + if (!sockets.some((existing) => existing.name === socket.name)) { + sockets.push(socket); + } + } } for (const service of pluginServices) { @@ -2672,11 +2680,13 @@ export class Miniflare { async #assembleAndUpdateConfig(reusePorts = false) { this.#hyperdriveProxyController.beginUpdate(); + this.#artifactsController.beginUpdate(); try { await this.#assembleAndUpdateConfigInternal(reusePorts); } finally { - // If assembly or runtime startup failed, keep the old listeners and - // close only the new candidates. A successful update already committed. + // A failed update retains listeners used by the previous runtime and + // closes only candidates started during this attempt. + await this.#artifactsController.abortUpdate(); this.#hyperdriveProxyController.abortUpdate(); } } @@ -2800,6 +2810,7 @@ export class Miniflare { } } this.#hyperdriveProxyController.commitUpdate(activeExternalAddresses); + await this.#artifactsController.commitUpdate(); // Note: `updateConfig()` doesn't resolve until ports for all required // sockets have been recorded. At this point, `maybeSocketPorts` contains // all of `requiredSockets` as keys. @@ -3915,6 +3926,14 @@ export class Miniflare { } const runtimeCleanupOutcome = await runtimeDisposeOutcome; + try { + await this.#artifactsController.dispose(); + } catch (error) { + if (!independentCleanupFailed) { + independentCleanupFailed = true; + independentCleanupError = error; + } + } this.#devRegistry.unregisterWorkers(); try { await Promise.all( diff --git a/packages/miniflare/src/plugins/artifacts/LOCAL_DEVELOPMENT.md b/packages/miniflare/src/plugins/artifacts/LOCAL_DEVELOPMENT.md new file mode 100644 index 00000000000..7a0e8286930 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/LOCAL_DEVELOPMENT.md @@ -0,0 +1,77 @@ +# Develop with local Artifacts + +## Prerequisite: Git + +This first version of the local Artifacts emulator requires Git 2.32 or newer installed globally on your development machine, with `git` available on the PATH of the process starting Wrangler, Vite, or Workers Vitest. Run `git --version` in that environment before starting the dev server or tests. If Git is missing, cannot run, or is older than 2.32, local Artifacts fails at startup with installation or upgrade instructions instead of waiting for a repository operation. Restart your dev tool after installing Git so it picks up the updated PATH. + +Git 2.32 provides `GIT_CONFIG_GLOBAL`, which the emulator uses to isolate host Git settings. Use an up-to-date, security-patched release. + +A binding with `"remote": true` does not start the local emulator and does not need local Git. Production Artifacts also does not require Git on your machine. We plan to open-source a JavaScript Git engine in the future; it is not part of this initial local release, which uses native Git. + +## Configure a binding + +For a Wrangler project, add an Artifacts binding to `wrangler.jsonc`: + +```jsonc +{ + "name": "example-worker", + "main": "src/index.ts", + "compatibility_date": "2026-09-03", + "artifacts": [{ "binding": "REPOS", "namespace": "examples" }], +} +``` + +To try this unreleased feature from the Workers SDK checkout, install its dependencies and build Miniflare and Wrangler, then start the checkout's CLI from the app directory: + +```sh +# From the workers-sdk root: +pnpm install --frozen-lockfile +pnpm --filter miniflare build +pnpm --filter wrangler build + +# From your app directory, using the absolute path to this checkout: +node /path/to/workers-sdk/packages/wrangler/bin/wrangler.js dev --port=0 --inspector-port=0 +``` + +Open the URL on Wrangler's `Ready on` line. With `@cloudflare/vite-plugin`, use the same Wrangler config with `cloudflare()` in your Vite config and start the Vite dev server. In Workers Vitest, point `cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" } })` at this config. These paths share the Miniflare implementation. To force local behavior explicitly, add `"remote": false` to the binding. The absence of `remote` is also local; this differs from older development behavior that implicitly used a remote Artifacts resource. Do not rely on production repositories being available locally. + +A Worker can create and inspect a local repository: + +```ts +const created = await env.REPOS.create("demo"); +const repo = await env.REPOS.get("demo"); +const listing = await env.REPOS.list(); +const info = await repo.info(); +// created.remote is a loopback HTTP Git URL; created.token is a secret. +``` + +To use Git smart HTTP, use the returned `remote` URL and a repository token. If `git clone` prompts for a username and password, enter any username (for example, `git`) and use `created.token` as the password. This is an Artifacts repository token, **not** a GitHub or Cloudflare account password. A configured Git credential helper may remember a password entered at the prompt. To avoid that, supply a bearer token for one Git command without storing it in source or in a Git remote URL: + +```sh +read -s REPO_TOKEN # Paste the token returned by create(); input is hidden. +printf '\n' +# Replace the URL with the `remote` value returned by create() or info(). +GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader \ + GIT_CONFIG_VALUE_0="Authorization: Bearer $REPO_TOKEN" \ + git clone 'http://127.0.0.1:/git/examples/demo.git' +unset REPO_TOKEN +``` + +After pushing `hello.txt` to `main`, read it in the Worker with `await repo.readFile({ ref: "main", path: "hello.txt" })` (a `Blob`, or `null` when absent). The initial create token permits writes. Its plaintext is returned only once; if you lose it, use `repo.createToken("read")` for cloning or `repo.createToken("write")` for pushing and save the returned `plaintext` securely. Revoke tokens that are no longer needed. Avoid printing tokens or committing them to config. A plain request without a token cannot clone a private local repository. Git's URL uses an ephemeral loopback port; obtain the current URL after restarting the dev server instead of hardcoding it. + +## Storage and network boundaries + +Local repositories and metadata persist under the dev tool's Miniflare resource persistence path (for example, Wrangler's `.wrangler/state/` when persistence is enabled). Use the same project root and persistence settings to keep local data across reloads and restarts. Disabling persistence (for example, Vite's `persistState: false`) makes the data temporary. Local and deployed Artifacts are separate: local create, push, and delete operations do not update remote resources. To opt in to a remote binding, set `"remote": true` on that Artifacts binding and configure the credentials required by your development tool. Remote mode uses a Cloudflare proxy, not the local Git store; do not assume local data migrates to it. + +To reset local state, stop the dev server first, back up any repositories you need, then delete the `artifacts/` directory inside that dev tool's resource persistence path. Older draft persisted Artifacts layouts are not automatically migrated and may require this backup-and-reset procedure. Do not delete unrelated binding data in the same persistence directory. + +A local binding does not contact Cloudflare for repository operations. **Explicit imports from HTTPS Git sources are different:** they make outbound network requests from your development machine to the supplied host, including private-network hosts. Import does not follow HTTP redirects; supply the final HTTPS Git URL so credentials cannot be redirected to another destination. Imports can require authentication and may fail due to TLS certificates, connectivity, or upstream Git permissions. For a private Git CA, set `GIT_SSL_CAINFO` to the trusted CA file in the dev tool's host environment before starting the dev server. Do not disable certificate verification. After a successful import, URL credentials are omitted from the local repository metadata and Git config. In-flight process-crash recovery is not yet validated; treat partial local storage after an interrupted import as sensitive if the URL contained credentials. Treat the import URL as a secret while calling `import()` and avoid logging it. Avoid importing untrusted or private URLs in examples or tests. No remote resources are contacted by the local-only examples above. + +## Troubleshooting and limits + +- If startup says local Artifacts requires Git, install Git globally, check `git --version` in the dev tool's environment, and restart the tool with `git` on PATH. +- If startup rejects an older storage layout, back it up and reset only the Artifacts persistence directory; a restart alone will not convert it. +- If Git asks for a username and password, use any username and the repository token as the password. If Git reports unauthorized, check the repository, scope, expiry, and revocation; do not use a Cloudflare or GitHub account password. +- If startup reports `Address already in use`, the dev or inspector port may be occupied. Pass `--port=0 --inspector-port=0` and use the printed `Ready on` URL. +- If a Git URL stops responding after restart, get the new `remote` from the binding; the loopback listener is tied to the dev process. +- Local development is not a replica of production placement or infrastructure. The local Git endpoint is HTTP on loopback only, and local state and tokens do not transfer to a remote namespace. Miniflare Artifacts, Wrangler dev, and Vite local fixtures have passed Windows CI; Workers Vitest's Artifacts fixture has not been verified on Windows. diff --git a/packages/miniflare/src/plugins/artifacts/README.md b/packages/miniflare/src/plugins/artifacts/README.md new file mode 100644 index 00000000000..a616f2637e4 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/README.md @@ -0,0 +1,171 @@ +# Artifacts in Miniflare + +For setup, persistence, Git authentication, and local/remote behavior, see +[Local development](./LOCAL_DEVELOPMENT.md). + +The Artifacts plugin implements the local Artifacts binding in workerd. This +first version requires Git 2.32 or newer installed globally on the host and +available on PATH. Miniflare checks the Git version at local startup and gives +installation or upgrade instructions if needed. A Node worker thread runs the +native Git backend. Unless `dev.remote` is explicitly `true`, an Artifacts +binding uses local storage rather than contacting Cloudflare. We plan to +open-source a JavaScript Git engine in the future; it is not included in this +version. + +## Local startup + +```mermaid +sequenceDiagram + participant Dev as Developer tool + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Controller as ArtifactsController + participant Sidecar as Node Git sidecar + participant Workerd as workerd + + Dev->>MF: Configure Artifacts binding + MF->>Plugin: getBindings() + Plugin->>Plugin: Select local and validate namespace + Plugin-->>MF: Bind to local namespace entrypoint + MF->>Plugin: getServices() + Plugin->>Plugin: Verify host Git is available on PATH + Plugin->>Controller: get(Git storage path) + Controller->>Sidecar: Start worker thread and private HTTP listener + Sidecar-->>Controller: Address and secret + Controller-->>Plugin: Sidecar endpoint and Git socket port + Plugin-->>MF: Binding Worker, DO storage, backend service, Git socket + MF->>Workerd: Start configured services and socket +``` + +`getBindings()` routes each local binding to a namespace-scoped service. +`getServices()` creates the binding Worker, namespace Durable Object, persistent +metadata disk, internal Git backend service, and Git socket. The controller +reuses a sidecar per Git storage path and closes it when Miniflare is disposed. +The socket used by Git clients and the sidecar's private HTTP listener are +**different endpoints**. If the sidecar exits after startup, the controller +forgets its endpoint and warns; an explicit Miniflare configuration reload +reconnects, but existing workerd connections do not recover automatically. +Restart the dev server or reload its configuration if Git operations stop +working after a sidecar exit. + +## Local binding RPC + +For example, `env.REPOS.create("demo")`: + +```mermaid +sequenceDiagram + participant App as User Worker + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Disk as Local disk + + App->>Entry: env.REPOS.create("demo") + Entry->>DO: RPC create("demo") + DO->>DO: Validate and serialize mutation + DO->>Backend: POST internal create request + Backend->>Sidecar: Inject private backend header + Sidecar->>Disk: git init --bare + Sidecar-->>DO: Git state (via backend service) + DO->>Disk: Persist metadata and token hash + DO-->>Entry: Repository info and initial token + Entry-->>App: Create result +``` + +The Durable Object stores repository metadata and token hashes on local disk; +the sidecar stores the bare Git repositories separately. `git-sidecar.ts` +handles HTTP routing and repository locks, while `git-client.ts` owns +repository-scoped Git commands and result parsing. Read operations such as +`readFile()` also use the binding Worker and native Git backend. Node callers +reach the same binding through Miniflare's Node binding proxy. Namespace and +repository names are hashed into short, namespace-scoped on-disk paths; this +leaves room for Git packfiles and workerd SQLite files on platforms with path +length limits. Creation, fork, and import use private staging directories; +interrupted staging is removed on the next backend startup. If a process dies +while creating a repository, a leftover creation lock blocks reuse until you +confirm no other dev server uses the directory and remove the lock named in +the error. Import URL credentials are passed to Git as an ephemeral HTTPS +authorization header, +not recorded in a repository's Git config. Host Git runs with an isolated HOME +so it cannot read the developer's `.netrc`; SSH/scp-style imports and Git remote +helpers are rejected rather than accessing host SSH keys or executing helpers. +The local Worker binding accepts only explicit HTTPS imports; filesystem paths +are available only to the internal Git client. Repository locks are process-local: +do not run two dev servers against the same Artifacts persistence directory at +the same time. Older draft storage layouts are not migrated automatically: +startup rejects a detected old layout rather than silently creating empty +repositories. Back up any local repositories before resetting the Artifacts +persistence directory. Do not delete the persistence directory without first +checking whether it holds work you need. + +## Git clone and push + +```mermaid +sequenceDiagram + participant Git as Git client + participant Socket as Git socket + participant Entry as Binding Worker + participant DO as Namespace DO + participant Backend as Backend service + participant Sidecar as Node Git sidecar + participant Native as Native Git + + Git->>Socket: Git smart HTTP at /git/:namespace/:repo.git + Socket->>Entry: Route request + Entry->>DO: fetch(request) + DO->>DO: Check endpoint, repository, token, and scope + DO->>Backend: Forward request without Authorization header + Backend->>Sidecar: Inject private backend header + Sidecar->>Native: Stream via git http-backend + Native-->>Sidecar: Discovery or pack response + Sidecar-->>DO: Git response (via backend service) + opt Successful receive-pack push + DO->>Backend: Read updated refs + Backend->>Sidecar: Internal refs request + Sidecar-->>DO: Refs (via backend service) + DO->>DO: Persist updated push metadata + end + DO-->>Entry: Git response + Entry-->>Socket: Git response + Socket-->>Git: Git response +``` + +The binding Worker checks the supported Git endpoints and authenticates +repository tokens before forwarding. The internal service injects the sidecar +secret; the client's `Authorization` header is removed before that hop. For a +successful push, the Durable Object refreshes repository metadata before +returning the response. The sidecar uses native `git http-backend` for Git smart +HTTP rather than reproducing Git's pack protocol. All native Git commands +(including the HTTP backend) override repository-local hook configuration. +A failed deletion leaves an inaccessible, persisted tombstone, so retrying +`delete()` or creating the same name can finish native Git cleanup even after a +process restart. + +## Explicit remote binding + +When `dev.remote` is `true`, this binding uses the remote-bindings proxy instead +of creating local Artifacts storage or a local Git sidecar for that namespace. + +```mermaid +sequenceDiagram + participant App as User Worker + participant MF as Miniflare + participant Plugin as Artifacts plugin + participant Proxy as Remote-bindings proxy + participant CF as Cloudflare Artifacts + + MF->>Plugin: getBindings() with remote: true + Plugin-->>MF: Bind to remote proxy service + MF->>Proxy: Start remote proxy Worker + App->>Proxy: env.REPOS method call + Proxy->>CF: Remote binding request + CF-->>Proxy: Result + Proxy-->>App: Result +``` + +A local binding does not automatically synchronize with production Artifacts. +An explicit repository import may contact its specified HTTPS Git remote from +the developer's host, including private-network hosts; it does not follow HTTP +redirects. This is distinct from enabling a remote Artifacts binding. Do not +pass an untrusted URL to `import()` when the host can access sensitive services. diff --git a/packages/miniflare/src/plugins/artifacts/controller.ts b/packages/miniflare/src/plugins/artifacts/controller.ts new file mode 100644 index 00000000000..37d1cebf515 --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/controller.ts @@ -0,0 +1,179 @@ +import { once } from "node:events"; +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import getPort from "get-port"; +import type { GitSidecar } from "./git-sidecar"; + +export interface LocalArtifactsBackend { + sidecar: GitSidecar; + port: number; +} + +// Node binding proxies make synchronous calls. The Git backend must run on +// another thread, otherwise those calls prevent it from answering workerd. +export class ArtifactsController { + #backends = new Map>(); + // Keep the old roots alive until workerd accepts the replacement config. + #activeRoots = new Set(); + #pendingRoots?: Set; + + beginUpdate(): void { + if (this.#pendingRoots) { + throw new Error("Artifacts runtime update already in progress"); + } + this.#pendingRoots = new Set(); + } + + async commitUpdate(): Promise { + const roots = this.#pendingRoots; + if (!roots) { + throw new Error("No Artifacts runtime update to commit"); + } + this.#pendingRoots = undefined; + this.#activeRoots = roots; + await this.#closeExcept(roots); + } + + async abortUpdate(): Promise { + if (!this.#pendingRoots) { + return; + } + this.#pendingRoots = undefined; + await this.#closeExcept(this.#activeRoots); + } + + get(root: string): Promise { + if (!this.#pendingRoots) { + throw new Error("Artifacts backend requested outside runtime update"); + } + this.#pendingRoots.add(root); + let backend = this.#backends.get(root); + if (!backend) { + backend = this.#start(root, () => { + if (this.#backends.get(root) === backend) { + this.#backends.delete(root); + } + }); + this.#backends.set(root, backend); + void backend.catch(() => { + if (this.#backends.get(root) === backend) { + this.#backends.delete(root); + } + }); + } + return backend; + } + + async #start( + root: string, + onExit: () => void + ): Promise { + const port = await getPort({ host: "127.0.0.1" }); + // __dirname is dist/src in the bundled Miniflare entry point. + const worker = new Worker( + path.join(__dirname, "plugins/artifacts/git-sidecar.js"), + { workerData: { root } } + ); + try { + const endpoint = await waitForSidecarReady(worker); + let stopping = false; + worker.on("error", (error) => { + process.emitWarning( + `Local Artifacts Git backend failed: ${error.message}` + ); + }); + worker.once("exit", (code) => { + onExit(); + if (!stopping) { + process.emitWarning( + `Local Artifacts Git backend exited (${code}). Restart the dev server or reload its configuration to reconnect.` + ); + } + }); + return { + port, + sidecar: { + ...endpoint, + async close() { + if (worker.threadId === -1) { + return; + } + stopping = true; + const exited = once(worker, "exit"); + worker.postMessage("close"); + await exited; + }, + }, + }; + } catch (error) { + await worker.terminate(); + throw error; + } + } + + async #closeExcept(roots: Set): Promise { + const retired = [...this.#backends].filter(([root]) => !roots.has(root)); + for (const [root] of retired) { + this.#backends.delete(root); + } + const results = await Promise.allSettled( + retired.map(([, backend]) => + backend.then( + ({ sidecar }) => sidecar.close(), + () => undefined + ) + ) + ); + for (const result of results) { + if (result.status === "rejected") { + process.emitWarning("Unable to stop a retired Artifacts Git backend"); + } + } + } + + async dispose(): Promise { + this.#pendingRoots = undefined; + this.#activeRoots.clear(); + await this.#closeExcept(this.#activeRoots); + } +} + +type SidecarStartupMessage = + | Pick + | { error: string }; + +function waitForSidecarReady( + worker: Worker +): Promise> { + return new Promise((resolve, reject) => { + function cleanup() { + worker.off("message", ready); + worker.off("error", failed); + worker.off("exit", exited); + } + function ready(value: SidecarStartupMessage) { + cleanup(); + if ("error" in value) { + reject( + new Error( + `Local Artifacts Git backend failed to start: ${value.error}` + ) + ); + } else { + resolve(value); + } + } + function failed(error: Error) { + cleanup(); + reject(error); + } + function exited(code: number) { + failed( + new Error(`Local Artifacts Git backend exited during startup (${code})`) + ); + } + worker.on("message", ready); + worker.on("error", failed); + worker.on("exit", exited); + }); +} diff --git a/packages/miniflare/src/plugins/artifacts/git-client.ts b/packages/miniflare/src/plugins/artifacts/git-client.ts index 84e80277047..01060c97538 100644 --- a/packages/miniflare/src/plugins/artifacts/git-client.ts +++ b/packages/miniflare/src/plugins/artifacts/git-client.ts @@ -1,5 +1,5 @@ import { Buffer } from "node:buffer"; -import { spawn } from "node:child_process"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { lstat, mkdtemp, open, rename, unlink } from "node:fs/promises"; import { devNull, tmpdir } from "node:os"; @@ -12,6 +12,7 @@ type GitOptions = { allowFailure?: boolean; timeout?: number; httpAuthorization?: { origin: string; value: string }; + signal?: AbortSignal; }; type GitResult = { stdout: Buffer; stderr: Buffer; status: number }; @@ -60,6 +61,35 @@ export function gitEnvironment( }; } +/** + * Stop a Git invocation's process tree so HTTPS helpers cannot keep its pipes open. + * POSIX invocations own a process group; Windows requires taskkill /T. + */ +export function stopGitProcess(child: ChildProcessWithoutNullStreams): void { + if (process.platform === "win32" && child.pid !== undefined) { + // Git for Windows can spawn git-remote-https; killing only git leaves it alive. + const killer = spawn("taskkill", ["/PID", String(child.pid), "/T", "/F"], { + windowsHide: true, + stdio: "ignore", + }); + killer.once("error", () => child.kill()); + killer.unref(); + } else if (child.pid !== undefined) { + try { + process.kill(-child.pid, "SIGTERM"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") { + child.kill(); + } + } + } else { + child.kill(); + } + child.stdin.destroy(); + child.stdout.destroy(); + child.stderr.destroy(); +} + /** Require Git 2.32's GIT_CONFIG_GLOBAL to isolate native Git subprocesses. */ export function assertSupportedGitVersion(output: string): void { const match = /^git version (\d+)\.(\d+)(?:\.\d+)?(?:[.+ -][^\r\n]*)?$/.exec( @@ -123,10 +153,23 @@ export async function runGit( const child = spawn("git", gitArgumentsWithoutHooks(args), { env, stdio: ["pipe", "pipe", "pipe"], + detached: process.platform !== "win32", + signal: options.signal, }); + if (options.signal?.aborted) { + stopGitProcess(child); + } else { + options.signal?.addEventListener("abort", stop, { once: true }); + } + function stop() { + stopGitProcess(child); + } const stdout: Buffer[] = []; const stderr: Buffer[] = []; - const timeout = setTimeout(() => child.kill(), options.timeout ?? 300_000); + const timeout = setTimeout( + () => stopGitProcess(child), + options.timeout ?? 300_000 + ); timeout.unref(); child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); @@ -135,6 +178,7 @@ export async function runGit( reject(error); }); child.once("close", (status) => { + options.signal?.removeEventListener("abort", stop); clearTimeout(timeout); const result = { stdout: Buffer.concat(stdout), @@ -222,19 +266,28 @@ export async function withRepositoryCreationLock( /** Repository-scoped Git operations; clone/init are supported before it exists. */ export class GitClient { - constructor(readonly repository: string) {} + constructor( + readonly repository: string, + private readonly signal?: AbortSignal + ) {} + + /** Create a client for another repository with the same cancellation signal. */ + withRepository(repository: string): GitClient { + return new GitClient(repository, this.signal); + } private git(args: string[], options?: GitOptions): Promise { - return runGit(["-C", this.repository, ...args], options); + return runGit(["-C", this.repository, ...args], { + ...options, + signal: this.signal, + }); } async init(defaultBranch: string): Promise { - await runGit([ - "init", - "--bare", - `--initial-branch=${defaultBranch}`, - this.repository, - ]); + await runGit( + ["init", "--bare", `--initial-branch=${defaultBranch}`, this.repository], + { signal: this.signal } + ); } async forkFrom( @@ -245,6 +298,7 @@ export class GitClient { const sourceBranch = `refs/heads/${branch}`; const validBranch = await runGit(["check-ref-format", sourceBranch], { allowFailure: true, + signal: this.signal, }); if (validBranch.status !== 0) { throw new Error("Git fork source branch is invalid"); @@ -261,16 +315,19 @@ export class GitClient { { allowFailure: true } ); if (defaultBranchOnly && sourceHasBranch.status === 0) { - await runGit([ - "clone", - "--bare", - "--local", - "--single-branch", - "--branch", - branch, - source.repository, - this.repository, - ]); + await runGit( + [ + "clone", + "--bare", + "--local", + "--single-branch", + "--branch", + branch, + source.repository, + this.repository, + ], + { signal: this.signal } + ); } else if (defaultBranchOnly) { // Only an unborn HEAD on the requested branch permits an empty fork. // A typo or a missing source must never create a repository instead. @@ -285,13 +342,10 @@ export class GitClient { } await this.init(branch); } else { - await runGit([ - "clone", - "--bare", - "--local", - source.repository, - this.repository, - ]); + await runGit( + ["clone", "--bare", "--local", source.repository, this.repository], + { signal: this.signal } + ); } } @@ -334,10 +388,17 @@ export class GitClient { } args.push(source.url, staged); try { - await runGit(args, { httpAuthorization: source.authorization }); + await runGit(args, { + httpAuthorization: source.authorization, + signal: this.signal, + }); // A bare clone records remote.origin.url in its local Git config. It may // include credentials used only for the import. Never retain the remote. - await new GitClient(staged).git(["remote", "remove", "origin"]); + await new GitClient(staged, this.signal).git([ + "remote", + "remove", + "origin", + ]); await assertImportTargetAvailable(this.repository); await rename(staged, this.repository); } catch (error) { diff --git a/packages/miniflare/src/plugins/artifacts/git-sidecar.ts b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts new file mode 100644 index 00000000000..b70b32ab91a --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/git-sidecar.ts @@ -0,0 +1,661 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { mkdir, mkdtemp, readdir, rename, stat } from "node:fs/promises"; +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import path from "node:path"; +import { parentPort, workerData } from "node:worker_threads"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { z } from "zod"; +import { + assertGitAvailable, + GitClient, + gitArgumentsWithoutHooks, + gitEnvironment, + stopGitProcess, + withRepositoryCreationLock, +} from "./git-client"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "./storage"; +import type { RepositoryState } from "./git-client"; + +const adminPath = "/__local_artifacts__"; +const componentPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +type RepositoryLocks = Map>; +type InFlightGitRequest = { + path: string; + generation: string; + abort: () => void; +}; +const decoder = new TextDecoder(); + +if (parentPort) { + const port = parentPort; + void startGitSidecar(workerData.root).then( + (sidecar) => { + port.postMessage({ address: sidecar.address, secret: sidecar.secret }); + port.once("message", () => { + void sidecar.close().then( + () => port.close(), + () => port.close() + ); + }); + }, + (error: unknown) => { + port.postMessage({ + error: + error instanceof Error + ? error.message + : "Unable to start local Artifacts Git backend", + }); + port.close(); + } + ); +} + +export interface GitSidecar { + address: string; + secret: string; + close(): Promise; +} + +const adminBase = z.object({ + namespace: z.string().regex(componentPattern), + name: z.string().regex(componentPattern), +}); +const generatedRepo = adminBase.extend({ generation: z.string().min(1) }); +const AdminRequestSchema = z.discriminatedUnion("action", [ + generatedRepo.extend({ + action: z.literal("create"), + defaultBranch: z.string().optional(), + }), + generatedRepo.extend({ action: z.literal("delete") }), + generatedRepo.extend({ + action: z.literal("fork"), + sourceName: z.string().regex(componentPattern), + sourceBranch: z.string().min(1), + defaultBranchOnly: z.boolean().optional(), + }), + generatedRepo.extend({ + action: z.literal("import"), + sourceUrl: z.string().min(1), + branch: z.string().optional(), + depth: z.number().optional(), + }), + adminBase.extend({ action: z.literal("refs") }), + adminBase.extend({ action: z.literal("readBlob"), hash: z.string().min(1) }), + adminBase.extend({ action: z.literal("readTree"), hash: z.string().min(1) }), + adminBase.extend({ + action: z.literal("readCommit"), + hash: z.string().min(1), + }), + adminBase.extend({ + action: z.literal("file"), + ref: z.string().min(1), + path: z.string().min(1), + }), + adminBase.extend({ + action: z.literal("log"), + ref: z.string().optional(), + limit: z.number().optional(), + offset: z.number().optional(), + }), + generatedRepo.extend({ + action: z.literal("abortPush"), + requestId: z.uuid(), + }), +]); + +type AdminRequest = z.infer; +type GitAction = Exclude; +type CreateRequest = Extract; +type ForkRequest = Extract; +type ImportRequest = Extract; +type NewRepositoryRequest = CreateRequest | ForkRequest | ImportRequest; + +function createGitSecret(): string { + return randomBytes(32).toString("hex"); +} + +export async function startGitSidecar(root: string): Promise { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + await assertGitAvailable(); + await cleanupInterruptedRepositories(root); + const secret = createGitSecret(); + const shutdown = new AbortController(); + const repositoryLocks: RepositoryLocks = new Map(); + const inFlight = new Map(); + const server = createServer((request, response) => { + if (request.headers["x-local-artifacts-backend"] !== secret) { + response.writeHead(403).end("Forbidden"); + return; + } + void handleRequest( + root, + repositoryLocks, + inFlight, + request, + response, + shutdown.signal + ).catch((error: unknown) => { + if (response.headersSent) { + response.destroy(error instanceof Error ? error : undefined); + } else { + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end( + error instanceof Error ? error.message : "Native Git backend failed" + ); + } + }); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + // The listener outlives this function; ArtifactsController closes it on disposal. + // `using` here would close it before workerd could make any requests. + server.unref(); + const addressInfo = server.address(); + if (!addressInfo || typeof addressInfo === "string") { + throw new Error("Unable to start native Git backend"); + } + const address = `127.0.0.1:${addressInfo.port}`; + return { + address, + secret, + close() { + shutdown.abort(); + server.closeAllConnections(); + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +function parseGitPath(pathname: string): { + namespace: string; + repository: string; + suffix: string; +} | null { + const match = pathname.match(/^\/git\/([^/]+)\/([^/]+\.git)(\/.*)?$/); + if (!match?.[1] || !match[2]) { + return null; + } + const namespace = decodeURIComponent(match[1]); + const repository = decodeURIComponent(match[2].slice(0, -4)); + validateComponent(namespace, "namespace"); + validateComponent(repository, "repository"); + return { namespace, repository, suffix: match[3] ?? "" }; +} + +// Native Git's CGI backend needs streaming Node requests and responses; keep +// packfiles on node:http rather than buffering them through a Worker router. +async function handleRequest( + root: string, + locks: RepositoryLocks, + inFlight: Map, + request: IncomingMessage, + response: ServerResponse, + signal: AbortSignal +): Promise { + const url = new URL(request.url ?? "/", "http://local-artifacts.invalid"); + if (url.pathname === adminPath) { + await handleAdmin(root, locks, inFlight, request, response, signal); + return; + } + const gitPath = parseGitPath(url.pathname); + if (!gitPath) { + response.writeHead(404).end("Not found"); + return; + } + const { namespace, repository, suffix } = gitPath; + const contentLength = Number(request.headers["content-length"] ?? 0); + if (!Number.isFinite(contentLength) || contentLength > 256 * 1024 * 1024) { + response.writeHead(413).end("Request body too large"); + return; + } + const path = repositoryPath(root, namespace, repository); + await withRepositoryLocks(locks, [path], async () => { + const generation = request.headers["x-local-artifacts-generation"]; + const configured = await new GitClient(path, signal).generation(); + if (typeof generation !== "string" || generation !== configured) { + response.writeHead(409).end("Repository generation changed"); + return; + } + const requestId = request.headers["x-local-artifacts-request-id"]; + const trackPush = + request.method === "POST" && + suffix === "/git-receive-pack" && + typeof requestId === "string" && + /^[0-9a-f-]{36}$/.test(requestId); + try { + await runGitHttpBackend( + root, + request, + response, + `/${repositoryDirectory(namespace, repository)}${suffix}`, + url.search.slice(1), + trackPush + ? (abort) => + inFlight.set(requestId, { + path, + generation, + abort, + }) + : undefined, + signal + ); + } finally { + if (trackPush) { + inFlight.delete(requestId); + } + } + }); +} + +async function handleAdmin( + root: string, + locks: RepositoryLocks, + inFlight: Map, + request: IncomingMessage, + response: ServerResponse, + signal: AbortSignal +): Promise { + if (request.method !== "POST") { + response.writeHead(405).end("Method not allowed"); + return; + } + const body = AdminRequestSchema.parse( + JSON.parse(decoder.decode(await readRequest(request))) + ); + const repository = repositoryPath(root, body.namespace, body.name); + if (body.action === "abortPush") { + const active = inFlight.get(body.requestId); + if (active?.path === repository && active.generation === body.generation) { + active.abort(); + } + response.writeHead(200, { "Content-Type": "application/json" }); + response.end("true"); + return; + } + const lockPaths = [repository]; + if (body.action === "fork") { + lockPaths.push(repositoryPath(root, body.namespace, body.sourceName)); + } + await withRepositoryLocks(locks, lockPaths, async () => { + const result = await handleAction( + root, + body, + new GitClient(repository, signal) + ); + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(result)); + }); +} + +async function handleAction( + root: string, + body: GitAction, + git: GitClient +): Promise { + switch (body.action) { + case "create": + return handleCreate(root, body, git); + case "delete": + // A retried tombstone must never delete a replacement repository. + if ((await git.generation()) === body.generation) { + await retireRepository(git.repository); + } + return true; + case "fork": + return handleFork(root, body, git); + case "import": + return handleImport(root, body, git); + case "refs": + return git.state(); + case "readBlob": + return git.readBlob(body.hash); + case "readTree": + return git.readTree(body.hash); + case "readCommit": + return git.readCommit(body.hash); + case "file": + return git.readFile(body.ref, body.path); + case "log": + return git.log(body.ref, body.limit, body.offset); + } +} + +function handleCreate( + root: string, + body: CreateRequest, + git: GitClient +): Promise { + return withNewRepository(root, body, git, (staged) => + staged.init(body.defaultBranch ?? "main") + ); +} + +function handleFork( + root: string, + body: ForkRequest, + git: GitClient +): Promise { + const { sourceName, sourceBranch } = body; + const source = git.withRepository( + repositoryPath(root, body.namespace, sourceName) + ); + return withNewRepository(root, body, git, (staged) => + staged.forkFrom(source, sourceBranch, body.defaultBranchOnly ?? true) + ); +} + +function handleImport( + root: string, + body: ImportRequest, + git: GitClient +): Promise { + const { sourceUrl } = body; + return withNewRepository(root, body, git, (staged) => + staged.importFrom(sourceUrl, body.branch, body.depth) + ); +} + +async function withNewRepository( + root: string, + body: NewRepositoryRequest, + git: GitClient, + initialize: (staged: GitClient) => Promise +): Promise { + await mkdir(root, { recursive: true }); + return withRepositoryCreationLock(git.repository, async () => { + if (await exists(git.repository)) { + // The namespace has no record for this name. A prior deletion may have + // committed metadata before the process stopped removing Git data. + // Never remove a directory that was not created by this simulator. + if ((await git.generation()) === null) { + throw new Error(`Repository "${body.name}" already exists on disk`); + } + await retireRepository(git.repository); + } + // All initialization and cleanup happens under an owned staging directory. + // A failed clone or fork must not remove a repository created concurrently + // at the final path by another host process. + const stagingDirectory = await mkdtemp( + path.join(root, ".artifacts-stage-") + ); + const staged = git.withRepository(path.join(stagingDirectory, "repo.git")); + try { + await initialize(staged); + await staged.configure(body.generation); + const state = await staged.state(); + if (await exists(git.repository)) { + throw new Error(`Repository "${body.name}" already exists on disk`); + } + await rename(staged.repository, git.repository); + return state; + } finally { + await removeDir(stagingDirectory); + } + }); +} + +// Moving the directory out of its active path is atomic on the same volume. +// A crash while removing its contents leaves only an inaccessible trash path; +// the next sidecar startup retries cleanup without blocking name reuse. +async function retireRepository(repository: string): Promise { + const retired = `${repository}.deleting-${randomBytes(16).toString("hex")}`; + try { + await rename(repository, retired); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + await removeDir(retired); +} + +async function cleanupInterruptedRepositories(root: string): Promise { + const retired = /^[0-9a-f]{32}\.git\.deleting-[0-9a-f]{32}$/; + const partialImport = /^\.artifacts-import-[a-zA-Z0-9]{6}$/; + const partialCreation = /^\.artifacts-stage-[a-zA-Z0-9]{6}$/; + // Only staging and retired directories are safe to remove. A creation + // lock may still be held by another process, so never clear it here. + for (const entry of await readdir(root, { withFileTypes: true })) { + if ( + entry.isDirectory() && + (retired.test(entry.name) || + partialImport.test(entry.name) || + partialCreation.test(entry.name)) + ) { + await removeDir(path.join(root, entry.name)); + } + } +} + +function parseGitBackendHeaders(headerBytes: Buffer): { + status: number; + headers: Record; +} { + const lines = decoder.decode(headerBytes).split(/\r?\n/); + let status = 200; + const headers: Record = {}; + for (const line of lines) { + const separator = line.indexOf(":"); + if (separator < 0) { + continue; + } + const name = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + if (name.toLowerCase() === "status") { + status = Number(value.slice(0, 3)); + } else { + headers[name] = value; + } + } + return { status, headers }; +} + +async function runGitHttpBackend( + root: string, + request: IncomingMessage, + response: ServerResponse, + pathInfo: string, + query: string, + registerAbort?: (abort: () => void) => void, + signal?: AbortSignal +): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", gitArgumentsWithoutHooks(["http-backend"]), { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: root, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: pathInfo, + QUERY_STRING: query, + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + REMOTE_USER: "local-artifacts", + REMOTE_ADDR: request.socket.remoteAddress ?? "127.0.0.1", + }, + stdio: ["pipe", "pipe", "pipe"], + detached: process.platform !== "win32", + signal, + }); + const stop = () => stopGitProcess(child); + if (signal?.aborted) { + stop(); + } else { + signal?.addEventListener("abort", stop, { once: true }); + } + let header = Buffer.alloc(0); + let headersSent = false; + let settled = false; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + stop(); + }, 300_000); + timeout.unref(); + const stderr: Buffer[] = []; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + signal?.removeEventListener("abort", stop); + request.off("aborted", abort); + response.off("close", abort); + if (error) { + reject(error); + } else { + resolve(); + } + }; + const abort = () => { + if (!response.writableEnded) { + stop(); + } + }; + registerAbort?.(() => { + stop(); + response.destroy(); + }); + const write = (chunk: Buffer) => { + if (!response.write(chunk)) { + child.stdout.pause(); + response.once("drain", () => child.stdout.resume()); + } + }; + request.once("aborted", abort); + response.once("close", abort); + child.stdin.on("error", (error) => { + if (!request.destroyed) { + finish(error); + } + }); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.stdout.on("data", (chunk: Buffer) => { + if (headersSent) { + write(chunk); + return; + } + header = Buffer.concat([header, chunk]); + const crlf = header.indexOf("\r\n\r\n"); + const lf = header.indexOf("\n\n"); + const index = crlf >= 0 ? crlf : lf; + if (index < 0) { + if (header.length > 64 * 1024) { + stop(); + } + return; + } + const separatorLength = crlf >= 0 ? 4 : 2; + const { status, headers } = parseGitBackendHeaders( + header.subarray(0, index) + ); + response.writeHead(status, headers); + headersSent = true; + const remaining = header.subarray(index + separatorLength); + if (remaining.length) { + write(remaining); + } + header = Buffer.alloc(0); + }); + child.stdout.on("end", () => { + if (headersSent) { + response.end(); + } + }); + child.once("error", finish); + child.once("close", (status) => { + if (timedOut) { + finish(new Error("git http-backend timed out")); + return; + } + if (!headersSent) { + finish( + new Error( + decoder.decode(Buffer.concat(stderr)).trim() || + `git http-backend exited with status ${status}` + ) + ); + } else { + finish(); + } + }); + request.pipe(child.stdin); + }); +} + +function readRequest(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => resolve(Buffer.concat(chunks))); + request.once("error", reject); + }); +} + +function validateComponent(value: string, type: string): void { + if (!componentPattern.test(value)) { + throw new Error(`Invalid ${type}`); + } +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +} + +async function withRepositoryLocks( + repositoryLocks: RepositoryLocks, + paths: string[], + action: () => Promise +): Promise { + const keys = [...new Set(paths)].sort(); + const previous = keys.map( + (key) => repositoryLocks.get(key) ?? Promise.resolve() + ); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + for (const key of keys) { + repositoryLocks.set(key, current); + } + await Promise.all(previous); + try { + return await action(); + } finally { + release(); + for (const key of keys) { + if (repositoryLocks.get(key) === current) { + repositoryLocks.delete(key); + } + } + } +} diff --git a/packages/miniflare/src/plugins/artifacts/index.ts b/packages/miniflare/src/plugins/artifacts/index.ts index 48242920859..3824c0a9dce 100644 --- a/packages/miniflare/src/plugins/artifacts/index.ts +++ b/packages/miniflare/src/plugins/artifacts/index.ts @@ -1,31 +1,88 @@ +import { createHash } from "node:crypto"; +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import SCRIPT from "worker:artifacts/binding"; +import { ARTIFACTS_EVENT_SERVICE_NAME } from "../../workers/queues/constants"; import { buildRemoteProxyProps, getEnvBindingsOfType, + getPersistPath, getRemoteProxyConnectionString, ProxyNodeBinding, remoteProxyClientWorker, } from "../shared"; -import type { Plugin } from "../shared"; +import { assertGitAvailable } from "./git-client"; +import { assertSupportedGitLayout } from "./storage"; +import type { Service, Socket } from "../../runtime"; +import type { MiniflareBinding, Plugin } from "../shared"; +import type { ArtifactsController } from "./controller"; + +export { ArtifactsController, type LocalArtifactsBackend } from "./controller"; +export type { GitSidecar } from "./git-sidecar"; export const ARTIFACTS_PLUGIN_NAME = "artifacts"; -// One shared remote-proxy service for every artifacts binding; per-binding -// config travels via props. const ARTIFACTS_REMOTE_SERVICE_NAME = `${ARTIFACTS_PLUGIN_NAME}:remote`; +const ENTRYPOINT = "LocalArtifactsNamespace"; +const OBJECT_CLASS = "LocalArtifactsNamespaceObject"; +const LOCAL_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +type ArtifactsBinding = Extract; +type ArtifactsEntry = [name: string, binding: ArtifactsBinding]; + +function isLocalArtifactsBinding(binding: ArtifactsBinding): boolean { + return binding.dev?.remote !== true; +} + +function getLocalNamespaceNames(bindings: ArtifactsEntry[]): Set { + return new Set( + bindings + .filter(([, binding]) => isLocalArtifactsBinding(binding)) + .map(([, binding]) => binding.namespace) + ); +} + +function validateLocalNamespace(namespace: string): void { + if (!LOCAL_NAMESPACE_PATTERN.test(namespace)) { + throw new Error(`Invalid local Artifacts namespace: ${namespace}`); + } +} + +function namespaceId(namespace: string): string { + // 128 bits keeps storage paths short enough for workerd's SQLite files on Windows. + return createHash("sha256").update(namespace).digest("hex").slice(0, 32); +} + +function localServiceName(namespace: string): string { + return `${ARTIFACTS_PLUGIN_NAME}-${namespaceId(namespace)}`; +} export const ARTIFACTS_PLUGIN: Plugin = { bindingTypeDescription: "Artifacts", - async getBindings(options) { + getBindings(options) { return getEnvBindingsOfType(options.config, "artifacts").map( - ([name, binding]) => ({ - name, - service: { - name: ARTIFACTS_REMOTE_SERVICE_NAME, - props: buildRemoteProxyProps( - getRemoteProxyConnectionString(binding, options.dev), - name - ), - }, - }) + ([name, binding]) => { + // Omitted remote and false use local storage; true opts into the remote proxy. + if (isLocalArtifactsBinding(binding)) { + validateLocalNamespace(binding.namespace); + return { + name, + service: { + name: localServiceName(binding.namespace), + entrypoint: ENTRYPOINT, + }, + }; + } + return { + name, + service: { + name: ARTIFACTS_REMOTE_SERVICE_NAME, + props: buildRemoteProxyProps( + getRemoteProxyConnectionString(binding, options.dev), + name + ), + }, + }; + } ); }, getNodeBindings(options) { @@ -36,16 +93,113 @@ export const ARTIFACTS_PLUGIN: Plugin = { ]) ); }, - async getServices({ options }) { - if (getEnvBindingsOfType(options.config, "artifacts").length === 0) { - return []; - } - - return [ - { + async getServices({ options, sharedOptions, tmpPath, artifactsController }) { + const bindings = getEnvBindingsOfType(options.config, "artifacts"); + const services: Service[] = []; + const sockets: Socket[] = []; + if (bindings.some(([, binding]) => !isLocalArtifactsBinding(binding))) { + services.push({ name: ARTIFACTS_REMOTE_SERVICE_NAME, worker: remoteProxyClientWorker(), - }, - ]; + }); + } + const namespaces = getLocalNamespaceNames(bindings); + if (namespaces.size > 0) { + await assertGitAvailable(); + } + const root = getPersistPath( + ARTIFACTS_PLUGIN_NAME, + tmpPath, + sharedOptions.resourcePersistencePath + ); + + for (const namespace of namespaces) { + const local = await createLocalNamespaceRuntime( + root, + namespace, + artifactsController + ); + sockets.push(local.socket); + services.push(...local.services); + } + return { services, extensions: [], sockets }; }, }; + +async function createLocalNamespaceRuntime( + root: string, + namespace: string, + artifactsController: ArtifactsController +): Promise<{ services: Service[]; socket: Socket }> { + await mkdir(root, { recursive: true }); + await assertSupportedGitLayout(root); + const persistPath = path.join(root, namespaceId(namespace)); + const metadataPath = path.join(persistPath, "metadata").replaceAll("\\", "/"); + await mkdir(metadataPath, { recursive: true }); + await assertSupportedGitLayout(metadataPath); + const { sidecar, port } = await artifactsController.get( + path.join(persistPath, "git") + ); + const serviceName = localServiceName(namespace); + const storageName = `${serviceName}-storage`; + const backendName = `${serviceName}-backend`; + const socket: Socket = { + name: `${serviceName}-git`, + address: `127.0.0.1:${port}`, + http: {}, + service: { name: serviceName, entrypoint: ENTRYPOINT }, + }; + const storageService: Service = { + name: storageName, + disk: { path: metadataPath, writable: true }, + }; + const gitBackendService: Service = { + name: backendName, + external: { + address: sidecar.address, + http: { + injectRequestHeaders: [ + { name: "X-Local-Artifacts-Backend", value: sidecar.secret }, + ], + }, + }, + }; + const bindingService: Service = { + name: serviceName, + worker: { + compatibilityDate: "2026-09-03", + modules: [{ name: "binding.worker.js", esModule: SCRIPT() }], + bindings: [ + { + name: "config", + json: JSON.stringify({ + namespace, + origin: `http://127.0.0.1:${port}`, + }), + }, + { + name: "localArtifactsNamespace", + durableObjectNamespace: { className: OBJECT_CLASS }, + }, + { name: "gitBackend", service: { name: backendName } }, + { + name: "QUEUE_WORKER", + service: { + name: ARTIFACTS_EVENT_SERVICE_NAME, + entrypoint: "ArtifactsEventProducer", + }, + }, + ], + // Keys must be unique across namespaces in one workerd, but should + // not repeat the long service prefix in SQLite's directory name. + durableObjectNamespaces: [ + { className: OBJECT_CLASS, uniqueKey: namespaceId(namespace) }, + ], + durableObjectStorage: { localDisk: storageName }, + }, + }; + return { + services: [storageService, gitBackendService, bindingService], + socket, + }; +} diff --git a/packages/miniflare/src/plugins/artifacts/storage.ts b/packages/miniflare/src/plugins/artifacts/storage.ts new file mode 100644 index 00000000000..ac693dc1bcc --- /dev/null +++ b/packages/miniflare/src/plugins/artifacts/storage.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { readdir } from "node:fs/promises"; +import path from "node:path"; + +/** + * Keep repository paths short: native Git adds long pack and lock filenames. + * Hash the tuple rather than nesting namespace and repository hashes. Each + * identifier retains 128 bits and no user input becomes a path component. + */ +export function repositoryDirectory(namespace: string, name: string): string { + const id = createHash("sha256") + .update(JSON.stringify([namespace, name])) + .digest("hex") + .slice(0, 32); + return `${id}.git`; +} + +export function repositoryPath( + root: string, + namespace: string, + name: string +): string { + return path.join(root, repositoryDirectory(namespace, name)); +} + +/** Do not quietly abandon repositories written by earlier unreleased layouts. */ +export async function assertSupportedGitLayout(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + if ( + entries.some( + (entry) => + entry.isDirectory() && + (/^[0-9a-f]{64}$/.test(entry.name) || /^artifacts[-:]/.test(entry.name)) + ) + ) { + throw new Error( + "Local Artifacts storage uses an older draft layout. Back up your local repositories, then reset the Artifacts persistence directory before restarting." + ); + } +} diff --git a/packages/miniflare/src/plugins/queues/index.ts b/packages/miniflare/src/plugins/queues/index.ts index bb40c6b5ef5..32df66ee923 100644 --- a/packages/miniflare/src/plugins/queues/index.ts +++ b/packages/miniflare/src/plugins/queues/index.ts @@ -1,4 +1,5 @@ import SCRIPT_QUEUE_BROKER_OBJECT from "worker:queues/broker"; +import SCRIPT_ARTIFACTS_EVENT_PRODUCER from "worker:queues/event-producer"; import { kVoid } from "../../runtime"; import { getQueueServiceName, @@ -6,6 +7,7 @@ import { SERVICE_QUEUE_PREFIX, SharedBindings, } from "../../workers"; +import { ARTIFACTS_EVENT_SERVICE_NAME } from "../../workers/queues/constants"; import { getUserServiceName } from "../core"; import { getEnvBindingsOfType, @@ -58,8 +60,12 @@ export const QUEUES_PLUGIN: Plugin = { const consumed = getTriggersOfType(options.config, "queue").map( (trigger) => trigger.name ); + const hasLocalArtifacts = getEnvBindingsOfType( + options.config, + "artifacts" + ).some(([, binding]) => binding.dev?.remote !== true); const queueIds = new Set([...produced, ...consumed]); - if (queueIds.size === 0) { + if (queueIds.size === 0 && !hasLocalArtifacts) { return []; } @@ -67,6 +73,23 @@ export const QUEUES_PLUGIN: Plugin = { name: getQueueServiceName(id), worker: objectEntryWorker(QUEUE_BROKER_OBJECT, id), })); + if (hasLocalArtifacts) { + services.push({ + name: ARTIFACTS_EVENT_SERVICE_NAME, + worker: { + compatibilityDate: "2026-09-03", + modules: [ + { + name: "event-producer.worker.js", + esModule: SCRIPT_ARTIFACTS_EVENT_PRODUCER(), + }, + ], + }, + }); + } + if (queueIds.size === 0) { + return services; + } const uniqueKey = `miniflare-${QUEUE_BROKER_OBJECT_CLASS_NAME}`; const objectService: Service = { diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 9c0658a0c3b..578f1a1deca 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -12,6 +12,7 @@ import type { Extension, Service, ServiceDesignator, + Socket, Worker_Binding, Worker_Module, } from "../../runtime"; @@ -21,6 +22,7 @@ import type { QueueConsumerSchema, QueueProducerSchema, } from "../../workers"; +import type { ArtifactsController } from "../artifacts/controller"; import type { ContainerPrivilegesCache } from "../core/container"; import type { DOContainerOptions } from "../do"; import type { HyperdriveProxyController } from "../hyperdrive/hyperdrive-proxy"; @@ -88,11 +90,14 @@ export interface PluginServicesOptions { devRegistryEnabled: boolean; containerPrivilegesCache: ContainerPrivilegesCache; hyperdriveProxyController: HyperdriveProxyController; + artifactsController: ArtifactsController; } export interface ServicesExtensions { services: Service[]; extensions: Extension[]; + // Internal plugin listeners (e.g. Git smart HTTP), deduplicated by name. + sockets?: Socket[]; } /** diff --git a/packages/miniflare/src/workers/artifacts/binding.worker.ts b/packages/miniflare/src/workers/artifacts/binding.worker.ts new file mode 100644 index 00000000000..5898275f2ed --- /dev/null +++ b/packages/miniflare/src/workers/artifacts/binding.worker.ts @@ -0,0 +1,1456 @@ +import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; +import { V1, type ArtifactsEvent, type RepoSnapshot } from "./events"; +import type { ArtifactsEventProducer } from "../queues/event-producer.worker"; + +const repoName = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +type TokenScope = "read" | "write"; + +type RepositorySource = + | { type: "fork"; repository: string } + | { type: "remote"; url: string }; + +interface RepositoryOptions { + description?: string | undefined; + readOnly?: boolean; + setDefaultBranch?: string; +} + +interface ForkOptions { + description?: string; + readOnly?: boolean; + defaultBranchOnly?: boolean; +} + +interface ImportRepositoryOptions { + source: { url: string; branch?: string; depth?: number }; + target: { name: string; opts?: { description?: string; readOnly?: boolean } }; +} + +interface RepositoryToken { + id: string; + tokenHash: string; + scope: TokenScope; + createdAt: string; + expiresAt: string; + revokedAt?: string; +} + +interface CreatedToken { + record: RepositoryToken; + plaintext: string; +} + +interface Repository { + id: string; + name: string; + namespace: string; + description?: string | undefined; + readOnly: boolean; + defaultBranch: string; + remote: string; + source?: RepositorySource | undefined; + createdAt: string; + updatedAt: string; + lastPushAt?: string | undefined; + // A failed native deletion stays retryable without exposing the repository. + deleting?: boolean; + tokens: RepositoryToken[]; +} + +interface RepositoryMetadata { + id: string; + name: string; + remote: string; + defaultBranch: string; + description: string | null; + source: string | null; + readOnly: boolean; + createdAt: string; + updatedAt: string; + lastPushAt: string | null; +} + +interface CreateRepositoryResult { + id: string; + name: string; + description: string | null; + defaultBranch: string; + remote: string; + token: string; +} + +interface CreateTokenResult { + id: string; + plaintext: string; + scope: TokenScope; + expiresAt: string; +} +type TokenInfo = Omit & { + state: "active" | "expired" | "revoked"; +}; + +interface RepositoryListOptions { + cursor?: string; + limit?: number; +} + +type RepositoryListResult = { + repos: Array< + RepositoryMetadata & { status: "ready"; jurisdiction: "unrestricted" } + >; + total: number; + cursor?: string; +}; + +interface RepositoryLogOptions { + ref?: string; + limit?: number; + offset?: number; +} + +interface BackendState { + defaultBranch: string; + refs: Record; +} + +type BackendRequest = + | { action: "create"; defaultBranch: string } + | { action: "delete" } + | { + action: "fork"; + sourceName: string; + sourceBranch: string; + defaultBranchOnly?: boolean; + } + | { action: "import"; sourceUrl: string; branch?: string; depth?: number } + | { action: "refs" } + | { action: "abortPush"; requestId: string } + | { action: "readBlob"; hash: string } + | { action: "readTree"; hash: string } + | { action: "readCommit"; hash: string } + | { action: "file"; ref: string; path: string } + | { action: "log"; ref?: string; limit?: number; offset?: number }; + +interface TreeEntry { + name: string; + mode: string; + hash: string; + type: "tree" | "blob" | "symlink" | "gitlink" | "exec"; +} + +interface CommitMetadata { + hash: string; + treeHash: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + parents: string[]; + authoredAt: number; + committedAt: number; +} + +interface LocalArtifactsRepo { + info(): Promise; + createToken(scope?: TokenScope, ttl?: number): Promise; + listTokens(): Promise<{ total: number; tokens: TokenInfo[] }>; + revokeToken(tokenOrId: string): Promise; + readBlob(hash: string): Promise; + readTree(hash: string): Promise; + readCommit(hash: string): Promise; + readFile(args: { ref: string; path: string }): Promise; + log(options?: RepositoryLogOptions): Promise; + fork(name: string, options?: ForkOptions): Promise; +} + +interface BackendBlob { + data: string; +} + +type ArtifactsErrorCode = + | "ALREADY_EXISTS" + | "NOT_FOUND" + | "INVALID_INPUT" + | "INVALID_REPO_NAME" + | "INVALID_TTL" + | "INVALID_URL" + | "REMOTE_AUTH_REQUIRED" + | "MEMORY_LIMIT" + | "UPSTREAM_UNAVAILABLE" + | "INTERNAL_ERROR"; + +const numericCodes: Record = { + INVALID_INPUT: 10100, + INVALID_REPO_NAME: 10101, + INVALID_TTL: 10103, + INVALID_URL: 10104, + REMOTE_AUTH_REQUIRED: 10106, + NOT_FOUND: 10200, + ALREADY_EXISTS: 10201, + INTERNAL_ERROR: 10400, + UPSTREAM_UNAVAILABLE: 10401, + MEMORY_LIMIT: 10402, +}; + +class ArtifactsError extends Error { + readonly code: ArtifactsErrorCode; + readonly numericCode: number; + + constructor(code: ArtifactsErrorCode, message: string) { + super(message); + this.name = "ArtifactsError"; + this.code = code; + this.numericCode = numericCodes[code]; + } +} + +interface LocalArtifactsEnvironment { + config: { namespace: string; origin: string }; + localArtifactsNamespace: DurableObjectNamespace; + gitBackend: Fetcher; + QUEUE_WORKER: Service; +} + +type RepositoryMap = Record; + +export class LocalArtifactsNamespace extends WorkerEntrypoint { + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return publicCall(() => this.#namespace().create(name, options)); + } + + async get(name: string): Promise { + const repository = await publicCall(() => this.#namespace().get(name)); + return repository as unknown as LocalArtifactsRepo; + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return publicCall(() => this.#namespace().import(options)); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + return publicCall(() => this.#namespace().list(options)); + } + + async delete(name: string): Promise { + return publicCall(() => this.#namespace().delete(name)); + } + + override async fetch(request: Request): Promise { + return this.#namespace().fetch(request); + } + + #namespace(): DurableObjectStub { + const { localArtifactsNamespace, config } = this.env; + return localArtifactsNamespace.get( + localArtifactsNamespace.idFromName(config.namespace) + ); + } +} + +export class LocalArtifactsNamespaceObject extends DurableObject { + // The resolved tail lets the first mutation run immediately. exclusive() + // serializes subsequent metadata and backend mutations across awaits. + private operations = Promise.resolve(); + + async create( + name: string, + options: RepositoryOptions = {} + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + await this.finishDeletion(repositories, name); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + + const repository = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + try { + const state = await this.backend(repository, { + action: "create", + defaultBranch: repository.defaultBranch, + }); + applyBackendState(repository, state); + repositories[name.toLowerCase()] = repository; + const token = await createToken(repository, "write"); + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist Artifacts repository" + ); + } + this.publish( + V1.repoCreated({ + repo: eventRepoSnapshot(repository), + }) + ); + this.publish( + V1.tokenCreated({ + repo: { namespace: repository.namespace, name: repository.name }, + token: eventToken(token.record), + }) + ); + return createRepoResult(repository, token); + } catch (error) { + if (error instanceof ArtifactsError) { + throw error; + } + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to create Artifacts repository" + ); + } + }); + } + + async get(name: string): Promise { + validateName(name); + const repository = await this.getRepo(name); + return createRepoHandle(this, repository.name, repository.id); + } + + async import( + options: ImportRepositoryOptions + ): Promise { + return this.exclusive(async () => { + const { source, target, sourceUrl } = parseImportRequest(options); + const repositories = await this.repos(); + await this.finishDeletion(repositories, target.name); + if (hasRepository(repositories, target.name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${target.name}` + ); + } + validateRepositoryOptions(target.opts ?? {}); + const repository = makeRepo( + target.name, + this.env.config.namespace, + this.env.config.origin, + target.opts ?? {} + ); + // The clone URL may contain one-time credentials. Keep them out of + // repository metadata returned by info() and persisted in storage. + const publicSourceUrl = new URL(sourceUrl); + publicSourceUrl.username = ""; + publicSourceUrl.password = ""; + publicSourceUrl.search = ""; + publicSourceUrl.hash = ""; + repository.source = { type: "remote", url: publicSourceUrl.href }; + const state = await this.backend(repository, { + action: "import", + sourceUrl: sourceUrl.href, + ...(source.branch === undefined ? {} : { branch: source.branch }), + ...(source.depth === undefined ? {} : { depth: source.depth }), + }); + applyBackendState(repository, state); + repository.updatedAt = new Date().toISOString(); + const token = await createToken(repository, "write"); + repositories[repository.name.toLowerCase()] = repository; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(repository); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist imported Artifacts repository" + ); + } + this.publish( + V1.repoImported({ + repo: eventRepoSnapshot(repository), + importedFrom: { + url: publicSourceUrl.href, + branch: source.branch ?? null, + }, + }) + ); + this.publish( + V1.tokenCreated({ + repo: { namespace: repository.namespace, name: repository.name }, + token: eventToken(token.record), + }) + ); + return createRepoResult(repository, token); + }); + } + + async list( + options: RepositoryListOptions = {} + ): Promise { + const limit = options.limit ?? 50; + if (!Number.isInteger(limit) || limit < 1 || limit > 200) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 200." + ); + } + const offset = decodeCursor(options.cursor); + const repositories = Object.values(await this.repos()) + .filter((repository) => !repository.deleting) + .sort((left, right) => right.createdAt.localeCompare(left.createdAt)); + const page = repositories + .slice(offset, offset + limit) + .map((repository) => ({ + ...repoMetadata(repository), + status: "ready" as const, + // Local storage has no placement jurisdiction; this is the public default. + jurisdiction: "unrestricted" as const, + })); + const cursor = + offset + page.length < repositories.length + ? encodeCursor(offset + page.length) + : undefined; + return { + repos: page, + total: repositories.length, + ...(cursor === undefined ? {} : { cursor }), + }; + } + + async delete(name: string): Promise { + return this.exclusive(async () => { + validateName(name); + const repositories = await this.repos(); + const key = name.toLowerCase(); + if (!Object.hasOwn(repositories, key)) { + return false; + } + const repository = repositories[key]; + // Persist a tombstone before Git cleanup. A storage failure leaves the + // live repository intact; a backend failure remains retryable by delete + // or the next creation of this name, including after a process restart. + if (!repository.deleting) { + repository.deleting = true; + await this.saveRepos(repositories); + } + await this.finishDeletion(repositories, name); + return true; + }); + } + + async createToken( + name: string, + scope: TokenScope = "write", + ttl?: number, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const token = await createToken(repository, scope, ttl); + await this.saveRepos(repositories); + this.publish( + V1.tokenCreated({ + repo: { namespace: repository.namespace, name: repository.name }, + token: eventToken(token.record), + }) + ); + return tokenResult(token); + }); + } + + async listTokens( + name: string, + expectedId?: string + ): Promise<{ total: number; tokens: TokenInfo[] }> { + const repository = await this.getRepo(name, expectedId); + const active = repository.tokens.filter( + (token) => !token.revokedAt && isActive(token) + ); + const tokens = active + .toReversed() + .slice(0, 30) + .map(({ tokenHash: _tokenHash, revokedAt: _revokedAt, ...token }) => ({ + ...token, + state: "active" as const, + })); + return { total: active.length, tokens }; + } + + async revokeToken( + name: string, + tokenOrId: string, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + if (!isTokenId(tokenOrId) && !isArtifactToken(tokenOrId)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid tokenOrId: must be a token id or artifact token." + ); + } + const repositories = await this.repos(); + const repository = getRepo(repositories, name, expectedId); + const tokenHash = isArtifactToken(tokenOrId) + ? await hashToken(tokenOrId) + : undefined; + const token = repository.tokens.find( + (candidate) => + candidate.id === tokenOrId || + (tokenHash !== undefined && + constantTimeEqual(candidate.tokenHash, tokenHash)) + ); + if (!token || token.revokedAt) { + return false; + } + token.revokedAt = new Date().toISOString(); + await this.saveRepos(repositories); + this.publish( + V1.tokenRevoked({ + repo: { namespace: repository.namespace, name: repository.name }, + token: { id: token.id }, + }) + ); + return true; + }); + } + + async fork( + sourceName: string, + name: string, + options: ForkOptions = {}, + expectedId?: string + ): Promise { + return this.exclusive(async () => { + validateName(name); + validateRepositoryOptions(options); + const repositories = await this.repos(); + const source = getRepo(repositories, sourceName, expectedId); + await this.finishDeletion(repositories, name); + if (hasRepository(repositories, name)) { + throw new ArtifactsError( + "ALREADY_EXISTS", + `repo already exists: ${name}` + ); + } + const fork = makeRepo( + name, + this.env.config.namespace, + this.env.config.origin, + options + ); + let state: BackendState; + try { + state = await this.backend(fork, { + action: "fork", + sourceName: source.name, + sourceBranch: source.defaultBranch, + ...(options.defaultBranchOnly === undefined + ? {} + : { defaultBranchOnly: options.defaultBranchOnly }), + }); + } catch { + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to fork Artifacts repository" + ); + } + applyBackendState(fork, state); + fork.source = { type: "fork", repository: source.name }; + const token = await createToken(fork, "write"); + repositories[name.toLowerCase()] = fork; + try { + await this.saveRepos(repositories); + } catch { + await this.deleteBackend(fork); + throw new ArtifactsError( + "INTERNAL_ERROR", + "Unable to persist forked Artifacts repository" + ); + } + this.publish( + V1.repoForked({ + repo: eventRepoSnapshot(fork), + forkedFrom: { namespace: source.namespace, name: source.name }, + }) + ); + this.publish( + V1.tokenCreated({ + repo: { namespace: fork.namespace, name: fork.name }, + token: eventToken(token.record), + }) + ); + return createRepoResult(fork, token); + }); + } + + override async fetch(request: Request): Promise { + return handleGitRequest( + this, + this.env.config.namespace, + request, + (promise) => this.ctx.waitUntil(promise) + ); + } + + async repos(): Promise { + const repositories = + (await this.ctx.storage.get("repos")) ?? + (Object.create(null) as RepositoryMap); + // The local Git listener can move after a process restart. + for (const repository of Object.values(repositories)) { + repository.remote = `${this.env.config.origin}/git/${repository.namespace}/${repository.name}.git`; + } + return repositories; + } + + saveRepos(repositories: RepositoryMap): Promise { + return this.ctx.storage.put("repos", repositories); + } + + private async finishDeletion( + repositories: RepositoryMap, + name: string + ): Promise { + const key = name.toLowerCase(); + if (!Object.hasOwn(repositories, key)) { + return; + } + const repository = repositories[key]; + if (!repository.deleting) { + return; + } + await this.backend(repository, { action: "delete" }); + delete repositories[key]; + await this.saveRepos(repositories); + this.publish( + V1.repoDeleted({ + repo: eventRepoSnapshot(repository), + }) + ); + } + + async getRepo(name: string, expectedId?: string): Promise { + return getRepo(await this.repos(), name, expectedId); + } + + async backend( + repository: Repository, + request: BackendRequest + ): Promise { + const response = await this.env.gitBackend.fetch( + "http://local-artifacts-backend/__local_artifacts__", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ...request, + namespace: repository.namespace, + name: repository.name, + generation: repository.id, + }), + } + ); + if (!response.ok) { + const message = await response.text(); + if (request.action === "import") { + throw classifyImportError(message); + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } + return response.json(); + } + + private async deleteBackend(repository: Repository): Promise { + try { + await this.backend(repository, { action: "delete" }); + } catch { + // Preserve the original persistence error; native delete is idempotent on retry. + } + } + + gitFetch( + repository: Repository, + request: Request, + requestId?: string + ): Promise { + const headers = new Headers(request.headers); + headers.delete("Authorization"); + headers.delete("X-Local-Artifacts-Request-Id"); + headers.set("X-Local-Artifacts-Generation", repository.id); + if (requestId) { + headers.set("X-Local-Artifacts-Request-Id", requestId); + } + const url = new URL(request.url); + url.pathname = url.pathname.replace( + /^\/git\/[^/]+\/[^/]+\.git/, + `/git/${repository.namespace}/${repository.name}.git` + ); + return this.env.gitBackend.fetch( + new Request(url, new Request(request, { headers })) + ); + } + + async syncAfterPush( + name: string, + id: string, + previousRefs: Record + ): Promise { + await this.exclusive(async () => { + const repositories = await this.repos(); + const repository = getRepo(repositories, name, id); + const state = await this.backend(repository, { + action: "refs", + }); + if (JSON.stringify(state.refs) !== JSON.stringify(previousRefs)) { + repository.updatedAt = new Date().toISOString(); + repository.lastPushAt = repository.updatedAt; + } + await this.saveRepos(repositories); + }); + } + + private publish(event: ArtifactsEvent): void { + // Committed mutations must not fail because local event delivery failed. + this.ctx.waitUntil( + this.env.QUEUE_WORKER.sendEvent(event).catch(() => { + console.warn("Unable to publish local Artifacts event"); + }) + ); + } + + private async exclusive(action: () => Promise): Promise { + const previous = this.operations; + let release!: () => void; + this.operations = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await action(); + } finally { + release(); + } + } +} + +function eventRepoSnapshot(repository: Repository): RepoSnapshot { + return { + id: repository.id, + namespace: repository.namespace, + name: repository.name, + jurisdiction: "unrestricted", + defaultBranch: repository.defaultBranch, + description: repository.description ?? null, + readOnly: repository.readOnly, + createdAt: repository.createdAt, + updatedAt: repository.updatedAt, + lastPushAt: repository.lastPushAt ?? null, + }; +} + +function eventToken(token: RepositoryToken): { + id: string; + scope: string; + expiresAt: string; +} { + return { id: token.id, scope: token.scope, expiresAt: token.expiresAt }; +} + +function createRepoHandle( + namespace: LocalArtifactsNamespaceObject, + name: string, + id: string +): LocalArtifactsRepo { + const read = (request: BackendRequest): Promise => + publicCall(async () => + namespace.backend(await namespace.getRepo(name, id), request) + ); + const readFile = async (args: { + ref: string; + path: string; + }): Promise => { + if (!args?.ref) { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: ref required."); + } + if (!args?.path) { + throw new ArtifactsError("INVALID_INPUT", "Invalid path: path required."); + } + const path = normalizePath(args.path); + if (path === null) { + return null; + } + const result = await read({ + action: "file", + ref: args.ref, + path, + }); + if (result === null) { + return null; + } + const bytes = decodeBase64(result.data); + return new Blob([bytes], { type: contentType(args.path, bytes) }); + }; + + return { + info: () => + publicCall(async () => repoMetadata(await namespace.getRepo(name, id))), + createToken: (scope: TokenScope = "write", ttl?: number) => + publicCall(() => namespace.createToken(name, scope, ttl, id)), + listTokens: () => publicCall(() => namespace.listTokens(name, id)), + revokeToken: (tokenOrId: string) => + publicCall(() => namespace.revokeToken(name, tokenOrId, id)), + readBlob: async (hash: string) => { + validateHash(hash); + const result = await read({ + action: "readBlob", + hash, + }); + return result === null ? null : new Blob([decodeBase64(result.data)]); + }, + readTree: async (hash: string) => { + validateHash(hash); + return await read({ action: "readTree", hash }); + }, + readCommit: async (hash: string) => { + validateHash(hash); + return await read({ action: "readCommit", hash }); + }, + readFile, + log: (options: RepositoryLogOptions = {}) => { + validateLogOptions(options); + return read({ + action: "log", + ...(options.ref === undefined ? {} : { ref: options.ref }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.offset === undefined ? {} : { offset: options.offset }), + }); + }, + fork: (targetName: string, options: ForkOptions = {}) => + publicCall(() => namespace.fork(name, targetName, options, id)), + }; +} + +function parseGitRoute( + request: Request, + namespace: string +): { name: string; write: boolean; push: boolean } | null { + const url = new URL(request.url); + const match = url.pathname.match(/^\/git\/([^/]+)\/([^/]+)\.git(?:\/(.*))?$/); + if (!match || match[1] !== namespace || !match[2]) { + return null; + } + const path = match[3] ?? ""; + const service = url.searchParams.get("service"); + const discovery = + request.method === "GET" && + path === "info/refs" && + (service === "git-upload-pack" || service === "git-receive-pack"); + const pack = + request.method === "POST" && + (path === "git-upload-pack" || path === "git-receive-pack"); + if (!discovery && !pack) { + return null; + } + return { + name: match[2], + write: path === "git-receive-pack" || service === "git-receive-pack", + push: request.method === "POST" && path === "git-receive-pack", + }; +} + +async function handleGitRequest( + state: LocalArtifactsNamespaceObject, + namespace: string, + request: Request, + keepAlive: (promise: Promise) => void +): Promise { + const route = parseGitRoute(request, namespace); + if (!route) { + return new Response("Not found", { status: 404 }); + } + let repository: Repository; + try { + repository = await state.getRepo(route.name); + } catch (error) { + if (error instanceof ArtifactsError && error.code === "NOT_FOUND") { + return new Response("Not found", { status: 404 }); + } + throw error; + } + const authorizationError = await authorize( + repository, + request, + route.write ? "write" : "read" + ); + if (authorizationError) { + return authorizationError; + } + if (route.write && repository.readOnly) { + return new Response("Artifacts repository is read-only", { status: 403 }); + } + + const previousRefs = route.push + ? (await state.backend(repository, { action: "refs" })).refs + : undefined; + const requestId = route.push ? crypto.randomUUID() : undefined; + const response = await state.gitFetch(repository, request, requestId); + if (!previousRefs || !requestId) { + return response; + } + + if (!response.ok) { + return response; + } + const syncRefs = async () => { + try { + await state.syncAfterPush(repository.name, repository.id, previousRefs); + } catch { + // Git has already accepted the push; metadata bookkeeping must not fail the client request. + } + }; + if (!response.body) { + await syncRefs(); + return response; + } + // A canceled response may leave the sidecar blocked on its output socket. + // Stop only this push, then synchronize after its repository lock releases. + const reader = response.body.getReader(); + let canceled = false; + let synchronization: Promise | undefined; + const syncOnce = () => (synchronization ??= syncRefs()); + const body = new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + await syncOnce(); + if (!canceled) { + controller.close(); + } + } else if (!canceled) { + controller.enqueue(value); + } + } catch (error) { + keepAlive(syncOnce()); + if (!canceled) { + controller.error(error); + } + } + }, + cancel() { + canceled = true; + keepAlive( + (async () => { + try { + await state.backend(repository, { action: "abortPush", requestId }); + } catch { + // The sidecar may already have finished or stopped. + } + void reader.cancel().catch(() => {}); + await syncOnce(); + })() + ); + }, + }); + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function authorize( + repository: Repository, + request: Request, + scope: TokenScope +): Promise { + const header = request.headers.get("Authorization") ?? ""; + let value = ""; + if (header.startsWith("Bearer ")) { + value = header.slice(7); + } else if (header.startsWith("Basic ")) { + try { + value = atob(header.slice(6)).split(":").slice(1).join(":"); + } catch { + value = ""; + } + } + const valueHash = await hashToken(value); + const token = repository.tokens.find( + (candidate) => + !candidate.revokedAt && + constantTimeEqual(candidate.tokenHash, valueHash) && + isActive(candidate) + ); + if (!token) { + return new Response("Invalid Artifacts token", { + status: 401, + headers: { "WWW-Authenticate": 'Basic realm="Artifacts"' }, + }); + } + if (scope === "write" && token.scope !== "write") { + return new Response("Artifacts token requires write scope", { + status: 403, + }); + } + return undefined; +} + +function applyBackendState(repository: Repository, state: BackendState): void { + repository.defaultBranch = state.defaultBranch; +} + +function makeRepo( + name: string, + namespace: string, + origin: string, + options: RepositoryOptions +): Repository { + const time = new Date().toISOString(); + return { + id: randomBase36(16), + name, + namespace, + description: options.description, + readOnly: Boolean(options.readOnly), + defaultBranch: options.setDefaultBranch ?? "main", + remote: `${origin}/git/${namespace}/${name}.git`, + createdAt: time, + updatedAt: time, + lastPushAt: undefined, + tokens: [], + }; +} + +function repoMetadata(repository: Repository): RepositoryMetadata { + return { + id: repository.id, + name: repository.name, + remote: repository.remote, + defaultBranch: repository.defaultBranch, + description: repository.description ?? null, + source: + repository.source === undefined + ? null + : repository.source.type === "fork" + ? `artifacts:${repository.namespace}/${repository.source.repository}` + : `git:${repository.source.url}`, + readOnly: repository.readOnly, + createdAt: repository.createdAt, + updatedAt: repository.updatedAt, + lastPushAt: repository.lastPushAt ?? null, + }; +} + +function createRepoResult( + repository: Repository, + token: CreatedToken +): CreateRepositoryResult { + return { + id: repository.id, + name: repository.name, + description: repository.description ?? null, + defaultBranch: repository.defaultBranch, + remote: repository.remote, + token: token.plaintext, + }; +} + +function tokenResult(token: CreatedToken): CreateTokenResult { + return { + id: token.record.id, + plaintext: token.plaintext, + scope: token.record.scope, + expiresAt: token.record.expiresAt, + }; +} + +function getRepo( + repositories: RepositoryMap, + name: string, + expectedId?: string +): Repository { + const key = name.toLowerCase(); + if (!Object.hasOwn(repositories, key)) { + throw new ArtifactsError("NOT_FOUND", `Repository not found: ${name}.`); + } + const repository = repositories[key]; + if ( + repository.deleting || + (expectedId !== undefined && repository.id !== expectedId) + ) { + throw new ArtifactsError("NOT_FOUND", `Repository not found: ${name}.`); + } + return repository; +} + +function hasRepository(repositories: RepositoryMap, name: string): boolean { + return Object.hasOwn(repositories, name.toLowerCase()); +} + +function parseImportRequest(options: ImportRepositoryOptions): { + source: ImportRepositoryOptions["source"]; + target: ImportRepositoryOptions["target"]; + sourceUrl: URL; +} { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts import options are required" + ); + } + const { source, target } = options; + validateName(target?.name); + let sourceUrl: URL; + try { + sourceUrl = new URL(source?.url); + } catch { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be a valid URL." + ); + } + if (sourceUrl.protocol !== "https:") { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.url: must be an HTTPS URL." + ); + } + if ( + source.depth !== undefined && + (!Number.isSafeInteger(source.depth) || source.depth < 1) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid source.depth: must be a positive integer." + ); + } + return { source, target, sourceUrl }; +} + +function validateName(name: unknown): asserts name is string { + if (typeof name !== "string" || !repoName.test(name)) { + throw new ArtifactsError("INVALID_REPO_NAME", "Invalid repo name."); + } +} + +function validateRepositoryOptions( + options: RepositoryOptions | ForkOptions +): void { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository options must be an object" + ); + } + if ( + options.description !== undefined && + typeof options.description !== "string" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository description must be a string" + ); + } + if (options.readOnly !== undefined && typeof options.readOnly !== "boolean") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts repository readOnly must be a boolean" + ); + } + if ("setDefaultBranch" in options && options.setDefaultBranch !== undefined) { + validateBranch(options.setDefaultBranch); + } + if ( + "defaultBranchOnly" in options && + options.defaultBranchOnly !== undefined && + typeof options.defaultBranchOnly !== "boolean" + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts defaultBranchOnly must be a boolean" + ); + } +} + +function validateBranch(branch: unknown): asserts branch is string { + if ( + typeof branch !== "string" || + branch.length < 1 || + branch.length > 256 || + branch.startsWith("-") || + branch.startsWith(".") || + branch.endsWith("/") || + branch.endsWith(".") || + branch.endsWith(".lock") || + branch === "@" || + branch.includes("..") || + branch.includes("//") || + branch.includes("@{") || + branch.split("/").some((segment) => segment.startsWith(".")) || + [...branch].some((character) => character.charCodeAt(0) <= 32) || + /[~^:?*[\\]/.test(branch) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts default branch is invalid" + ); + } +} + +function validateLogOptions( + options: unknown +): asserts options is RepositoryLogOptions { + if (!options || typeof options !== "object") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts log options must be an object" + ); + } + const { ref, limit, offset } = options as RepositoryLogOptions; + if (ref !== undefined && typeof ref !== "string") { + throw new ArtifactsError("INVALID_INPUT", "Invalid ref: must be a string."); + } + if ( + limit !== undefined && + (!Number.isInteger(limit) || limit < 1 || limit > 1000) + ) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid limit: must be between 1 and 1000." + ); + } + if (offset !== undefined && (!Number.isInteger(offset) || offset < 0)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid offset: must be a non-negative integer." + ); + } +} + +async function createToken( + repository: Repository, + scope: TokenScope = "write", + ttl = 86_400 +): Promise { + if (scope !== "read" && scope !== "write") { + throw new ArtifactsError( + "INVALID_INPUT", + "Artifacts token scope must be read or write" + ); + } + if (!Number.isInteger(ttl) || ttl < 60 || ttl > 31_536_000) { + throw new ArtifactsError( + "INVALID_TTL", + `Invalid TTL ${ttl}: must be between 60 and 31536000 seconds.` + ); + } + const createdAt = new Date(); + const expiresAt = new Date(createdAt.getTime() + ttl * 1000); + const plaintext = `art_v2_x_${randomHex(20)}?expires=${Math.floor(expiresAt.getTime() / 1000)}`; + const record = { + id: randomBase36(16), + tokenHash: await hashToken(plaintext), + scope, + createdAt: createdAt.toISOString(), + expiresAt: expiresAt.toISOString(), + }; + repository.tokens.push(record); + return { record, plaintext }; +} + +function isActive(token: Pick): boolean { + return Date.parse(token.expiresAt) > Date.now(); +} + +function randomHex(bytes: number): string { + return [...crypto.getRandomValues(new Uint8Array(bytes))] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +async function hashToken(value: string): Promise { + const secret = value.split("?", 1)[0] ?? ""; + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(secret) + ); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function constantTimeEqual(left: string, right: string): boolean { + if (left.length !== right.length) { + return false; + } + let difference = 0; + for (let index = 0; index < left.length; index++) { + difference |= left.charCodeAt(index) ^ right.charCodeAt(index); + } + return difference === 0; +} + +function randomBase36(length: number): string { + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"; + return [...crypto.getRandomValues(new Uint8Array(length))] + .map((byte) => alphabet[byte % alphabet.length]) + .join(""); +} + +function isTokenId(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-z]{16}$/.test(value); +} + +function isArtifactToken(value: unknown): value is string { + return ( + typeof value === "string" && + /^art_v(?:1_[0-9a-f]{40}|2_[xeuf]_[0-9a-f]{40})(?:\?expires=\d+)?$/.test( + value + ) + ); +} + +function validateHash(hash: unknown): asserts hash is string { + if (!hash) { + throw new ArtifactsError("INVALID_INPUT", "Invalid hash: hash required."); + } + if (typeof hash !== "string" || !/^[0-9a-f]{40}$/.test(hash)) { + throw new ArtifactsError( + "INVALID_INPUT", + "Invalid hash: expected a 40-character SHA-1 hex hash." + ); + } +} + +function normalizePath(path: string): string | null { + const segments = path + .split("/") + .filter((segment) => segment !== "" && segment !== "."); + if (segments.includes("..") || segments.length === 0) { + return null; + } + return segments.join("/"); +} + +function contentType(path: string, bytes: Uint8Array): string { + const extension = path.toLowerCase().match(/\.[^./]+$/)?.[0]; + const known: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".ico": "image/x-icon", + ".pdf": "application/pdf", + }; + if (extension && known[extension]) { + return known[extension]; + } + const sample = bytes.subarray(0, 8192); + if (sample.includes(0)) { + return "application/octet-stream"; + } + try { + // A sample may stop midway through a valid multibyte character. + new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(sample, { + stream: sample.length < bytes.length, + }); + return "text/plain;charset=utf-8"; + } catch { + return "application/octet-stream"; + } +} + +function decodeBase64(value: string): Uint8Array { + return Uint8Array.from(atob(value), (character) => character.charCodeAt(0)); +} + +function encodeCursor(offset: number): string { + return btoa(JSON.stringify({ offset })); +} + +function decodeCursor(cursor: string | undefined): number { + if (cursor === undefined) { + return 0; + } + try { + const value = JSON.parse(atob(cursor)) as { offset?: unknown }; + return typeof value.offset === "number" && + Number.isInteger(value.offset) && + value.offset >= 0 + ? value.offset + : 0; + } catch { + return 0; + } +} + +async function publicCall(action: () => Promise): Promise { + try { + return await action(); + } catch (error) { + if (isArtifactsError(error)) { + throw error; + } + throw new ArtifactsError("INTERNAL_ERROR", "An internal error occurred."); + } +} + +function isArtifactsError(error: unknown): error is ArtifactsError { + return ( + error instanceof Error && + error.name === "ArtifactsError" && + "code" in error && + typeof error.code === "string" && + "numericCode" in error && + typeof error.numericCode === "number" + ); +} + +function classifyImportError(message: string): ArtifactsError { + const normalized = message.toLowerCase(); + if ( + normalized.includes("authentication failed") || + normalized.includes("could not read username") || + normalized.includes("http 401") + ) { + return new ArtifactsError( + "REMOTE_AUTH_REQUIRED", + "Authentication is required to access the remote repository." + ); + } + if ( + normalized.includes("remote branch") && + normalized.includes("not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote branch not found."); + } + if ( + normalized.includes("not found") || + normalized.includes("repository not found") + ) { + return new ArtifactsError("NOT_FOUND", "Remote repository not found."); + } + if (normalized.includes("does not appear to be a git repository")) { + return new ArtifactsError( + "INVALID_URL", + "The URL does not point to a git repository." + ); + } + return new ArtifactsError( + "UPSTREAM_UNAVAILABLE", + "The upstream service is unavailable. Please retry." + ); +} diff --git a/packages/miniflare/src/workers/artifacts/events.ts b/packages/miniflare/src/workers/artifacts/events.ts new file mode 100644 index 00000000000..df95c18ec3a --- /dev/null +++ b/packages/miniflare/src/workers/artifacts/events.ts @@ -0,0 +1,182 @@ +/** Local Artifacts repository snapshot used in canonical lifecycle events. */ +export interface RepoSnapshot { + id: string; + namespace: string; + name: string; + jurisdiction: "unrestricted"; + defaultBranch: string; + description: string | null; + readOnly: boolean; + createdAt: string; + updatedAt: string; + lastPushAt: string | null; +} + +type NamespaceSnapshot = { + name: string; + jurisdiction: "unrestricted"; + createdAt: string; + updatedAt: string; +}; +type RepoLocality = { namespace: string; name: string }; +type GitIdentity = { name: string; email: string }; +type PushCommit = { + id: string; + message: string; + messageTruncated: boolean; + timestamp: string; + author: GitIdentity; + committer: GitIdentity; + parents: string[]; +}; +type Push = { + ref: string; + before: string; + after: string; + commits: PushCommit[]; + totalCommitsCount: number; + commitsTruncated: boolean; +}; + +type TransportEventType = + | "repo.pushed.v1" + | "repo.cloned.v1" + | "repo.fetched.v1" + | "repo.size_observed.v1"; + +type Event = { + envelopeVersion: 1; + eventId: string; + type: Type; + timestamp: number; + source: "artifacts"; + actor: { + account: { tag: string; internalId: string }; + } & (Type extends TransportEventType + ? { user?: never } + : { user?: { id: string; email: string } }); + payload: Payload; +}; + +/** Namespace creation event. */ +export type NamespaceCreatedV1 = Event< + "namespace.created.v1", + { namespace: NamespaceSnapshot } +>; +/** Namespace deletion event. */ +export type NamespaceDeletedV1 = Event< + "namespace.deleted.v1", + { namespace: NamespaceSnapshot } +>; +/** Repository creation event. */ +export type RepoCreatedV1 = Event<"repo.created.v1", { repo: RepoSnapshot }>; +/** Repository deletion event. */ +export type RepoDeletedV1 = Event<"repo.deleted.v1", { repo: RepoSnapshot }>; +/** Repository fork event. */ +export type RepoForkedV1 = Event< + "repo.forked.v1", + { repo: RepoSnapshot; forkedFrom: RepoLocality } +>; +/** Repository import event. */ +export type RepoImportedV1 = Event< + "repo.imported.v1", + { repo: RepoSnapshot; importedFrom: { url: string; branch: string | null } } +>; +/** One successful Git ref update. */ +export type RepoPushedV1 = Event< + "repo.pushed.v1", + { repo: RepoLocality; push: Push } +>; +/** Git storage-size observation. */ +export type RepoSizeObservedV1 = Event< + "repo.size_observed.v1", + { repo: RepoLocality; sequence: number; gitServerSizeBytes: number } +>; +/** Repository clone event. */ +export type RepoClonedV1 = Event<"repo.cloned.v1", { repo: RepoLocality }>; +/** Repository incremental fetch event. */ +export type RepoFetchedV1 = Event<"repo.fetched.v1", { repo: RepoLocality }>; +/** Repository access-token creation event. */ +export type TokenCreatedV1 = Event< + "token.created.v1", + { + repo: RepoLocality; + token: { id: string; scope: string; expiresAt: string }; + } +>; +/** Repository access-token revocation event. */ +export type TokenRevokedV1 = Event< + "token.revoked.v1", + { repo: RepoLocality; token: { id: string } } +>; + +/** Canonical local events; consumer-facing projections are separate. */ +export type ArtifactsEvent = + | NamespaceCreatedV1 + | NamespaceDeletedV1 + | RepoCreatedV1 + | RepoDeletedV1 + | RepoForkedV1 + | RepoImportedV1 + | RepoPushedV1 + | RepoSizeObservedV1 + | RepoClonedV1 + | RepoFetchedV1 + | TokenCreatedV1 + | TokenRevokedV1; + +function make( + type: Type, + payload: Payload +): Event { + return { + envelopeVersion: 1, + eventId: crypto.randomUUID(), + type, + timestamp: Date.now(), + source: "artifacts", + // Local development has no authenticated account or user. + actor: { account: { tag: "local", internalId: "local" } }, + payload, + }; +} + +/** Constructors for version-one canonical Artifacts events. */ +export const V1 = { + namespaceCreated(payload: NamespaceCreatedV1["payload"]): NamespaceCreatedV1 { + return make("namespace.created.v1", payload); + }, + namespaceDeleted(payload: NamespaceDeletedV1["payload"]): NamespaceDeletedV1 { + return make("namespace.deleted.v1", payload); + }, + repoCreated(payload: RepoCreatedV1["payload"]): RepoCreatedV1 { + return make("repo.created.v1", payload); + }, + repoDeleted(payload: RepoDeletedV1["payload"]): RepoDeletedV1 { + return make("repo.deleted.v1", payload); + }, + repoForked(payload: RepoForkedV1["payload"]): RepoForkedV1 { + return make("repo.forked.v1", payload); + }, + repoImported(payload: RepoImportedV1["payload"]): RepoImportedV1 { + return make("repo.imported.v1", payload); + }, + repoPushed(payload: RepoPushedV1["payload"]): RepoPushedV1 { + return make("repo.pushed.v1", payload); + }, + repoSizeObserved(payload: RepoSizeObservedV1["payload"]): RepoSizeObservedV1 { + return make("repo.size_observed.v1", payload); + }, + repoCloned(payload: RepoClonedV1["payload"]): RepoClonedV1 { + return make("repo.cloned.v1", payload); + }, + repoFetched(payload: RepoFetchedV1["payload"]): RepoFetchedV1 { + return make("repo.fetched.v1", payload); + }, + tokenCreated(payload: TokenCreatedV1["payload"]): TokenCreatedV1 { + return make("token.created.v1", payload); + }, + tokenRevoked(payload: TokenRevokedV1["payload"]): TokenRevokedV1 { + return make("token.revoked.v1", payload); + }, +}; diff --git a/packages/miniflare/src/workers/queues/constants.ts b/packages/miniflare/src/workers/queues/constants.ts index e5e25940723..60ff99da079 100644 --- a/packages/miniflare/src/workers/queues/constants.ts +++ b/packages/miniflare/src/workers/queues/constants.ts @@ -1,3 +1,5 @@ +export const ARTIFACTS_EVENT_SERVICE_NAME = "queues:artifacts-event-producer"; + export const QueueBindings = { SERVICE_WORKER_PREFIX: "MINIFLARE_WORKER_", MAYBE_JSON_QUEUE_PRODUCERS: "MINIFLARE_QUEUE_PRODUCERS", diff --git a/packages/miniflare/src/workers/queues/event-producer.worker.ts b/packages/miniflare/src/workers/queues/event-producer.worker.ts new file mode 100644 index 00000000000..3d690ccb7b6 --- /dev/null +++ b/packages/miniflare/src/workers/queues/event-producer.worker.ts @@ -0,0 +1,10 @@ +import { WorkerEntrypoint } from "cloudflare:workers"; +import type { ArtifactsEvent } from "../artifacts/events"; + +/** Private RPC entrypoint for locally produced Artifacts events. */ +export class ArtifactsEventProducer extends WorkerEntrypoint { + /** Accept an event; delivery will be implemented with subscriptions. */ + async sendEvent(_event: ArtifactsEvent): Promise { + // No consumer exists yet. + } +} diff --git a/packages/miniflare/test/config/v4-convert.spec.ts b/packages/miniflare/test/config/v4-convert.spec.ts index d121d3da45d..4ded53d0d7c 100644 --- a/packages/miniflare/test/config/v4-convert.spec.ts +++ b/packages/miniflare/test/config/v4-convert.spec.ts @@ -4,6 +4,28 @@ import { convertV4MiniflareOptions } from "../../src/config/v4-convert"; import type { RemoteProxyConnectionString } from "../../src/plugins/shared"; describe("convertV4MiniflareOptions", () => { + test("Artifacts preserves omitted remote settings for local-default routing", ({ + expect, + }) => { + const remoteProxyConnectionString = new URL( + "http://127.0.0.1:8787" + ) as RemoteProxyConnectionString; + const converted = convertV4MiniflareOptions({ + name: "worker", + script: "export default {};", + artifacts: { + LOCAL: { namespace: "local" }, + REMOTE: { namespace: "remote", remoteProxyConnectionString }, + }, + }); + expect(converted.workers[0].config.env?.LOCAL).toEqual({ + type: "artifacts", + namespace: "local", + }); + expect(converted.workers[0].config.env?.REMOTE).toMatchObject({ + dev: { remote: true }, + }); + }); test("preserves opaque K2 stream IDs through v4 parsing and conversion", ({ expect, }) => { diff --git a/packages/miniflare/test/plugins/artifacts/contract.spec.ts b/packages/miniflare/test/plugins/artifacts/contract.spec.ts new file mode 100644 index 00000000000..4f692de7752 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract.spec.ts @@ -0,0 +1,163 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { singleModuleManifest, useDispose } from "../../test-shared"; +import { runScenarios } from "./contract/scenarios"; +import type { Created, ArtifactsBinding } from "./contract/scenarios"; + +const exec = promisify(execFile); +const SCRIPT = `export default { fetch() { return new Response("contract"); } };`; + +async function prepareLocalFixture(created: Created): Promise { + const source = await mkdtemp(path.join(tmpdir(), "artifacts-contract-")); + const environment = { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + GIT_AUTHOR_DATE: "2020-01-02T03:04:05Z", + GIT_COMMITTER_DATE: "2020-01-02T03:04:05Z", + }; + const git = async (...args: string[]) => + exec("git", args, { cwd: source, env: environment }); + try { + await git("init", "--initial-branch=main"); + await mkdir(path.join(source, "folder")); + await writeFile(path.join(source, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(source, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await writeFile(path.join(source, "folder", "nested.txt"), "nested\n"); + await git("add", "."); + await git("commit", "-m", "first"); + await writeFile(path.join(source, "second.txt"), "second\n"); + await git("add", "."); + await git("commit", "-m", "second"); + // Keep the token out of argv, stderr, and test output. The child alone sees the header. + const pushEnv = { + ...environment, + GIT_CONFIG_COUNT: "2", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `Authorization: Bearer ${created.token}`, + }; + await exec("git", ["push", created.remote, "main"], { + cwd: source, + env: pushEnv, + }); + await git("checkout", "-b", "feature"); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git("add", "."); + await git("commit", "-m", "feature"); + await exec("git", ["push", created.remote, "feature"], { + cwd: source, + env: pushEnv, + }); + } finally { + await removeDir(source); + } +} + +function local(): Miniflare { + return new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { + type: "artifacts", + namespace: "contract-local", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); +} + +test("shared Artifacts contract runs against the local RPC binding and cleans up", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + const observations = await runScenarios( + REPOS, + "contract-test-", + prepareLocalFixture + ); + const find = (step: string) => + observations.find((item) => item.step === step); + expect(observations.length).toBeGreaterThan(50); + expect(find("file binary")).toMatchObject({ + outcome: "value", + value: { + blob: { type: "application/octet-stream", bytes: [0, 255, 128, 1] }, + }, + }); + expect(find("file text")).toMatchObject({ + outcome: "value", + value: { blob: { type: "text/plain;charset=utf-8" } }, + }); + expect(find("duplicate case")).toMatchObject({ outcome: "error" }); + expect(find("retained after delete")).toMatchObject({ outcome: "error" }); + for (const kind of ["blob", "tree", "commit"]) { + expect(find(`${kind} present`)).toMatchObject({ outcome: "value" }); + expect(find(`${kind} missing`)).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find(`${kind} wrong type`)).toMatchObject({ + outcome: "value", + value: null, + }); + } + expect(find("file missing ref")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("file missing path")).toMatchObject({ + outcome: "value", + value: null, + }); + expect(find("list next page")).toMatchObject({ outcome: "value" }); + expect(find("token revoke wrong repo")).toMatchObject({ + outcome: "value", + value: false, + }); + expect(find("fork all feature")).toMatchObject({ + outcome: "value", + value: { blob: { bytes: [...new TextEncoder().encode("feature\n")] } }, + }); + expect(find("fork main feature")).toMatchObject({ + outcome: "value", + value: null, + }); + expect((await REPOS.list()).total).toBe(0); +}); + +test("local contract scenarios refuse a nonempty namespace", async ({ + expect, +}) => { + const mf = local(); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ REPOS: ArtifactsBinding }>(); + await REPOS.create("contract-test-main"); + await expect( + runScenarios(REPOS, "contract-test-", prepareLocalFixture) + ).rejects.toThrow(/not empty/); + expect((await REPOS.list()).total).toBe(1); + await REPOS.delete("contract-test-main"); +}); diff --git a/packages/miniflare/test/plugins/artifacts/contract/README.md b/packages/miniflare/test/plugins/artifacts/contract/README.md new file mode 100644 index 00000000000..d8a184747f7 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/README.md @@ -0,0 +1,5 @@ +# Artifacts local contract tests + +`../contract.spec.ts` runs shared scenarios against Miniflare's real RPC binding with a deterministic, locally pushed Git fixture. The scenarios assert returned objects, error fields, Blob MIME types and bytes, missing/null outcomes, repository lifecycle, tokens, Git objects and files, and forks. Git HTTP authorization is tested separately in `../index.spec.ts`. The scenarios refuse a nonempty namespace and clean up repositories they create. + +**These are local-only observations, not a specification or evidence of deployed-service parity.** No production binding is contacted by CI, and this PR does not include an unexercised live-comparison harness. A future live comparison requires the namespace owner's explicit approval of an exclusively reserved disposable namespace and a trusted private test context; do not point these tests at a production or shared namespace. Keep credentials, raw observations, and workspace-only comparison tools out of public PRs. diff --git a/packages/miniflare/test/plugins/artifacts/contract/scenarios.ts b/packages/miniflare/test/plugins/artifacts/contract/scenarios.ts new file mode 100644 index 00000000000..d9a3ea549ff --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/contract/scenarios.ts @@ -0,0 +1,331 @@ +/* Shared observable scenarios. No HTTP RPC shim: calls use the real Worker binding. */ +export interface Repository { + info(): Promise; + createToken( + scope?: "read" | "write", + ttl?: number + ): Promise<{ id: string; plaintext: string }>; + listTokens(): Promise; + revokeToken(tokenOrId: string): Promise; + readBlob(hash: string): Promise; + readTree(hash: string): Promise | null>; + readCommit(hash: string): Promise<{ treeHash: string } | null>; + readFile(args: { ref: string; path: string }): Promise; + log(options?: { + ref?: string; + limit?: number; + offset?: number; + }): Promise>; + fork( + name: string, + options?: { + readOnly?: boolean; + defaultBranchOnly?: boolean; + description?: string; + } + ): Promise; +} + +export interface Created { + id: string; + name: string; + remote: string; + token: string; +} + +export interface ArtifactsBinding { + create( + name: string, + options?: { readOnly?: boolean; description?: string } + ): Promise; + get(name: string): Promise; + list(options?: { limit?: number; cursor?: string }): Promise<{ + repos: Array<{ name: string }>; + total: number; + cursor?: string; + }>; + delete(name: string): Promise; +} + +export type Observation = { + step: string; + outcome: "value" | "error"; + value: unknown; +}; +export type PrepareFixture = (created: Created) => Promise; + +/** Execute a bounded, namespace-scoped set of RPC calls and capture their exact outcomes. */ +export async function runScenarios( + binding: ArtifactsBinding, + prefix: string, + prepareFixture: PrepareFixture +): Promise { + if (!/^[a-z][a-z0-9-]{7,40}-$/.test(prefix)) { + throw new Error( + "Fixture prefix must be lowercase, 8–41 characters, and end with '-'" + ); + } + const names = [ + "main", + "second", + "third", + "fork", + "all", + "readonly", + "reused", + ].map((suffix) => prefix + suffix); + // The namespace must be empty: global list totals/cursors are otherwise affected + // by repositories outside this fixture, and cleanup must never touch them. + const seenCursors = new Set(); + let cursor: string | undefined; + do { + const page = await binding.list({ + limit: 200, + ...(cursor ? { cursor } : {}), + }); + if (page.repos.length || page.total !== 0) { + throw new Error("Disposable fixture namespace is not empty"); + } + cursor = page.cursor; + if (cursor && seenCursors.has(cursor)) { + throw new Error("Fixture preflight pagination repeated a cursor"); + } + if (cursor) { + seenCursors.add(cursor); + } + } while (cursor); + const [main, second, third, fork, all, readonly, reused] = names; + const observations: Observation[] = []; + const owned = new Set(); + const recordCreation = async ( + step: string, + name: string, + call: () => Promise + ): Promise => { + await record(step, async () => { + const created = await call(); + owned.add(name); + return created; + }); + }; + const record = async ( + step: string, + call: () => Promise + ): Promise => { + try { + observations.push({ + step, + outcome: "value", + value: await snapshot(await call()), + }); + } catch (error) { + observations.push({ + step, + outcome: "error", + value: errorSnapshot(error), + }); + } + }; + // Only successfully created fixtures belong to this run. Failed RPC calls + // may have unknown effects; never assume a failed create grants ownership. + let scenarioFailure: { error: unknown } | undefined; + try { + await record("empty list", () => binding.list({ limit: 1 })); + await record("invalid name", async () => + (await binding.get(prefix + "invalid/name")).info() + ); + await record("missing get info", async () => + (await binding.get(prefix + "absent")).info() + ); + await record("missing delete", () => binding.delete(prefix + "absent")); + await recordCreation("create", main, () => + binding.create(main, { description: "contract fixture" }) + ); + await recordCreation("duplicate case", main.toUpperCase(), () => + binding.create(main.toUpperCase()) + ); + await record("case get", async () => + (await binding.get(main.toUpperCase())).info() + ); + await recordCreation("create second", second, () => binding.create(second)); + await recordCreation("create third", third, () => binding.create(third)); + await record("list first page", () => binding.list({ limit: 2 })); + const page = await binding.list({ limit: 2 }); + await record("list next page", () => + binding.list({ limit: 2, cursor: page.cursor }) + ); + await record("list limit zero", () => binding.list({ limit: 0 })); + await record("list limit high", () => binding.list({ limit: 201 })); + await record("list malformed cursor", () => + binding.list({ limit: 1, cursor: "not-base64!" }) + ); + const retained = await binding.get(second); + await record("delete retained", () => binding.delete(second)); + await record("retained after delete", () => retained.info()); + await recordCreation("recreate retained", second, () => + binding.create(second) + ); + await record("retained after recreate", () => retained.info()); + await record("delete case", () => binding.delete(third.toUpperCase())); + await record("delete twice", () => binding.delete(third)); + const repo = await binding.get(main); + await record("token list initial", () => repo.listTokens()); + let readToken: { id: string; plaintext: string } | undefined; + await record( + "token create read", + async () => (readToken = await repo.createToken("read", 60)) + ); + await record("token list read", () => repo.listTokens()); + await record("token ttl low", () => repo.createToken("write", 59)); + await record("token ttl high", () => repo.createToken("write", 31_536_001)); + if (!readToken) { + throw new Error("Could not create fixture read token"); + } + const tokenId = readToken.id; + await record("token revoke wrong repo", async () => + (await binding.get(second)).revokeToken(tokenId) + ); + await record("token revoke", () => repo.revokeToken(tokenId)); + await record("token revoke twice", () => repo.revokeToken(tokenId)); + await record("token list revoked", () => repo.listTokens()); + await record("token invalid id", () => repo.revokeToken("bad")); + await recordCreation("fork empty", fork, async () => + (await binding.get(second)).fork(fork) + ); + await recordCreation("fork duplicate case", fork.toUpperCase(), async () => + (await binding.get(second)).fork(fork.toUpperCase()) + ); + await recordCreation("fork read only", readonly, async () => + (await binding.get(second)).fork(readonly, { readOnly: true }) + ); + await recordCreation("fork missing source", reused, async () => + (await binding.get(prefix + "absent")).fork(reused) + ); + const created = (await repo.info()) as Created; + // The initial create token is only returned by create, not info; prepare via a fresh token. + const write = await repo.createToken("write"); + await prepareFixture({ ...created, token: write.plaintext }); + const log = await repo.log({ ref: "main" }); + await record("log main", () => repo.log({ ref: "main" })); + await record("log page", () => + repo.log({ ref: "main", limit: 1, offset: 1 }) + ); + await record("log invalid limit", () => repo.log({ limit: 0 })); + await record("log missing ref", () => repo.log({ ref: "missing" })); + const hash = log[0]?.hash; + if (!hash) { + throw new Error("Fixture push did not produce a main commit"); + } + const commit = await repo.readCommit(hash); + if (!commit) { + throw new Error("Fixture main commit is missing"); + } + const tree = await repo.readTree(commit.treeHash); + if (!tree) { + throw new Error("Fixture main tree is missing"); + } + const blob = tree.find((entry) => entry.name === "hello.txt")?.hash; + if (!blob) { + throw new Error("Fixture hello.txt blob is missing"); + } + const missingHash = "0".repeat(40); + for (const [kind, read, wrong] of [ + ["blob", (h: string) => repo.readBlob(h), commit.treeHash], + ["tree", (h: string) => repo.readTree(h), blob], + ["commit", (h: string) => repo.readCommit(h), blob], + ] as const) { + await record(`${kind} present`, () => + read(kind === "blob" ? blob : kind === "tree" ? commit.treeHash : hash) + ); + await record(`${kind} missing`, () => read(missingHash)); + await record(`${kind} invalid hash`, () => read("bad")); + await record(`${kind} wrong type`, () => read(wrong)); + } + for (const [label, ref, path] of [ + ["text", "main", "hello.txt"], + ["binary", "main", "bytes.bin"], + ["nested", "main", "folder/nested.txt"], + ["directory", "main", "folder"], + ["missing path", "main", "absent"], + ["traversal", "main", "../hello.txt"], + ["missing ref", "missing", "hello.txt"], + ["commit ref", hash, "hello.txt"], + ] as const) { + await record(`file ${label}`, () => repo.readFile({ ref, path })); + } + await record("file invalid path", () => + repo.readFile({ ref: "main", path: "" }) + ); + await recordCreation("fork main only", reused, () => + repo.fork(reused, { defaultBranchOnly: true }) + ); + await recordCreation("fork all refs", all, () => + repo.fork(all, { defaultBranchOnly: false, description: "all refs" }) + ); + await record("fork main feature", async () => + (await binding.get(reused)).readFile({ + ref: "feature", + path: "feature.txt", + }) + ); + await record("fork all feature", async () => + (await binding.get(all)).readFile({ ref: "feature", path: "feature.txt" }) + ); + } catch (error) { + scenarioFailure = { error }; + } + const cleanupFailures: string[] = []; + for (const name of owned) { + try { + await binding.delete(name); + } catch { + cleanupFailures.push(name); + } + } + if (cleanupFailures.length) { + throw new Error( + `Fixture cleanup failed for: ${cleanupFailures.join(", ")}`, + { cause: scenarioFailure?.error } + ); + } + if (scenarioFailure) { + throw scenarioFailure.error; + } + return observations; +} + +function errorSnapshot(error: unknown): unknown { + if (error === null || typeof error !== "object") { + return { thrown: String(error) }; + } + return Object.fromEntries( + ["name", "message", "code", "numericCode"] + .filter((key) => key in error) + .map((key) => [key, (error as Record)[key]]) + ); +} + +async function snapshot(value: unknown): Promise { + if (value instanceof Blob) { + return { + blob: { + type: value.type, + bytes: [...new Uint8Array(await value.arrayBuffer())], + }, + }; + } + if (Array.isArray(value)) { + return Promise.all(value.map(snapshot)); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + await Promise.all( + Object.entries(value).map(async ([key, item]) => [ + key, + await snapshot(item), + ]) + ) + ); + } + return value === undefined ? { undefined: true } : value; +} diff --git a/packages/miniflare/test/plugins/artifacts/events.spec.ts b/packages/miniflare/test/plugins/artifacts/events.spec.ts new file mode 100644 index 00000000000..f564ba73c70 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/events.spec.ts @@ -0,0 +1,27 @@ +import { expectTypeOf, test } from "vitest"; +import { V1 } from "../../../src/workers/artifacts/events"; + +test("V1 constructors return event-specific types", ({ expect }) => { + expect(Object.keys(V1)).toHaveLength(12); + expectTypeOf< + ReturnType["type"] + >().toEqualTypeOf<"repo.created.v1">(); + expectTypeOf< + ReturnType["payload"]["forkedFrom"] + >().toEqualTypeOf<{ + namespace: string; + name: string; + }>(); + expectTypeOf< + ReturnType< + typeof V1.repoPushed + >["payload"]["push"]["commits"][number]["messageTruncated"] + >().toEqualTypeOf(); + expectTypeOf< + ReturnType["payload"]["token"] + >().toEqualTypeOf<{ + id: string; + scope: string; + expiresAt: string; + }>(); +}); diff --git a/packages/miniflare/test/plugins/artifacts/https-import.spec.ts b/packages/miniflare/test/plugins/artifacts/https-import.spec.ts new file mode 100644 index 00000000000..9f6357dcdfe --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/https-import.spec.ts @@ -0,0 +1,569 @@ +import { execFile, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:https"; +import path from "node:path"; +import { promisify } from "node:util"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { gitEnvironment } from "../../../src/plugins/artifacts/git-client"; +import { repositoryPath } from "../../../src/plugins/artifacts/storage"; +import { singleModuleManifest, useTmp } from "../../test-shared"; + +const exec = promisify(execFile); +const WORKER = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name ? await env.REPOS.get(name) : env.REPOS; + const result = await target[method](...args); + if (result instanceof Blob) { + return Response.json({ bytes: [...new Uint8Array(await result.arrayBuffer())] }); + } + return Response.json(result); + } catch (error) { + return Response.json({ error: error.message, code: error.code }, { status: 400 }); + } + } +}; +`; + +type ImportOptions = { + source: { url: string; branch?: string; depth?: number }; + target: { name: string }; +}; + +async function git(args: string[], cwd?: string): Promise { + const { stdout } = await exec("git", args, { + cwd, + env: { + ...gitEnvironment(), + GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }); + return stdout.trim(); +} + +async function makeSource(directory: string): Promise<{ + first: string; + second: string; + feature: string; +}> { + const source = path.join(directory, "source"); + await git(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "first\n"); + await git(["add", "README"], source); + await git(["commit", "-m", "first"], source); + const first = await git(["rev-parse", "HEAD"], source); + await writeFile(path.join(source, "README"), "second\n"); + await git(["commit", "-am", "second"], source); + const second = await git(["rev-parse", "HEAD"], source); + await git(["checkout", "-b", "feature"], source); + await writeFile(path.join(source, "feature.txt"), "feature\n"); + await git(["add", "feature.txt"], source); + await git(["commit", "-m", "feature"], source); + const feature = await git(["rev-parse", "HEAD"], source); + const bare = path.join(directory, "source.git"); + await git(["clone", "--bare", source, bare]); + await git(["symbolic-ref", "HEAD", "refs/heads/main"], bare); + return { first, second, feature }; +} + +/** Serve native Git's smart HTTP CGI over a local, self-signed HTTPS origin. */ +async function startHttpsGit(directory: string) { + const keyPath = path.join(directory, "key.pem"); + const certPath = path.join(directory, "cert.pem"); + await exec("openssl", [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + "-keyout", + keyPath, + "-out", + certPath, + ]); + const server = createServer( + { key: await readFile(keyPath), cert: await readFile(certPath) }, + (request, response) => { + const auth = `Basic ${Buffer.from("reader:correct-password").toString("base64")}`; + if (request.headers.authorization !== auth) { + response + .writeHead(401, { "WWW-Authenticate": 'Basic realm="fixture"' }) + .end(); + return; + } + if (request.url?.startsWith("/redirected.git")) { + redirectedRequests++; + response.writeHead(500).end("Unexpected redirected request"); + return; + } + if (redirectTarget && request.url?.startsWith("/source.git/info/refs")) { + response.writeHead(302, { Location: redirectTarget }).end(); + return; + } + if (stallInfoRefs && request.url?.startsWith("/source.git/info/refs")) { + stalledRequests++; + return; + } + if ( + interruptUpload && + request.url?.startsWith("/source.git/git-upload-pack") + ) { + interrupted++; + response.writeHead(200, { + "Content-Type": "application/x-git-upload-pack-result", + }); + response.write("0008NAK\n"); + response.destroy(); + return; + } + const url = new URL(request.url ?? "/", "https://localhost"); + const child = spawn("git", ["http-backend"], { + env: { + ...gitEnvironment(), + GIT_PROJECT_ROOT: directory, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: url.pathname, + QUERY_STRING: url.search.slice(1), + REQUEST_METHOD: request.method ?? "GET", + CONTENT_TYPE: request.headers["content-type"] ?? "", + CONTENT_LENGTH: request.headers["content-length"] ?? "", + HTTP_GIT_PROTOCOL: String(request.headers["git-protocol"] ?? ""), + }, + stdio: ["pipe", "pipe", "pipe"], + }); + const output: Buffer[] = []; + const errors: Buffer[] = []; + request.pipe(child.stdin); + child.stdout.on("data", (chunk: Buffer) => output.push(chunk)); + child.stderr.on("data", (chunk: Buffer) => errors.push(chunk)); + response.once("close", () => child.kill()); + child.once("error", () => response.destroy()); + child.once("close", (status) => { + if (response.destroyed) { + return; + } + const bytes = Buffer.concat(output); + const separator = bytes.indexOf("\r\n\r\n") >= 0 ? "\r\n\r\n" : "\n\n"; + const end = bytes.indexOf(separator); + if (end < 0) { + response.writeHead(500).end(Buffer.concat(errors)); + return; + } + const headers = bytes.subarray(0, end).toString().split(/\r?\n/); + const statusLine = headers.find((line) => line.startsWith("Status:")); + const statusCode = statusLine + ? Number(statusLine.match(/^Status: (\d{3})/)?.[1]) + : 200; + for (const line of headers) { + const colon = line.indexOf(":"); + if (colon > 0 && !line.startsWith("Status:")) { + response.setHeader( + line.slice(0, colon), + line.slice(colon + 1).trim() + ); + } + } + response.writeHead(status === 0 ? statusCode : 500); + response.end(bytes.subarray(end + separator.length)); + }); + } + ); + let interruptUpload = false; + let interrupted = 0; + let stallInfoRefs = false; + let stalledRequests = 0; + let redirectTarget: string | undefined; + let redirectedRequests = 0; + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("HTTPS fixture did not bind a port"); + } + return { + certPath, + url: `https://localhost:${address.port}/source.git`, + get interrupted() { + return interrupted; + }, + get stalledRequests() { + return stalledRequests; + }, + set stallInfoRefs(value: boolean) { + stallInfoRefs = value; + }, + get redirectedRequests() { + return redirectedRequests; + }, + set redirectTarget(value: string | undefined) { + redirectTarget = value; + }, + set interruptUpload(value: boolean) { + interruptUpload = value; + }, + async close() { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +async function call( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise<{ ok: boolean; body: unknown }> { + const response = await mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); + return { ok: response.ok, body: await response.json() }; +} + +function importOptions( + url: string, + name: string, + branch?: string, + depth?: number +): ImportOptions { + return { + source: { url, ...(branch ? { branch } : {}), ...(depth ? { depth } : {}) }, + target: { name }, + }; +} + +async function withFixture( + callback: (context: { + mf: Miniflare; + fixture: Awaited>; + commits: Awaited>; + directory: string; + }) => Promise, + trust = true +) { + const directory = await useTmp(); + const commits = await makeSource(directory); + const fixture = await startHttpsGit(directory); + const previousCA = process.env.GIT_SSL_CAINFO; + // The controller's sidecar snapshots this environment at startup. Git gets + // this CA only via GIT_SSL_CAINFO; neither Node nor workerd trust is changed. + if (trust) { + process.env.GIT_SSL_CAINFO = fixture.certPath; + } else { + delete process.env.GIT_SSL_CAINFO; + } + let mf: Miniflare | undefined; + try { + mf = new Miniflare({ + cf: false, + resourcePersistencePath: path.join(directory, "persist"), + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { + type: "artifacts", + namespace: "https-import", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(WORKER), + }, + }, + ], + }); + await mf.ready; + restoreCA(previousCA); + await callback({ mf, fixture, commits, directory }); + } finally { + restoreCA(previousCA); + try { + await mf?.dispose(); + } finally { + await fixture.close(); + } + } +} + +function restoreCA(value: string | undefined): void { + if (value === undefined) { + delete process.env.GIT_SSL_CAINFO; + } else { + process.env.GIT_SSL_CAINFO = value; + } +} + +test("HTTPS import: credentials never follow a Git redirect", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + fixture.redirectTarget = fixture.url.replace( + "/source.git", + "/redirected.git" + ); + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const result = await call(mf, "import", [importOptions(url, "redirect")]); + expect(result.ok).toBe(false); + expect(fixture.redirectedRequests).toBe(0); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + }); +}); + +test("HTTPS import: full import exposes refs, file, log and Git clone through the public binding", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, commits, directory }) => { + // Repository-local settings at the HTTPS source are not transferred by + // a bare clone, even when the source itself uses native Git. + await git( + ["config", "core.hooksPath", path.join(directory, "source-hooks")], + path.join(directory, "source.git") + ); + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const imported = await call(mf, "import", [importOptions(url, "full")]); + expect(imported.ok, JSON.stringify(imported.body)).toBe(true); + expect(imported.body).toMatchObject({ + name: "full", + defaultBranch: "main", + }); + const namespaceId = createHash("sha256") + .update("https-import") + .digest("hex") + .slice(0, 32); + const config = await readFile( + path.join( + repositoryPath( + path.join(directory, "persist", "artifacts", namespaceId, "git"), + "https-import", + "full" + ), + "config" + ), + "utf8" + ); + expect(config).not.toContain("correct-password"); + expect(config).not.toContain("source-hooks"); + expect(config).not.toContain('remote "origin"'); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "full")) + .body + ).toEqual({ bytes: [...Buffer.from("second\n")] }); + expect((await call(mf, "log", [], "full")).body).toMatchObject([ + { hash: commits.second }, + { hash: commits.first }, + ]); + expect( + ( + await call( + mf, + "readFile", + [{ ref: "feature", path: "feature.txt" }], + "full" + ) + ).body + ).toEqual({ bytes: [...Buffer.from("feature\n")] }); + const info = (await call(mf, "info", [], "full")).body as { + remote: string; + source: string; + }; + expect(info.source).toBe(`git:${fixture.url}`); + expect(JSON.stringify(info)).not.toContain("correct-password"); + const token = (imported.body as { token: string }).token; + await exec( + "git", + [ + "-c", + `http.extraHeader=Authorization: Bearer ${token}`, + "clone", + info.remote, + path.join(directory, "clone"), + ], + { env: gitEnvironment() } + ); + expect( + await readFile(path.join(directory, "clone", "README"), "utf8") + ).toBe("second\n"); + }); +}); + +test("HTTPS import: depth and branch constrain the imported history and refs", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, commits }) => { + const url = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const result = await call(mf, "import", [ + importOptions(url, "shallow", "feature", 1), + ]); + expect(result.ok, JSON.stringify(result.body)).toBe(true); + expect(result.body).toMatchObject({ defaultBranch: "feature" }); + expect((await call(mf, "log", [], "shallow")).body).toMatchObject([ + { hash: commits.feature }, + ]); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "shallow")) + .body + ).toBeNull(); + }); +}); + +test("HTTPS import: disposal stops a Git clone stalled by the origin", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + fixture.stallInfoRefs = true; + const pending = call(mf, "import", [ + importOptions( + fixture.url.replace("https://", "https://reader:correct-password@"), + "stalled" + ), + ]).catch(() => undefined); + for ( + let attempt = 0; + attempt < 100 && fixture.stalledRequests === 0; + attempt++ + ) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + expect(fixture.stalledRequests).toBeGreaterThan(0); + await expect(mf.dispose()).resolves.toBeUndefined(); + await pending; + }); +}, 15_000); + +test("HTTPS import: Basic URL credentials, missing repos/branches, duplicate targets and retries", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const invalid = fixture.url.replace( + "https://", + "https://reader:wrong-password@" + ); + for (const [url, name] of [ + [fixture.url, "no-auth"], + [invalid, "wrong-auth"], + ]) { + const failure = await call(mf, "import", [importOptions(url, name)]); + expect(failure.ok).toBe(false); + expect(failure.body).toMatchObject({ code: "REMOTE_AUTH_REQUIRED" }); + expect(JSON.stringify(failure.body)).not.toContain("wrong-password"); + } + const missing = await call(mf, "import", [ + importOptions(valid.replace("source.git", "missing.git"), "absent"), + ]); + expect(missing.ok).toBe(false); + expect(missing.body).toMatchObject({ code: "NOT_FOUND" }); + const branch = await call(mf, "import", [ + importOptions(valid, "absent-branch", "nonexistent"), + ]); + expect(branch.ok).toBe(false); + expect(branch.body).toMatchObject({ code: "NOT_FOUND" }); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + expect( + (await call(mf, "import", [importOptions(valid, "absent")])).ok + ).toBe(true); + const duplicate = await call(mf, "import", [ + importOptions(valid, "ABSENT"), + ]); + expect(duplicate.ok).toBe(false); + expect(duplicate.body).toMatchObject({ code: "ALREADY_EXISTS" }); + }); +}); + +test("HTTPS import: untrusted TLS fails closed without a CA", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + const failed = await call(mf, "import", [ + importOptions(valid, "tls-failed"), + ]); + expect(failed.ok).toBe(false); + expect(failed.body).toMatchObject({ code: "UPSTREAM_UNAVAILABLE" }); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + }, false); +}); + +test("HTTPS import: interrupted upload cleans partial target and retry succeeds", async ({ + expect, +}) => { + await withFixture(async ({ mf, fixture, directory }) => { + const valid = fixture.url.replace( + "https://", + "https://reader:correct-password@" + ); + fixture.interruptUpload = true; + const failed = await call(mf, "import", [importOptions(valid, "retry")]); + expect(failed.ok).toBe(false); + expect(failed.body).toMatchObject({ code: "UPSTREAM_UNAVAILABLE" }); + expect(fixture.interrupted).toBeGreaterThan(0); + expect((await call(mf, "list")).body).toMatchObject({ total: 0 }); + const [namespaceDirectory] = await readdir( + path.join(directory, "persist", "artifacts") + ); + expect( + await readdir( + path.join(directory, "persist", "artifacts", namespaceDirectory, "git") + ) + ).toEqual([]); + fixture.interruptUpload = false; + expect((await call(mf, "import", [importOptions(valid, "retry")])).ok).toBe( + true + ); + expect( + (await call(mf, "readFile", [{ ref: "main", path: "README" }], "retry")) + .body + ).toEqual({ bytes: [...Buffer.from("second\n")] }); + }); +}); + +test("Git child only inherits explicit CA trust, never host TLS bypass or credentials", ({ + expect, +}) => { + const env = gitEnvironment({ + PATH: "/usr/bin", + GIT_SSL_CAINFO: "/tmp/explicit-ca.pem", + GIT_SSL_NO_VERIFY: "true", + GIT_ASKPASS: "/tmp/askpass", + GIT_CONFIG_PARAMETERS: "'http.extraHeader=Authorization: secret'", + }); + expect(env.GIT_SSL_CAINFO).toBe("/tmp/explicit-ca.pem"); + expect(env.GIT_SSL_NO_VERIFY).toBeUndefined(); + expect(env.GIT_ASKPASS).toBeUndefined(); + expect(env.GIT_CONFIG_PARAMETERS).toBeUndefined(); +}); diff --git a/packages/miniflare/test/plugins/artifacts/index.spec.ts b/packages/miniflare/test/plugins/artifacts/index.spec.ts new file mode 100644 index 00000000000..1ce7ec0efb8 --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/index.spec.ts @@ -0,0 +1,1402 @@ +import { execFile } from "node:child_process"; +import { + chmod, + mkdir, + readFile, + rename, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { removeDir } from "@cloudflare/workers-utils/fs-helpers"; +import { ArtifactsController, Miniflare } from "miniflare"; +import { test } from "vitest"; +import { + GitClient, + gitEnvironment, +} from "../../../src/plugins/artifacts/git-client"; +import { + startGitSidecar, + type GitSidecar, +} from "../../../src/plugins/artifacts/git-sidecar"; +import { repositoryPath } from "../../../src/plugins/artifacts/storage"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; +import type { MiniflareOptions } from "miniflare"; + +const exec = promisify(execFile); +const SCRIPT = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name === undefined ? env.REPOS : await env.REPOS.get(name); + if (method === "keys") return Response.json(Object.keys(target)); + const result = await target[method](...args); + if (result instanceof Blob) { + return Response.json({ bytes: [...new Uint8Array(await result.arrayBuffer())], type: result.type }); + } + return Response.json(result); + } catch (error) { + return Response.json({ error: error.message, name: error.name, code: error.code, numericCode: error.numericCode }, { status: 400 }); + } + } +}; +`; + +function options( + namespace = "test", + binding = "REPOS", + remote: boolean | null = false +): MiniflareOptions { + return { + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + [binding]: { + type: "artifacts", + namespace, + ...(remote === null ? {} : { dev: { remote } }), + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }; +} + +function rpcRequest( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + return mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); +} + +async function rpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +) { + const response = await rpcRequest(mf, method, args, name); + const result = (await response.json()) as any; + if (!response.ok) { + throw new Error(result.error); + } + return result; +} + +async function rpcFailure( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +) { + const response = await rpcRequest(mf, method, args, name); + return { + status: response.status, + ...((await response.json()) as { + error: string; + name: string; + code: string; + numericCode: number; + }), + }; +} + +function git(args: string[], cwd?: string, extraEnv: NodeJS.ProcessEnv = {}) { + return exec("git", args, { + cwd, + env: { + ...gitEnvironment(), + ...extraEnv, + GIT_AUTHOR_NAME: "Miniflare", + GIT_AUTHOR_EMAIL: "miniflare@example.com", + GIT_COMMITTER_NAME: "Miniflare", + GIT_COMMITTER_EMAIL: "miniflare@example.com", + }, + }); +} + +async function pushFixture(mf: Miniflare, directory: string) { + const created = await rpc(mf, "create", ["Repo"]); + await git(["init", "--initial-branch=main", directory]); + await writeFile(path.join(directory, "hello.txt"), "hello artifacts\n"); + await writeFile( + path.join(directory, "bytes.bin"), + new Uint8Array([0, 255, 128, 1]) + ); + await writeFile( + path.join(directory, "payload.dat"), + new Uint8Array([255, 254, 128]) + ); + await git(["add", "."], directory); + await git(["commit", "-m", "fixture"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "main", + ], + directory + ); + return created; +} + +async function createSidecarRepository( + sidecar: GitSidecar, + generation: string +): Promise { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { + "X-Local-Artifacts-Backend": sidecar.secret, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + action: "create", + namespace: "test", + name: "repo", + generation, + }), + } + ); + if (!response.ok) { + throw new Error(`Sidecar create failed: ${await response.text()}`); + } + await response.body?.cancel(); +} + +test("artifacts: sidecar parses admin actions before modifying repositories", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + const admin = (body: unknown) => + fetch(`http://${sidecar.address}/__local_artifacts__`, { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify(body), + }); + const common = { namespace: "test", name: "repo" }; + try { + for (const body of [ + { ...common, action: "unknown" }, + { ...common, action: "create" }, + { ...common, action: "create", generation: "gen", namespace: "../bad" }, + { ...common, action: "fork", generation: "gen", sourceName: 42 }, + { ...common, action: "import", generation: "gen", sourceUrl: 42 }, + { ...common, action: "delete", generation: null }, + { ...common, action: "abortPush", generation: "gen", requestId: "bad" }, + { ...common, action: "readBlob", hash: null }, + { ...common, action: "file", ref: "main", path: [] }, + ]) { + const response = await admin(body); + expect(response.status, JSON.stringify(body)).toBe(500); + await response.body?.cancel(); + } + const created = await admin({ + ...common, + action: "create", + generation: "gen", + }); + expect(created.status).toBe(200); + await created.body?.cancel(); + const refs = await admin({ ...common, action: "refs" }); + expect(refs.status).toBe(200); + expect(await refs.json()).toMatchObject({ refs: {} }); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: real RPC exposes current methods, not metadata or legacy methods", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["Repo"]); + const keys = await rpc(mf, "keys", [], "Repo"); + expect(keys).toContain("readFile"); + expect(keys).not.toContain("id"); + expect(keys).not.toContain("name"); + expect(keys).not.toContain("file"); + expect(keys).not.toContain("raw"); + expect( + await rpc(mf, "readFile", [{ ref: "HEAD", path: "missing" }], "Repo") + ).toBeNull(); +}); + +test("artifacts: interrupted deletion leaves a replaceable orphaned generation", async ({ + expect, +}) => { + const root = await useTmp(); + const sidecar = await startGitSidecar(root); + try { + await createSidecarRepository(sidecar, "original"); + const client = new GitClient(repositoryPath(root, "test", "repo")); + expect(await client.generation()).toBe("original"); + // The namespace has committed the deletion, but the process exited + // before the native repository could be removed. + await createSidecarRepository(sidecar, "replacement"); + expect(await client.generation()).toBe("replacement"); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: restart finishes cleanup after an interrupted Git deletion", async ({ + expect, +}) => { + const root = await useTmp(); + const active = repositoryPath(root, "test", "repo"); + const retired = `${active}.deleting-${"a".repeat(32)}`; + const first = await startGitSidecar(root); + try { + await createSidecarRepository(first, "original"); + // Simulate termination after metadata removal and the atomic rename, + // but before the retired directory's contents have been removed. + await rename(active, retired); + } finally { + await first.close(); + } + const restarted = await startGitSidecar(root); + try { + await expect(stat(retired)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(stat(active)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await restarted.close(); + } +}); + +test("artifacts: config update closes retired sidecars but abort preserves active ones", async ({ + expect, +}) => { + const directory = await useTmp(); + const controller = new ArtifactsController(); + const reachable = (address: string) => fetch(`http://${address}/`); + try { + controller.beginUpdate(); + const first = await controller.get(path.join(directory, "first")); + await controller.commitUpdate(); + controller.beginUpdate(); + const candidate = await controller.get(path.join(directory, "candidate")); + await controller.abortUpdate(); + expect((await reachable(first.sidecar.address)).status).toBe(403); + await expect(reachable(candidate.sidecar.address)).rejects.toThrow(); + controller.beginUpdate(); + const next = await controller.get(path.join(directory, "next")); + await controller.commitUpdate(); + await expect(reachable(first.sidecar.address)).rejects.toThrow(); + expect((await reachable(next.sidecar.address)).status).toBe(403); + } finally { + await controller.dispose(); + } +}); + +test.skipIf(process.platform === "win32")( + "artifacts: disposing a sidecar stops a stalled Git import", + async ({ expect }) => { + const root = await useTmp(); + const { stdout: gitPath } = await exec("which", ["git"]); + const wrapper = path.join(root, "git"); + const marker = path.join(root, "clone-started"); + await writeFile( + wrapper, + `#!/bin/sh +case " $* " in + *" clone "*) + printf started > '${marker}' + exec sleep 120 ;; +esac +exec '${gitPath.trim().replaceAll("'", "'\\''")}' "$@" +` + ); + await chmod(wrapper, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = `${root}:${previousPath}`; + const controller = new ArtifactsController(); + try { + controller.beginUpdate(); + const { sidecar } = await controller.get(path.join(root, "repos")); + await controller.commitUpdate(); + const pending = fetch(`http://${sidecar.address}/__local_artifacts__`, { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "import", + namespace: "test", + name: "stalled", + generation: "stalled-generation", + sourceUrl: "https://example.invalid/source.git", + }), + signal: AbortSignal.timeout(10_000), + }).catch(() => undefined); + let started = false; + for (let attempt = 0; attempt < 50; attempt++) { + started = await stat(marker).then( + () => true, + () => false + ); + if (started) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + expect(started).toBe(true); + await expect(controller.dispose()).resolves.toBeUndefined(); + await pending; + } finally { + process.env.PATH = previousPath; + await controller.dispose(); + } + }, + 12_000 +); + +test("artifacts: sidecar startup failures reject without crashing Miniflare", async ({ + expect, +}) => { + const root = await useTmp(); + const file = path.join(root, "not-a-directory"); + await writeFile(file, "not a directory"); + const controller = new ArtifactsController(); + controller.beginUpdate(); + try { + await expect(controller.get(file)).rejects.toThrow( + /Local Artifacts Git backend failed to start/ + ); + await controller.abortUpdate(); + } finally { + await controller.dispose(); + } +}); + +test("artifacts: a stopped backend is replaced on the next config update", async ({ + expect, +}) => { + const controller = new ArtifactsController(); + const root = path.join(await useTmp(), "repos"); + try { + controller.beginUpdate(); + const first = await controller.get(root); + await controller.commitUpdate(); + await first.sidecar.close(); + controller.beginUpdate(); + const replacement = await controller.get(root); + expect(replacement.sidecar.secret).not.toBe(first.sidecar.secret); + await controller.commitUpdate(); + } finally { + await controller.dispose(); + } +}); + +test("artifacts: omitted remote remains offline", async ({ expect }) => { + const mf = new Miniflare(options("test", "REPOS", null)); + useDispose(mf); + expect((await rpc(mf, "create", ["offline"])).name).toBe("offline"); +}); + +test("artifacts: rejects invalid local namespace before starting services", async ({ + expect, +}) => { + const mf = new Miniflare(options("../invalid")); + await expect(mf.ready).rejects.toThrow( + "Invalid local Artifacts namespace: ../invalid" + ); + // dispose() preserves a startup failure after cleaning up the instance. + await expect(mf.dispose()).rejects.toThrow( + "Invalid local Artifacts namespace: ../invalid" + ); +}); + +test("artifacts: names are case-insensitive for create, get, fork and delete", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["Repo"]); + expect((await rpc(mf, "info", [], "rEpO")).id).toBe(created.id); + await expect(rpc(mf, "create", ["repo"])).rejects.toThrow(/already exists/i); + await expect(rpc(mf, "fork", ["REPO"], "repo")).rejects.toThrow( + /already exists/i + ); + expect(await rpc(mf, "delete", ["REPO"])).toBe(true); + expect(await rpc(mf, "delete", ["repo"])).toBe(false); +}); + +test("artifacts: retained handles cannot access a replacement repository", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { + create(name: string): Promise<{ id: string }>; + get(name: string): Promise<{ + info(): Promise<{ id: string }>; + createToken(): Promise; + }>; + delete(name: string): Promise; + }; + }>(); + const original = await REPOS.create("repo"); + const retained = await REPOS.get("repo"); + await REPOS.delete("repo"); + const replacement = await REPOS.create("repo"); + expect(replacement.id).not.toBe(original.id); + await expect(async () => retained.info()).rejects.toThrow(/not found/i); + await expect(async () => retained.createToken()).rejects.toThrow( + /not found/i + ); + expect((await (await REPOS.get("repo")).info()).id).toBe(replacement.id); +}); + +test("artifacts: restored repository maps ignore inherited names and pin handles", async ({ + expect, +}) => { + const resourcePersistencePath = await useTmp(); + const first = new Miniflare({ ...options(), resourcePersistencePath }); + useDispose(first); + await rpc(first, "create", ["other"]); + await first.dispose(); + const second = new Miniflare({ ...options(), resourcePersistencePath }); + useDispose(second); + for (const name of ["constructor", "toString"]) { + await expect(rpc(second, "get", [name])).rejects.toThrow(/not found/i); + expect(await rpc(second, "delete", [name])).toBe(false); + } + const { REPOS } = await second.getBindings<{ + REPOS: { + get(name: string): Promise<{ info(): Promise<{ id: string }> }>; + }; + }>(); + const created = await rpc(second, "create", ["constructor"]); + const retained = await REPOS.get("constructor"); + expect((await retained.info()).id).toBe(created.id); + expect(await rpc(second, "delete", ["constructor"])).toBe(true); + const replacement = await rpc(second, "create", ["constructor"]); + expect(replacement.id).not.toBe(created.id); + await expect(async () => retained.info()).rejects.toThrow(/not found/i); + expect((await (await REPOS.get("constructor")).info()).id).toBe( + replacement.id + ); +}); + +test("artifacts: supports Node binding proxies", async ({ expect }) => { + const mf = new Miniflare(options()); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { + create(name: string): Promise<{ name: string }>; + list(): Promise<{ total: number }>; + }; + }>(); + expect(await REPOS.create("node-repo")).toMatchObject({ name: "node-repo" }); + expect(await REPOS.list()).toMatchObject({ total: 1 }); +}); + +test("artifacts: aliases share a namespace while distinct namespaces are isolated", async ({ + expect, +}) => { + const mf = new Miniflare({ + cf: false, + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { + type: "artifacts", + namespace: "shared", + dev: { remote: false }, + }, + ALIAS: { + type: "artifacts", + namespace: "shared", + dev: { remote: false }, + }, + OTHER: { + type: "artifacts", + namespace: "other", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }, + ], + }); + useDispose(mf); + const bindings = await mf.getBindings<{ + REPOS: { create(name: string): Promise }; + ALIAS: { list(): Promise<{ total: number }> }; + OTHER: { list(): Promise<{ total: number }> }; + }>(); + await bindings.REPOS.create("shared-repo"); + expect((await bindings.ALIAS.list()).total).toBe(1); + expect((await bindings.OTHER.list()).total).toBe(0); +}); + +test("artifacts: two Workers can share a namespace through different bindings", async ({ + expect, +}) => { + const worker = (name: string, binding: string) => ({ + config: { + name, + compatibilityDate: "2026-09-03", + env: { + [binding]: { + type: "artifacts" as const, + namespace: "shared", + dev: { remote: false }, + }, + }, + manifest: singleModuleManifest(SCRIPT), + }, + }); + const mf = new Miniflare({ + cf: false, + workers: [worker("first", "REPOS"), worker("second", "ALIAS")], + }); + useDispose(mf); + const { REPOS } = await mf.getBindings<{ + REPOS: { create(name: string): Promise<{ name: string }> }; + }>("first"); + const { ALIAS } = await mf.getBindings<{ + ALIAS: { list(): Promise<{ total: number }> }; + }>("second"); + await REPOS.create("shared-repo"); + expect((await ALIAS.list()).total).toBe(1); +}); + +test("artifacts: native Git reads commits, trees, blobs and files", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const source = path.join(await useTmp(), "source"); + await pushFixture(mf, source); + const { stdout } = await git(["rev-parse", "HEAD"], source); + const hash = stdout.trim(); + const commit = await rpc(mf, "readCommit", [hash], "Repo"); + expect(commit.hash).toBe(hash); + expect((await rpc(mf, "log", [], "Repo"))[0].hash).toBe(hash); + const tree = await rpc(mf, "readTree", [commit.treeHash], "Repo"); + const entry = tree.find(({ name }: { name: string }) => name === "hello.txt"); + expect(entry).toBeDefined(); + expect((await rpc(mf, "readBlob", [entry.hash], "Repo")).bytes).toEqual([ + ...new TextEncoder().encode("hello artifacts\n"), + ]); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "hello.txt" }], "Repo") + ).toEqual({ + bytes: [...new TextEncoder().encode("hello artifacts\n")], + type: "text/plain;charset=utf-8", + }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "bytes.bin" }], "Repo") + ).toEqual({ bytes: [0, 255, 128, 1], type: "application/octet-stream" }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "payload.dat" }], "Repo") + ).toEqual({ bytes: [255, 254, 128], type: "application/octet-stream" }); +}); + +test("artifacts: native Git push/clone and token revocation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = await useTmp(); + const created = await pushFixture(mf, path.join(directory, "source")); + expect(new URL(created.remote).hostname).toBe("127.0.0.1"); + const token = await rpc(mf, "createToken", ["read"], "Repo"); + await git([ + "-c", + `http.extraHeader=Authorization: Bearer ${token.plaintext}`, + "clone", + created.remote.replace("Repo.git", "repo.git"), + path.join(directory, "clone"), + ]); + expect( + await readFile(path.join(directory, "clone", "hello.txt"), "utf8") + ).toBe("hello artifacts\n"); + const advertise = `${created.remote}/info/refs?service=git-upload-pack`; + const discovery = await fetch(advertise, { + headers: { + Authorization: `Bearer ${token.plaintext}`, + "Git-Protocol": "version=2", + }, + }); + expect(discovery.status).toBe(200); + expect(discovery.headers.get("content-type")).toContain( + "git-upload-pack-advertisement" + ); + await discovery.body?.cancel(); + const unsupported = await fetch(`${created.remote}/HEAD`, { + headers: { Authorization: `Bearer ${created.token}` }, + }); + expect(unsupported.status).toBe(404); + await unsupported.body?.cancel(); + const unauthenticated = await fetch(advertise); + expect(unauthenticated.status).toBe(401); + await unauthenticated.body?.cancel(); + await rpc(mf, "revokeToken", [token.id], "Repo"); + const revoked = await fetch(advertise, { + headers: { Authorization: `Bearer ${token.plaintext}` }, + }); + expect(revoked.status).toBe(401); + await revoked.body?.cancel(); +}); + +test.skipIf(process.platform === "win32")( + "artifacts: large push replies drain before ref synchronization", + async ({ expect }) => { + const root = await useTmp(); + const { stdout: gitPath } = await exec("which", ["git"]); + const wrapper = path.join(root, "git"); + await writeFile( + wrapper, + `#!/bin/sh +case " $* " in + *" http-backend "*) + printf 'Content-Type: application/x-git-receive-pack-result\\r\\n\\r\\n' + dd if=/dev/zero bs=1048576 count=2 2>/dev/null + exit 0 ;; +esac +exec '${gitPath.trim().replaceAll("'", "'\\''")}' "$@" +` + ); + await chmod(wrapper, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = `${root}:${previousPath}`; + try { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["repo"]); + const response = await fetch(`${created.remote}/git-receive-pack`, { + method: "POST", + headers: { + Authorization: `Bearer ${created.token}`, + "Content-Type": "application/x-git-receive-pack-request", + }, + body: "", + signal: AbortSignal.timeout(15_000), + }); + expect(response.status).toBe(200); + expect((await response.arrayBuffer()).byteLength).toBe(2 * 1024 * 1024); + } finally { + process.env.PATH = previousPath; + } + }, + 20_000 +); + +test.skipIf(process.platform === "win32")( + "artifacts: canceled push reply synchronizes committed refs", + async ({ expect }) => { + const root = await useTmp(); + const { stdout: gitPath } = await exec("which", ["git"]); + const gitBinary = gitPath.trim().replaceAll("'", "'\\''"); + const wrapper = path.join(root, "git"); + await writeFile( + wrapper, + `#!/bin/sh +case " $* " in + *" http-backend "*) + if [ "$REQUEST_METHOD" = POST ]; then + repo="$GIT_PROJECT_ROOT\${PATH_INFO%/git-receive-pack}" + tree=$('${gitBinary}' -C "$repo" hash-object -t tree -w --stdin '${path.join(root, "ref-marker")}' + printf 'Content-Type: application/x-git-receive-pack-result\\r\\n\\r\\n' + dd if=/dev/zero bs=1048576 count=32 2>/dev/null + exit 0 + fi ;; +esac +exec '${gitBinary}' "$@" +` + ); + await chmod(wrapper, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = `${root}:${previousPath}`; + try { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["repo"]); + expect((await rpc(mf, "info", [], "repo")).lastPushAt).toBeNull(); + const response = await fetch(`${created.remote}/git-receive-pack`, { + method: "POST", + headers: { + Authorization: `Bearer ${created.token}`, + "Content-Type": "application/x-git-receive-pack-request", + }, + body: "", + signal: AbortSignal.timeout(15_000), + }); + expect(response.status).toBe(200); + if (!response.body) { + throw new Error("Git response has no body"); + } + const reader = response.body.getReader(); + expect((await reader.read()).done).toBe(false); + await reader.cancel(); + const [repo, commit] = ( + await readFile(path.join(root, "ref-marker"), "utf8") + ) + .trim() + .split(String.fromCharCode(10)); + const { stdout: actualRef } = await exec(gitPath.trim(), [ + "-C", + repo, + "rev-parse", + "refs/heads/main", + ]); + expect(actualRef.trim()).toBe(commit); + let lastPushAt: string | null = null; + for (let attempt = 0; attempt < 50; attempt++) { + lastPushAt = (await rpc(mf, "info", [], "repo")).lastPushAt; + if (lastPushAt) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + expect(lastPushAt).toEqual(expect.any(String)); + } finally { + process.env.PATH = previousPath; + } + }, + 20_000 +); + +test("artifacts: read-only and read-scoped tokens reject Git pushes", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = await useTmp(); + const writable = await pushFixture(mf, path.join(directory, "source")); + const readToken = await rpc(mf, "createToken", ["read"], "Repo"); + const withReadToken = await fetch( + `${writable.remote}/info/refs?service=git-receive-pack`, + { + headers: { Authorization: `Bearer ${readToken.plaintext}` }, + } + ); + expect(withReadToken.status).toBe(403); + await withReadToken.body?.cancel(); + const readOnly = await rpc( + mf, + "fork", + ["read-only", { readOnly: true }], + "Repo" + ); + const writeAttempt = await fetch( + `${readOnly.remote}/info/refs?service=git-receive-pack`, + { + headers: { Authorization: `Bearer ${readOnly.token}` }, + } + ); + expect(writeAttempt.status).toBe(403); + await writeAttempt.body?.cancel(); + const readAttempt = await fetch( + `${readOnly.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${readOnly.token}` } } + ); + expect(readAttempt.status).toBe(200); + await readAttempt.body?.cancel(); +}); + +test("artifacts: metadata and Git data survive restart and binding rename", async ({ + expect, +}) => { + const root = await useTmp(); + const first = new Miniflare({ ...options(), resourcePersistencePath: root }); + useDispose(first); + const created = await pushFixture(first, path.join(await useTmp(), "source")); + await first.dispose(); + const second = new Miniflare({ + ...options("test", "RENAMED"), + resourcePersistencePath: root, + }); + useDispose(second); + const { RENAMED } = await second.getBindings<{ + RENAMED: { + get(name: string): Promise<{ + info(): Promise<{ id: string; remote: string }>; + readFile(args: { ref: string; path: string }): Promise; + }>; + }; + }>(); + const repo = await RENAMED.get("repo"); + const info = await repo.info(); + expect(info.id).toBe(created.id); + expect( + await (await repo.readFile({ ref: "main", path: "hello.txt" })).text() + ).toBe("hello artifacts\n"); + const response = await fetch( + `${info.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${created.token}` } } + ); + expect(response.status).toBe(200); + await response.body?.cancel(); +}); + +test("artifacts: namespaces and instances are isolated; reload preserves state", async ({ + expect, +}) => { + const first = new Miniflare(options()); + const second = new Miniflare(options()); + useDispose(first); + useDispose(second); + await rpc(first, "create", ["repo"]); + expect((await rpc(second, "list")).total).toBe(0); + await first.setOptions(options()); + expect((await rpc(first, "list")).total).toBe(1); + await first.setOptions(options("another")); + expect((await rpc(first, "list")).total).toBe(0); +}); + +test("artifacts: lists pages and deletes a repository and its handle", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + for (const name of ["one", "two", "three"]) { + await rpc(mf, "create", [name]); + } + const first = await rpc(mf, "list", [{ limit: 2 }]); + expect(first).toMatchObject({ total: 3 }); + expect(first.repos).toHaveLength(2); + expect(first.repos[0]).toMatchObject({ + status: "ready", + jurisdiction: "unrestricted", + }); + expect(first.cursor).toEqual(expect.any(String)); + const second = await rpc(mf, "list", [{ limit: 2, cursor: first.cursor }]); + expect(second).toMatchObject({ total: 3 }); + expect(second.repos).toHaveLength(1); + expect(second.cursor).toBeUndefined(); + expect( + [...first.repos, ...second.repos].map((repo) => repo.name).sort() + ).toEqual(["one", "three", "two"]); + const { REPOS } = await mf.getBindings<{ + REPOS: { + get(name: string): Promise<{ info(): Promise<{ name: string }> }>; + }; + }>(); + const handle = await REPOS.get("ONE"); + expect((await handle.info()).name).toBe("one"); + expect(await rpc(mf, "delete", ["one"])).toBe(true); + await expect(async () => handle.info()).rejects.toThrow(/not found/i); + expect(await rpcFailure(mf, "get", ["one"])).toMatchObject({ + code: "NOT_FOUND", + }); + expect((await rpc(mf, "list")).total).toBe(2); +}); + +test("artifacts: token metadata, revocation and validation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const created = await rpc(mf, "create", ["repo"]); + const readToken = await rpc(mf, "createToken", ["read", 60], "repo"); + expect(readToken).toMatchObject({ + scope: "read", + expiresAt: expect.any(String), + }); + const listed = await rpc(mf, "listTokens", [], "repo"); + expect(listed.total).toBe(2); + expect(JSON.stringify(listed)).not.toContain(created.token); + expect(JSON.stringify(listed)).not.toContain(readToken.plaintext); + expect( + listed.tokens.every((token: { state: string }) => token.state === "active") + ).toBe(true); + expect(await rpc(mf, "revokeToken", [readToken.plaintext], "repo")).toBe( + true + ); + expect(await rpc(mf, "revokeToken", [readToken.id], "repo")).toBe(false); + expect((await rpc(mf, "listTokens", [], "repo")).total).toBe(1); + expect( + await rpcFailure(mf, "createToken", ["write", 59], "repo") + ).toMatchObject({ + status: 400, + name: "ArtifactsError", + code: "INVALID_TTL", + }); + expect( + await rpcFailure(mf, "createToken", ["write", 31_536_001], "repo") + ).toMatchObject({ + code: "INVALID_TTL", + }); + expect(await rpcFailure(mf, "revokeToken", ["bad"], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); +}); + +test("artifacts: rejects invalid inputs and returns null for missing objects", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + expect(await rpcFailure(mf, "create", ["invalid/name"])).toMatchObject({ + code: "INVALID_REPO_NAME", + numericCode: 10101, + }); + await rpc(mf, "create", ["repo"]); + expect(await rpcFailure(mf, "list", [{ limit: 0 }])).toMatchObject({ + code: "INVALID_INPUT", + }); + expect(await rpcFailure(mf, "readBlob", ["bad"], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); + const absentHash = "0".repeat(40); + expect(await rpc(mf, "readBlob", [absentHash], "repo")).toBeNull(); + expect(await rpc(mf, "readTree", [absentHash], "repo")).toBeNull(); + expect(await rpc(mf, "readCommit", [absentHash], "repo")).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "missing" }], "repo") + ).toBeNull(); + expect( + await rpcFailure(mf, "readFile", [{ ref: "main" }], "repo") + ).toMatchObject({ + code: "INVALID_INPUT", + }); + expect(await rpcFailure(mf, "log", [{ limit: 0 }], "repo")).toMatchObject({ + code: "INVALID_INPUT", + }); + expect( + await rpcFailure(mf, "import", [ + { + source: { url: "http://example.test/repo.git" }, + target: { name: "imported" }, + }, + ]) + ).toMatchObject({ + code: "INVALID_INPUT", + }); + expect( + await rpcFailure(mf, "import", [ + { source: { url: "not a url" }, target: { name: "imported" } }, + ]) + ).toMatchObject({ + code: "INVALID_INPUT", + }); + for (const depth of [0, -1, 1.5, "1", null]) { + expect( + await rpcFailure(mf, "import", [ + { + source: { url: "https://example.test/repo.git", depth }, + target: { name: "imported" }, + }, + ]) + ).toMatchObject({ code: "INVALID_INPUT" }); + } + expect((await rpc(mf, "list")).total).toBe(1); +}); + +test("artifacts: Git credentials are scoped to the repository and operation", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const first = await rpc(mf, "create", ["first"]); + const second = await rpc(mf, "create", ["second"]); + const readToken = await rpc(mf, "createToken", ["read"], "first"); + const firstRead = `${first.remote}/info/refs?service=git-upload-pack`; + const firstWrite = `${first.remote}/info/refs?service=git-receive-pack`; + const secondRead = `${second.remote}/info/refs?service=git-upload-pack`; + const basic = `Basic ${Buffer.from(`git:${readToken.plaintext}`).toString("base64")}`; + const read = await fetch(firstRead, { headers: { Authorization: basic } }); + expect(read.status).toBe(200); + await read.body?.cancel(); + const write = await fetch(firstWrite, { headers: { Authorization: basic } }); + expect(write.status).toBe(403); + await write.body?.cancel(); + const wrongRepo = await fetch(secondRead, { + headers: { Authorization: `Bearer ${first.token}` }, + }); + expect(wrongRepo.status).toBe(401); + await wrongRepo.body?.cancel(); +}); + +test("artifacts: forks an empty repo and selects which branches to copy", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["empty"]); + const emptyFork = await rpc(mf, "fork", ["empty-fork"], "empty"); + expect(emptyFork.defaultBranch).toBe("main"); + + const directory = path.join(await useTmp(), "source"); + const created = await pushFixture(mf, directory); + await git(["checkout", "-b", "feature"], directory); + await writeFile(path.join(directory, "feature.txt"), "feature branch\n"); + await git(["add", "."], directory); + await git(["commit", "-m", "feature"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "feature", + ], + directory + ); + const mainOnly = await rpc(mf, "fork", ["main-only"], "Repo"); + const all = await rpc( + mf, + "fork", + ["all", { defaultBranchOnly: false, description: "all refs" }], + "Repo" + ); + expect(mainOnly.defaultBranch).toBe("main"); + expect((await rpc(mf, "info", [], "all")).description).toBe("all refs"); + expect( + await rpc(mf, "readFile", [{ ref: "feature", path: "feature.txt" }], "all") + ).toMatchObject({ bytes: [...new TextEncoder().encode("feature branch\n")] }); + expect( + await rpc( + mf, + "readFile", + [{ ref: "feature", path: "feature.txt" }], + "main-only" + ) + ).toBeNull(); + expect(all.token).toEqual(expect.any(String)); +}); + +test("artifacts: log pages commits and files resolve only valid paths", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + const directory = path.join(await useTmp(), "source"); + const created = await pushFixture(mf, directory); + const firstHash = (await git(["rev-parse", "HEAD"], directory)).stdout.trim(); + await writeFile(path.join(directory, "second.txt"), "second\n"); + await mkdir(path.join(directory, "folder")); + await writeFile(path.join(directory, "folder", "nested.txt"), "nested\n"); + await git(["add", "."], directory); + await git(["commit", "-m", "second"], directory); + await git( + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "main", + ], + directory + ); + const secondHash = ( + await git(["rev-parse", "HEAD"], directory) + ).stdout.trim(); + expect( + (await rpc(mf, "log", [{ limit: 1 }], "Repo")).map( + (commit: { hash: string }) => commit.hash + ) + ).toEqual([secondHash]); + expect( + (await rpc(mf, "log", [{ limit: 1, offset: 1 }], "Repo")).map( + (commit: { hash: string }) => commit.hash + ) + ).toEqual([firstHash]); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "../hello.txt" }], "Repo") + ).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "folder" }], "Repo") + ).toBeNull(); + expect( + await rpc( + mf, + "readFile", + [{ ref: "main", path: "folder/nested.txt" }], + "Repo" + ) + ).toMatchObject({ bytes: [...new TextEncoder().encode("nested\n")] }); + expect( + await rpc(mf, "readFile", [{ ref: "main", path: "missing" }], "Repo") + ).toBeNull(); + expect( + await rpc(mf, "readFile", [{ ref: "missing", path: "hello.txt" }], "Repo") + ).toBeNull(); +}); + +test("artifacts: an explicit reset removes both metadata and Git repositories", async ({ + expect, +}) => { + const persistence = await useTmp(); + const first = new Miniflare({ + ...options(), + resourcePersistencePath: persistence, + }); + useDispose(first); + const created = await pushFixture(first, path.join(await useTmp(), "source")); + await first.dispose(); + await removeDir(path.join(persistence, "artifacts")); + const second = new Miniflare({ + ...options(), + resourcePersistencePath: persistence, + }); + useDispose(second); + expect((await rpc(second, "list")).total).toBe(0); + const fresh = await rpc(second, "create", ["Repo"]); + expect(fresh.id).not.toBe(created.id); + const oldToken = await fetch( + `${fresh.remote}/info/refs?service=git-upload-pack`, + { headers: { Authorization: `Bearer ${created.token}` } } + ); + expect(oldToken.status).toBe(401); + await oldToken.body?.cancel(); +}); + +test("artifacts: missing Git gives an actionable error during local startup", async ({ + expect, +}) => { + const root = await useTmp(); + const previousPath = process.env.PATH; + process.env.PATH = root; + try { + for (const remote of [false, null]) { + const mf = new Miniflare(options("test", "REPOS", remote)); + await expect(mf.ready).rejects.toThrow( + /requires Git 2\.32 or newer on PATH.*Install Git.*restart/i + ); + // dispose() preserves the startup error after cleaning up the instance. + await expect(mf.dispose()).rejects.toThrow( + /requires Git 2\.32 or newer on PATH/ + ); + } + await expect(startGitSidecar(path.join(root, "repos"))).rejects.toThrow( + /requires Git 2\.32 or newer on PATH/ + ); + } finally { + process.env.PATH = previousPath; + } +}); + +test.skipIf(process.platform === "win32")( + "artifacts: Git older than 2.32 fails during local startup with upgrade instructions", + async ({ expect }) => { + const root = await useTmp(); + const fakeGit = path.join(root, "git"); + await writeFile(fakeGit, "#!/bin/sh\nprintf 'git version 2.31.99\\n'\n"); + await chmod(fakeGit, 0o755); + const previousPath = process.env.PATH; + process.env.PATH = root; + try { + const mf = new Miniflare(options()); + await expect(mf.ready).rejects.toThrow( + /requires Git 2\.32 or newer.*Found git version 2\.31\.99.*Upgrade Git.*restart/i + ); + await expect(mf.dispose()).rejects.toThrow(/requires Git 2\.32 or newer/); + await expect(startGitSidecar(path.join(root, "repos"))).rejects.toThrow( + /requires Git 2\.32 or newer.*Upgrade Git/ + ); + } finally { + process.env.PATH = previousPath; + } + } +); + +test("artifacts: sidecar creates a repository without host Git config", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "create", + namespace: "test", + name: "repo", + generation: "test-generation", + defaultBranch: "main", + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + expect(JSON.parse(body)).toEqual({ defaultBranch: "main", refs: {} }); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: stale deletion cannot remove a replacement generation", async ({ + expect, +}) => { + const root = path.join(await useTmp(), "repos"); + const sidecar = await startGitSidecar(root); + try { + const deleteGeneration = async (generation: string) => { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "delete", + namespace: "test", + name: "repo", + generation, + }), + } + ); + expect(response.status).toBe(200); + await response.body?.cancel(); + }; + await createSidecarRepository(sidecar, "original-generation"); + await deleteGeneration("original-generation"); + await createSidecarRepository(sidecar, "replacement-generation"); + await deleteGeneration("original-generation"); + expect( + await new GitClient(repositoryPath(root, "test", "repo")).generation() + ).toBe("replacement-generation"); + } finally { + await sidecar.close(); + } +}); + +test.skipIf(process.platform === "win32")( + "artifacts: smart HTTP pushes cannot run repository-local hooks", + async ({ expect }) => { + const root = path.join(await useTmp(), "repos"); + const sidecar = await startGitSidecar(root); + try { + await createSidecarRepository(sidecar, "hook-generation"); + const repository = repositoryPath(root, "test", "repo"); + const hooks = path.join(await useTmp(), "hooks"); + const marker = path.join(await useTmp(), "hook-ran"); + await mkdir(hooks); + const hook = path.join(hooks, "pre-receive"); + await writeFile(hook, `#!/bin/sh\nprintf ran > '${marker}'\n`); + await chmod(hook, 0o755); + await git(["-C", repository, "config", "core.hooksPath", hooks]); + + const source = path.join(await useTmp(), "source"); + await git(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "fixture\n"); + await git(["add", "README"], source); + await git(["commit", "-m", "fixture"], source); + await git( + ["push", `http://${sidecar.address}/git/test/repo.git`, "main"], + source, + { + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `X-Local-Artifacts-Backend: ${sidecar.secret}`, + GIT_CONFIG_KEY_2: "http.extraHeader", + GIT_CONFIG_VALUE_2: "X-Local-Artifacts-Generation: hook-generation", + } + ); + await expect(stat(marker)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await sidecar.close(); + } + } +); + +test("artifacts: shallow imports stay shallow when cloned with Git protocol v2", async ({ + expect, +}) => { + const workspace = await useTmp(); + const source = path.join(workspace, "source"); + await git(["init", "--initial-branch=main", source]); + for (const index of [1, 2, 3]) { + await writeFile(path.join(source, "README"), `version ${index}\n`); + await git(["add", "README"], source); + await git(["commit", "-m", `version ${index}`], source); + } + + const sidecar = await startGitSidecar(path.join(workspace, "repos")); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "import", + namespace: "test", + name: "repo", + generation: "shallow-generation", + sourceUrl: pathToFileURL(source).href, + branch: "main", + depth: 1, + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + + const gitHeaders = { + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_1: "http.extraHeader", + GIT_CONFIG_VALUE_1: `X-Local-Artifacts-Backend: ${sidecar.secret}`, + GIT_CONFIG_KEY_2: "http.extraHeader", + GIT_CONFIG_VALUE_2: "X-Local-Artifacts-Generation: shallow-generation", + }; + const clone = path.join(workspace, "clone"); + const remote = `http://${sidecar.address}/git/test/repo.git`; + await git( + ["-c", "protocol.version=2", "clone", remote, clone], + undefined, + gitHeaders + ); + expect( + (await git(["rev-parse", "--is-shallow-repository"], clone)).stdout.trim() + ).toBe("true"); + expect( + (await git(["rev-list", "--count", "HEAD"], clone)).stdout.trim() + ).toBe("1"); + await git( + ["-c", "protocol.version=2", "fetch", "origin"], + clone, + gitHeaders + ); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: disposing the sidecar closes its private listener", async ({ + expect, +}) => { + const sidecar = await startGitSidecar(path.join(await useTmp(), "repos")); + const address = `http://${sidecar.address}/missing`; + try { + const response = await fetch(address, { + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + }); + expect(response.status).toBe(404); + await response.body?.cancel(); + } finally { + await sidecar.close(); + } + await expect(fetch(address)).rejects.toThrow(); +}); diff --git a/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts b/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts new file mode 100644 index 00000000000..f77723c754a --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/lifecycle.spec.ts @@ -0,0 +1,627 @@ +import { execFile, spawn } from "node:child_process"; +import { once } from "node:events"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { build } from "esbuild"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; +import type { MiniflareOptions } from "miniflare"; + +const namespace = "lifecycle"; +const workerSource = path.resolve( + __dirname, + "../../../src/workers/artifacts/binding.worker.ts" +); +const clientScript = ` +export default { + async fetch(request, env) { + const { method, name, args = [] } = await request.json(); + try { + const target = name === undefined ? env.REPOS : await env.REPOS.get(name); + return Response.json(await target[method](...args)); + } catch (error) { + return Response.json({ code: error.code, message: error.message }, { status: 400 }); + } + } +};`; + +function options(persistencePath?: string): MiniflareOptions { + return { + cf: false, + ...(persistencePath === undefined + ? {} + : { resourcePersistencePath: persistencePath }), + workers: [ + { + config: { + name: "", + compatibilityDate: "2026-09-03", + env: { + REPOS: { type: "artifacts", namespace, dev: { remote: false } }, + }, + manifest: singleModuleManifest(clientScript), + }, + }, + ], + }; +} + +type RpcResult = Record; + +async function rpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + const response = await mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify({ method, args, name }), + }); + const result = (await response.json()) as RpcResult; + if (!response.ok) { + throw new Error(String(result.message)); + } + return result; +} + +// Run the actual binding worker and its DO inside workerd. Only this test +// source has a controlled clock and a one-shot metadata write fault; neither +// hook exists in the shipped worker. +async function clockedWorker(): Promise { + let source = await readFile(workerSource, "utf8"); + const replacements = [ + ["const createdAt = new Date();", "const createdAt = new Date(testClock);"], + [ + "return Date.parse(token.expiresAt) > Date.now();", + "return Date.parse(token.expiresAt) > testClock;", + ], + [ + "\t// The resolved tail lets the first mutation run immediately.", + `\tprivate failNextSave = false; + failSaveOnce(): void { this.failNextSave = true; } + setTestClock(time: number): void { testClock = time; } + // The resolved tail lets the first mutation run immediately.`, + ], + [ + 'return this.ctx.storage.put("repos", repositories);', + `if (this.failNextSave) { + this.failNextSave = false; + throw new Error("Injected metadata write failure"); + } + return this.ctx.storage.put("repos", repositories);`, + ], + ]; + for (const [original, replacement] of replacements) { + if (source.split(original).length !== 2) { + throw new Error(`Artifacts clock test source changed: ${original}`); + } + source = source.replace(original, replacement); + } + source = `let testClock = 0;\n${source}\n${clockedFetch}`; + const result = await build({ + stdin: { + contents: source, + resolveDir: path.dirname(workerSource), + sourcefile: workerSource, + loader: "ts", + }, + bundle: true, + format: "esm", + write: false, + external: ["cloudflare:workers"], + }); + return result.outputFiles[0].text; +} + +const clockedFetch = ` +export default { + async fetch(request, env) { + const { action, name, method, args = [], time, token, scope } = await request.json(); + const namespace = env.localArtifactsNamespace; + const state = namespace.get(namespace.idFromName("lifecycle")); + if (action === "clock") { + await state.setTestClock(time); + return Response.json(true); + } + if (action === "failSave") { + await state.failSaveOnce(); + return Response.json(true); + } + if (action === "backendStatus" || action === "failDelete") { + return env.gitBackend.fetch("http://backend/__local_artifacts__", { + method: "POST", body: JSON.stringify({ action: action === "failDelete" ? "failDelete" : "status" }) + }); + } + if (action === "git") { + return state.fetch(new Request( + "http://localhost/git/lifecycle/repo.git/info/refs?service=git-" + scope + "-pack", + { headers: { Authorization: "Bearer " + token } } + )); + } + try { + const target = name === undefined ? state : await state.get(name); + return Response.json(await target[method](...args)); + } catch (error) { + return Response.json({ code: error.code, message: error.message }, { status: 400 }); + } + } +}; +// The lifecycle fixture tests storage failure handling, not event delivery. +export class EventSink extends WorkerEntrypoint { + async sendEvent() {} +}`; + +function clockOptions( + script: string, + persistencePath?: string +): MiniflareOptions { + return { + cf: false, + ...(persistencePath === undefined + ? {} + : { resourcePersistencePath: persistencePath }), + workers: [ + { + config: { + name: "clocked", + compatibilityDate: "2026-09-03", + manifest: singleModuleManifest(script), + env: { + config: { + type: "json", + value: { namespace, origin: "http://localhost" }, + }, + localArtifactsNamespace: { + type: "durable-object", + worker: "clocked", + exportName: "LocalArtifactsNamespaceObject", + }, + gitBackend: { type: "worker", worker: "backend" }, + QUEUE_WORKER: { + type: "worker", + worker: "clocked", + exportName: "EventSink", + }, + }, + exports: { + LocalArtifactsNamespaceObject: { + type: "durable-object", + storage: "sqlite", + }, + }, + }, + }, + { + config: { + name: "backend", + compatibilityDate: "2026-09-03", + manifest: singleModuleManifest(`let deleteCount = 0; + let failDelete = false; + export default { + async fetch(request) { + if (new URL(request.url).pathname !== "/__local_artifacts__") + return new Response("Git advertisement", { status: 200 }); + const { action } = await request.json(); + if (action === "status") return Response.json({ deleteCount }); + if (action === "failDelete") { failDelete = true; return Response.json(true); } + if (action === "delete") { + if (failDelete) { + failDelete = false; + return new Response("Injected Git deletion failure", { status: 500 }); + } + deleteCount++; + } + if (action === "create" || action === "fork") + return Response.json({ defaultBranch: "main", refs: {} }); + return Response.json({ refs: {} }); + } + }`), + }, + }, + ], + }; +} + +async function clockRequest( + mf: Miniflare, + request: Record +): Promise { + return mf.dispatchFetch("http://localhost", { + method: "POST", + body: JSON.stringify(request), + }); +} + +async function setClock(mf: Miniflare, time: number): Promise { + const response = await clockRequest(mf, { action: "clock", time }); + await response.body?.cancel(); +} + +async function clockRpc( + mf: Miniflare, + method: string, + args: unknown[] = [], + name?: string +): Promise { + const response = await clockRequest(mf, { method, args, name }); + const result = (await response.json()) as RpcResult; + if (!response.ok) { + throw new Error(JSON.stringify(result)); + } + return result; +} + +async function gitStatus( + mf: Miniflare, + token: string, + scope: "upload" | "receive" +): Promise { + const response = await clockRequest(mf, { + action: "git", + token, + scope, + }); + await response.body?.cancel(); + return response.status; +} + +const start = Date.parse("2026-01-01T00:00:00.000Z"); + +test("artifacts: a failed metadata write cannot delete Git contents", async ({ + expect, +}) => { + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + const created = await clockRpc(mf, "create", ["repo"]); + await (await clockRequest(mf, { action: "failSave" })).body?.cancel(); + const failed = await clockRequest(mf, { method: "delete", args: ["repo"] }); + expect(failed.status).toBe(400); + await failed.body?.cancel(); + expect((await clockRpc(mf, "info", [], "repo")).id).toBe(created.id); + expect( + await (await clockRequest(mf, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 0 }); + expect(await clockRpc(mf, "delete", ["repo"])).toBe(true); + expect( + await (await clockRequest(mf, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 1 }); +}); + +test("artifacts: failed Git deletion stays retryable across a restart", async ({ + expect, +}) => { + const script = await clockedWorker(); + const root = await useTmp(); + const first = new Miniflare(clockOptions(script, root)); + useDispose(first); + const created = await clockRpc(first, "create", ["repo"]); + await (await clockRequest(first, { action: "failDelete" })).body?.cancel(); + const failed = await clockRequest(first, { + method: "delete", + args: ["repo"], + }); + expect(failed.status).toBe(400); + await failed.body?.cancel(); + expect((await clockRpc(first, "list")).total).toBe(0); + const inaccessible = await clockRequest(first, { + method: "get", + args: ["repo"], + }); + expect(inaccessible.status).toBe(400); + await inaccessible.body?.cancel(); + expect( + await (await clockRequest(first, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 0 }); + await first.dispose(); + + const restarted = new Miniflare(clockOptions(script, root)); + useDispose(restarted); + expect(await clockRpc(restarted, "delete", ["repo"])).toBe(true); + expect( + await (await clockRequest(restarted, { action: "backendStatus" })).json() + ).toEqual({ deleteCount: 1 }); + const replacement = await clockRpc(restarted, "create", ["repo"]); + expect(replacement.id).not.toBe(created.id); +}); + +test("artifacts: token expiry is exclusive at the exact workerd clock boundary and persists", async ({ + expect, +}) => { + const script = await clockedWorker(); + const root = await useTmp(); + const first = new Miniflare(clockOptions(script, root)); + useDispose(first); + await setClock(first, start); + await clockRpc(first, "create", ["repo"]); + const read = await clockRpc(first, "createToken", ["read", 60], "repo"); + const write = await clockRpc(first, "createToken", ["write", 60], "repo"); + const expiration = start + 60_000; + expect(read.expiresAt).toBe(new Date(expiration).toISOString()); + expect(String(read.plaintext)).toContain(`?expires=${expiration / 1000}`); + expect(write.expiresAt).toBe(read.expiresAt); + const beforeExpiry = await clockRpc(first, "listTokens", [], "repo"); + expect(beforeExpiry.tokens).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: read.id, + scope: "read", + state: "active", + expiresAt: read.expiresAt, + }), + expect.objectContaining({ + id: write.id, + scope: "write", + state: "active", + expiresAt: write.expiresAt, + }), + ]) + ); + expect(JSON.stringify(beforeExpiry)).not.toContain(String(read.plaintext)); + await setClock(first, expiration - 1); + expect((await clockRpc(first, "listTokens", [], "repo")).total).toBe(3); + expect(await gitStatus(first, String(read.plaintext), "upload")).toBe(200); + expect(await gitStatus(first, String(read.plaintext), "receive")).toBe(403); + expect(await gitStatus(first, String(write.plaintext), "receive")).toBe(200); + expect(await gitStatus(first, String(write.plaintext), "upload")).toBe(200); + + await setClock(first, expiration); + expect((await clockRpc(first, "listTokens", [], "repo")).total).toBe(1); + expect(await gitStatus(first, String(read.plaintext), "upload")).toBe(401); + expect(await gitStatus(first, String(write.plaintext), "receive")).toBe(401); + expect(await gitStatus(first, String(read.plaintext), "receive")).toBe(401); + await first.dispose(); + + const restarted = new Miniflare(clockOptions(script, root)); + useDispose(restarted); + await setClock(restarted, expiration + 1); + expect((await clockRpc(restarted, "listTokens", [], "repo")).total).toBe(1); + expect(await gitStatus(restarted, String(write.plaintext), "upload")).toBe( + 401 + ); + expect(await gitStatus(restarted, String(read.plaintext), "upload")).toBe( + 401 + ); + const replacement = await clockRpc( + restarted, + "createToken", + ["read", 60], + "repo" + ); + expect( + await gitStatus(restarted, String(replacement.plaintext), "upload") + ).toBe(200); + expect( + await clockRpc(restarted, "revokeToken", [replacement.id], "repo") + ).toBe(true); + expect( + await gitStatus(restarted, String(replacement.plaintext), "upload") + ).toBe(401); +}); + +test("artifacts: concurrent same-name creates and forks commit one winner each", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await rpc(mf, "create", ["source"]); + const creates = await Promise.allSettled([ + rpc(mf, "create", ["Target"]), + rpc(mf, "create", ["target"]), + ]); + expect(creates.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + const forks = await Promise.allSettled([ + rpc(mf, "fork", ["Child"], "source"), + rpc(mf, "fork", ["child"], "source"), + ]); + expect(forks.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + expect((await rpc(mf, "list")).total).toBe(3); + const child = await rpc(mf, "info", [], "child"); + expect(child.source).toBe(`artifacts:${namespace}/source`); +}); + +test("artifacts: a delayed push cannot update a replacement repository", async ({ + expect, +}) => { + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + const original = await clockRpc(mf, "create", ["repo"]); + await clockRpc(mf, "delete", ["repo"]); + const replacement = await clockRpc(mf, "create", ["repo"]); + expect(replacement.id).not.toBe(original.id); + const before = await clockRpc(mf, "info", [], "repo"); + await expect( + clockRpc(mf, "syncAfterPush", ["repo", original.id, {}]) + ).rejects.toThrow(/NOT_FOUND/); + const after = await clockRpc(mf, "info", [], "repo"); + expect(after.updatedAt).toBe(before.updatedAt); + expect(after.lastPushAt).toBe(before.lastPushAt); +}); + +test("artifacts: import and fork metadata remain distinct across concurrent creations", async ({ + expect, +}) => { + // The backend is a local stub: this exercises namespace state transitions, + // not network cloning (which is covered by the native Git tests). + const mf = new Miniflare(clockOptions(await clockedWorker())); + useDispose(mf); + const results = await Promise.allSettled([ + clockRpc(mf, "import", [ + { + source: { + url: "https://example.com/source.git", + branch: "release", + depth: 1, + }, + target: { name: "Imported", opts: { description: "remote copy" } }, + }, + ]), + clockRpc(mf, "import", [ + { + source: { url: "https://example.com/other.git" }, + target: { name: "imported" }, + }, + ]), + ]); + expect(results.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + const imported = await clockRpc(mf, "info", [], "imported"); + expect(imported.source).toMatch( + /^git:https:\/\/example\.com\/(source|other)\.git$/ + ); + const fork = await clockRpc( + mf, + "fork", + ["forked", { readOnly: true }], + "imported" + ); + expect(fork.name).toBe("forked"); + expect(await clockRpc(mf, "info", [], "forked")).toMatchObject({ + readOnly: true, + source: `artifacts:${namespace}/${imported.name}`, + }); + expect((await clockRpc(mf, "list")).total).toBe(2); +}); + +test("artifacts: concurrent independent mutations preserve all repositories and tokens", async ({ + expect, +}) => { + const mf = new Miniflare(options()); + useDispose(mf); + await Promise.all( + ["one", "two", "three"].map((name) => rpc(mf, "create", [name])) + ); + await Promise.all( + ["one", "two", "three"].map(async (name) => { + await Promise.all([ + rpc(mf, "createToken", ["read", 60], name), + rpc(mf, "fork", [`${name}-fork`], name), + ]); + }) + ); + for (const name of ["one", "two", "three"]) { + expect((await rpc(mf, "listTokens", [], name)).total).toBe(2); + expect((await rpc(mf, "info", [], `${name}-fork`)).source).toBe( + `artifacts:${namespace}/${name}` + ); + } + expect((await rpc(mf, "list")).total).toBe(6); +}); + +async function killChild(child: ReturnType): Promise { + if (child.exitCode !== null || child.pid === undefined) { + return; + } + const closed = once(child, "close"); + if (process.platform === "win32") { + // Killing only Node would orphan workerd and leave its listener and + // SQLite files open. taskkill /T stops the entire child process tree. + await promisify(execFile)("taskkill", [ + "/PID", + String(child.pid), + "/T", + "/F", + ]); + } else { + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") { + throw error; + } + } + } + await closed; +} + +// The child deliberately never calls dispose(): the parent kills it after +// create() has committed, then starts a new listener against the same root. +test("artifacts: abrupt child-process exit leaves committed repositories restartable", async ({ + expect, +}) => { + const root = await useTmp(); + const marker = path.join(root, "ready"); + const childScript = path.join(root, "child.cjs"); + const modulePath = path.resolve(__dirname, "../../../dist/src/index.js"); + await writeFile( + childScript, + ` +const { Miniflare } = require(${JSON.stringify(modulePath)}); +const { writeFileSync } = require("node:fs"); +const mf = new Miniflare({ + cf: false, + resourcePersistencePath: ${JSON.stringify(root)}, + workers: [{ config: { + name: "", compatibilityDate: "2026-09-03", + env: { REPOS: { type: "artifacts", namespace: "lifecycle", dev: { remote: false } } }, + manifest: { mainModule: "index.mjs", modulesRoot: ${JSON.stringify(root)}, modules: { "index.mjs": { type: "esm", contents: "export default { fetch() { return new Response('ok') } }" } } } + }}] +}); +(async () => { + const { REPOS } = await mf.getBindings(); + const created = await REPOS.create("survivor"); + writeFileSync(${JSON.stringify(marker)}, JSON.stringify(created)); + process.stdout.write("READY\\n"); +})().catch(error => { process.stderr.write(String(error)); process.exitCode = 1; }); +` + ); + const child = spawn(process.execPath, [childScript], { + stdio: ["ignore", "pipe", "pipe"], + // workerd is a subprocess: kill the process group to avoid orphaning it. + detached: process.platform !== "win32", + }); + let listener = ""; + let token = ""; + let stderr = ""; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + try { + if (child.stdout === null) { + throw new Error("Child stdout pipe unavailable"); + } + await Promise.race([ + once(child.stdout, "data", { signal: AbortSignal.timeout(15_000) }), + once(child, "exit").then(() => { + throw new Error(`Child exited before ready: ${stderr}`); + }), + ]); + const created = JSON.parse(await readFile(marker, "utf8")) as { + remote: string; + token: string; + }; + expect(created.remote).toMatch(/^http:\/\/127\.0\.0\.1:/); + listener = `${created.remote}/info/refs?service=git-upload-pack`; + token = created.token; + } finally { + await killChild(child); + } + await expect( + fetch(listener, { signal: AbortSignal.timeout(2_000) }) + ).rejects.toThrow(); + const restarted = new Miniflare(options(root)); + useDispose(restarted); + const info = await rpc(restarted, "info", [], "survivor"); + expect(info.name).toBe("survivor"); + const advertised = await fetch( + `${info.remote}/info/refs?service=git-upload-pack`, + { + headers: { Authorization: `Bearer ${token}` }, + } + ); + expect(advertised.status).toBe(200); + await advertised.body?.cancel(); + const created = await rpc(restarted, "create", ["after-restart"]); + expect(created.name).toBe("after-restart"); + expect((await rpc(restarted, "list")).total).toBe(2); +}); diff --git a/packages/miniflare/test/plugins/artifacts/storage.spec.ts b/packages/miniflare/test/plugins/artifacts/storage.spec.ts new file mode 100644 index 00000000000..33407aeb46a --- /dev/null +++ b/packages/miniflare/test/plugins/artifacts/storage.spec.ts @@ -0,0 +1,190 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { Miniflare } from "miniflare"; +import { test } from "vitest"; +import { GitClient, runGit } from "../../../src/plugins/artifacts/git-client"; +import { startGitSidecar } from "../../../src/plugins/artifacts/git-sidecar"; +import { + assertSupportedGitLayout, + repositoryDirectory, + repositoryPath, +} from "../../../src/plugins/artifacts/storage"; +import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; + +const namespace = "namespace-with-a-long-name-for-platform-testing"; + +test("artifacts: repository paths are compact, deterministic and namespace-scoped", ({ + expect, +}) => { + const directory = repositoryDirectory(namespace, "repo"); + expect(directory).toMatch(/^[0-9a-f]{32}\.git$/); + expect(repositoryDirectory(namespace, "repo")).toBe(directory); + expect(repositoryDirectory("other", "repo")).not.toBe(directory); + expect(repositoryDirectory(namespace, "other")).not.toBe(directory); + expect(repositoryDirectory("ab", "c")).not.toBe( + repositoryDirectory("a", "bc") + ); + expect(repositoryDirectory("../namespace", "../repo")).toMatch( + /^[0-9a-f]{32}\.git$/ + ); +}); + +test("artifacts: sidecar rejects legacy storage without removing it", async ({ + expect, +}) => { + const root = await useTmp(); + const legacy = path.join(root, "a".repeat(64)); + await mkdir(legacy); + await writeFile(path.join(legacy, "sentinel"), "preserve me"); + await expect(startGitSidecar(root)).rejects.toThrow( + /older draft layout.*Back up/ + ); + expect(await readFile(path.join(legacy, "sentinel"), "utf8")).toBe( + "preserve me" + ); +}); + +test("artifacts: restart removes interrupted creation and import staging", async ({ + expect, +}) => { + const root = await useTmp(); + const creating = await mkdtemp(path.join(root, ".artifacts-stage-")); + const importing = await mkdtemp(path.join(root, ".artifacts-import-")); + await writeFile(path.join(creating, "partial"), "private data"); + await writeFile(path.join(importing, "partial"), "private data"); + const unrelated = path.join(root, "unrelated"); + await mkdir(unrelated); + await writeFile(path.join(unrelated, "keep"), "keep me"); + + const sidecar = await startGitSidecar(root); + try { + expect(await readdir(root)).toEqual(["unrelated"]); + expect(await readFile(path.join(unrelated, "keep"), "utf8")).toBe( + "keep me" + ); + } finally { + await sidecar.close(); + } +}); + +test("artifacts: legacy metadata layout is detected without resetting it", async ({ + expect, +}) => { + const root = await useTmp(); + const id = createHash("sha256").update(namespace).digest("hex").slice(0, 32); + const metadata = path.join(root, "artifacts", id, "metadata"); + const legacy = path.join(metadata, `artifacts-${id}`); + await mkdir(legacy, { recursive: true }); + await writeFile(path.join(legacy, "sentinel"), "preserve me"); + await expect(assertSupportedGitLayout(metadata)).rejects.toThrow( + /older draft layout.*Back up/ + ); + expect(await readFile(path.join(legacy, "sentinel"), "utf8")).toBe( + "preserve me" + ); +}); + +test("artifacts: namespace metadata and Git data survive restart under paths with spaces", async ({ + expect, +}) => { + const root = path.join(await useTmp(), "development project"); + const config = { + name: "test", + compatibilityDate: "2026-09-03", + env: { + REPOS: { type: "artifacts" as const, namespace, dev: { remote: false } }, + }, + manifest: singleModuleManifest( + `export default { fetch() { return new Response("ok"); } };` + ), + }; + type Bindings = { + REPOS: { + create( + name: string + ): Promise<{ id: string; remote: string; token: string }>; + get(name: string): Promise<{ info(): Promise<{ id: string }> }>; + }; + }; + const options = { + cf: false as const, + resourcePersistencePath: root, + workers: [{ config }], + }; + const first = new Miniflare(options); + useDispose(first); + const { REPOS } = await first.getBindings(); + const created = await REPOS.create("repo"); + const id = created.id; + await first.dispose(); + + const second = new Miniflare(options); + useDispose(second); + const restarted = await second.getBindings(); + expect(await (await restarted.REPOS.get("repo")).info()).toMatchObject({ + id, + }); + const namespaces = await readdir(path.join(root, "artifacts")); + expect(namespaces).toHaveLength(1); + const persisted = path.join(root, "artifacts", namespaces[0]); + expect(await readdir(path.join(persisted, "git"))).toEqual([ + repositoryDirectory(namespace, "repo"), + ]); + expect( + (await readdir(path.join(persisted, "metadata"))).length + ).toBeGreaterThan(0); +}); + +test("artifacts: long Git storage roots can initialize and repack repositories", async ({ + expect, +}) => { + const workspace = await useTmp(); + const root = path.join(workspace, "project with spaces " + "x".repeat(32)); + const sidecar = await startGitSidecar(root); + try { + const response = await fetch( + `http://${sidecar.address}/__local_artifacts__`, + { + method: "POST", + headers: { "X-Local-Artifacts-Backend": sidecar.secret }, + body: JSON.stringify({ + action: "create", + namespace, + name: "repo", + generation: "test-generation", + }), + } + ); + const body = await response.text(); + expect(response.status, body).toBe(200); + const repository = repositoryPath(root, namespace, "repo"); + const source = path.join(workspace, "source"); + await runGit(["init", "--initial-branch=main", source]); + await writeFile(path.join(source, "README"), "long-path regression\n"); + await runGit(["-C", source, "add", "README"]); + await runGit([ + "-C", + source, + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-m", + "fixture", + ]); + await runGit(["-C", source, "push", repository, "main"]); + await runGit(["-C", repository, "repack", "-ad"]); + expect( + (await readdir(path.join(repository, "objects", "pack"))).some((file) => + file.endsWith(".pack") + ) + ).toBe(true); + expect(await new GitClient(repository).generation()).toBe( + "test-generation" + ); + } finally { + await sidecar.close(); + } +}); diff --git a/packages/remote-bindings/src/maybe-start-or-update-session.test.ts b/packages/remote-bindings/src/maybe-start-or-update-session.test.ts index f5d6601ab59..d0fdd8f343b 100644 --- a/packages/remote-bindings/src/maybe-start-or-update-session.test.ts +++ b/packages/remote-bindings/src/maybe-start-or-update-session.test.ts @@ -35,6 +35,20 @@ describe("maybeStartOrUpdateRemoteProxySession", () => { } ); + it("keeps Artifacts offline unless remote is explicitly enabled", ({ + expect, + }) => { + const local = { type: "artifacts" as const, namespace: "local" }; + const remote = { + type: "artifacts" as const, + namespace: "remote", + remote: true, + }; + expect(pickRemoteBindings({ LOCAL: local, REMOTE: remote })).toEqual({ + REMOTE: remote, + }); + }); + it("updates an existing session when all remote bindings are removed", async ({ expect, }) => { diff --git a/packages/vite-plugin-cloudflare/src/__tests__/artifacts-local.spec.ts b/packages/vite-plugin-cloudflare/src/__tests__/artifacts-local.spec.ts new file mode 100644 index 00000000000..cd12ff13f92 --- /dev/null +++ b/packages/vite-plugin-cloudflare/src/__tests__/artifacts-local.spec.ts @@ -0,0 +1,180 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath as pathFromUrl } from "node:url"; +import { promisify } from "node:util"; +import { removeDirSync } from "@cloudflare/workers-utils"; +import { runInTempDir } from "@cloudflare/workers-utils/test-helpers"; +import { createServer } from "vite"; +import { afterEach, test, vi } from "vitest"; +import { cloudflare } from "../index"; + +vi.mock("node:fs", async (importOriginal) => { + const original = await importOriginal(); + const { fileURLToPath } = await import("node:url"); + const source = fileURLToPath(new URL("../workers/", import.meta.url)); + const built = fileURLToPath(new URL("../../dist/workers/", import.meta.url)); + return { + ...original, + readFileSync( + file: Parameters[0], + options: Parameters[1] + ) { + if ( + typeof file === "string" && + file.startsWith(source) && + file.endsWith(".js") + ) { + file = built + file.slice(source.length); + } + return original.readFileSync(file, options); + }, + }; +}); + +const git = promisify(execFile); + +runInTempDir(); +afterEach(() => vi.unstubAllEnvs()); + +test.for([undefined, false] as const)( + "Vite dev serves local Artifacts without credentials (remote=%s)", + async (remote, { expect, onTestFinished }) => { + vi.stubEnv("CLOUDFLARE_API_TOKEN", undefined); + vi.stubEnv("CLOUDFLARE_ACCOUNT_ID", undefined); + // On Windows, the system temp directory can use an 8.3 path (RUNNER~1). + // Vite resolves that entry as /index.js, but cannot load it from the + // virtual Worker module. Keep the fixture under the checkout instead. + const fixtureDir = fs.mkdtempSync( + path.join( + fs.realpathSync(path.dirname(pathFromUrl(import.meta.url))), + "artifacts-local-" + ) + ); + onTestFinished(() => removeDirSync(fixtureDir)); + fs.writeFileSync( + path.join(fixtureDir, "package.json"), + JSON.stringify({ type: "module" }) + ); + fs.writeFileSync( + path.join(fixtureDir, "wrangler.jsonc"), + JSON.stringify({ + name: "vite-artifacts-local", + main: "index.js", + compatibility_date: "2026-09-03", + artifacts: [ + { + binding: "REPOS", + namespace: "examples", + ...(remote === undefined ? {} : { remote }), + }, + ], + }) + ); + fs.writeFileSync( + path.join(fixtureDir, "index.js"), + `export default { + async fetch(request, env) { + if (new URL(request.url).pathname === "/create") return Response.json(await env.REPOS.create("demo")); + if (new URL(request.url).pathname === "/list") return Response.json(await env.REPOS.list()); + const repo = await env.REPOS.get("demo"); + if (new URL(request.url).pathname === "/file") { + const file = await repo.readFile({ ref: "main", path: "hello.txt" }); + return new Response(file ?? "missing", { status: file ? 200 : 404 }); + } + return Response.json(await repo.info()); + } +};` + ); + const server = await createServer({ + root: fixtureDir, + configFile: false, + logLevel: "silent", + server: { port: 0 }, + plugins: [ + cloudflare({ + configPath: path.join(fixtureDir, "wrangler.jsonc"), + inspectorPort: false, + persistState: false, + }), + ], + }); + let gitUrl: string | undefined; + try { + await server.listen(); + const url = server.resolvedUrls?.local[0]; + if (!url) { + throw new Error("Vite server did not provide a local URL"); + } + const created = (await (await fetch(new URL("create", url))).json()) as { + remote: string; + token: string; + }; + gitUrl = created.remote; + expect(new URL(gitUrl).hostname).toBe("127.0.0.1"); + const directory = path.join(fixtureDir, "git-fixture"); + await git("git", ["init", "--initial-branch=main", directory]); + fs.writeFileSync(path.join(directory, "hello.txt"), "hello artifacts\n"); + // Avoid inheriting host Git credentials or Cloudflare environment. + const inherited = Object.fromEntries( + [ + "PATH", + "SystemRoot", + "WINDIR", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + "LANG", + ].map((key) => [key, process.env[key]]) + ); + const gitOptions = { + cwd: directory, + env: { + ...inherited, + GIT_AUTHOR_NAME: "Example", + GIT_AUTHOR_EMAIL: "example@example.invalid", + GIT_COMMITTER_NAME: "Example", + GIT_COMMITTER_EMAIL: "example@example.invalid", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: path.join(directory, "absent-global-gitconfig"), + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "credential.helper", + GIT_CONFIG_VALUE_0: "", + GIT_TERMINAL_PROMPT: "0", + }, + }; + await git("git", ["add", "hello.txt"], gitOptions); + await git("git", ["commit", "-m", "fixture"], gitOptions); + await git( + "git", + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + gitUrl, + "main", + ], + gitOptions + ); + expect(await (await fetch(new URL("file", url))).text()).toBe( + "hello artifacts\n" + ); + const listed = (await (await fetch(new URL("list", url))).json()) as { + repos: unknown[]; + }; + expect(listed.repos).toHaveLength(1); + const info = (await (await fetch(new URL("info", url))).json()) as { + name: string; + }; + expect(info.name).toBe("demo"); + } finally { + await server.close(); + } + if (gitUrl) { + await expect( + fetch(gitUrl + "/info/refs?service=git-upload-pack") + ).rejects.toThrow(); + } + } +); diff --git a/packages/vitest-plugin/test/artifacts-local.test.ts b/packages/vitest-plugin/test/artifacts-local.test.ts new file mode 100644 index 00000000000..ce29ce16bf4 --- /dev/null +++ b/packages/vitest-plugin/test/artifacts-local.test.ts @@ -0,0 +1,48 @@ +import dedent from "ts-dedent"; +import { test, vitestConfig } from "./helpers"; + +test.for([undefined, false] as const)( + "Workers Vitest uses local Artifacts without credentials (remote=%s)", + async (remote, { expect, seed, vitestRun }) => { + await seed({ + "vitest.config.mts": vitestConfig({ + wrangler: { configPath: "./wrangler.jsonc" }, + }), + "wrangler.jsonc": JSON.stringify({ + name: "artifacts-test-worker", + main: "./index.ts", + compatibility_date: "2026-09-03", + artifacts: [ + { + binding: "REPOS", + namespace: "examples", + ...(remote === undefined ? {} : { remote }), + }, + ], + }), + "index.ts": dedent` + export default { + async fetch(request, env) { + return Response.json(await env.REPOS.list()); + } + }; + `, + "index.test.ts": dedent` + import { env } from "cloudflare:test"; + import { it } from "vitest"; + + it("creates, gets and lists a repository locally", async ({ expect }) => { + const created = await env.REPOS.create("demo"); + expect(new URL(created.remote).hostname).toBe("127.0.0.1"); + const repo = await env.REPOS.get("demo"); + expect((await repo.info()).name).toBe("demo"); + expect((await env.REPOS.list()).repos).toHaveLength(1); + expect(await repo.readFile({ ref: "main", path: "missing.txt" })).toBeNull(); + }); + `, + }); + + const result = await vitestRun(); + await expect(result.exitCode, result.stderr).resolves.toBe(0); + } +); diff --git a/packages/workers-utils/src/config/binding-local-support.ts b/packages/workers-utils/src/config/binding-local-support.ts index 724bb7f2619..636a8a8d96a 100644 --- a/packages/workers-utils/src/config/binding-local-support.ts +++ b/packages/workers-utils/src/config/binding-local-support.ts @@ -58,6 +58,7 @@ const BINDING_LOCAL_SUPPORT: Record< // TODO: Miniflare currently ignores `remote: true` on queues, tracked in #13727. queue: "local-and-remote", flagship: "local-and-remote", + artifacts: "local-and-remote", vectorize: "remote", mtls_certificate: "remote", @@ -70,7 +71,6 @@ const BINDING_LOCAL_SUPPORT: Record< ai_search_namespace: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", media: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", - artifacts: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", vpc_service: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", vpc_network: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", agent_memory: "DO-NOT-USE-this-resource-will-never-have-a-local-simulator", diff --git a/packages/workers-utils/src/print-bindings.ts b/packages/workers-utils/src/print-bindings.ts index f07de4ce3bb..dd13beca9ee 100644 --- a/packages/workers-utils/src/print-bindings.ts +++ b/packages/workers-utils/src/print-bindings.ts @@ -457,12 +457,14 @@ export function printBindings( if (artifacts.length > 0) { output.push( - ...artifacts.map(({ binding, namespace }) => { + ...artifacts.map(({ binding, namespace, remote }) => { return { name: binding, type: getBindingTypeFriendlyName("artifacts"), value: namespace, - mode: getMode({ isSimulatedLocally: false }), + mode: getMode({ + isSimulatedLocally: context.remoteBindingsDisabled || !remote, + }), }; }) ); diff --git a/packages/workers-utils/tests/config/binding-local-support.test.ts b/packages/workers-utils/tests/config/binding-local-support.test.ts index 63c5eb6b989..2f7ee953d98 100644 --- a/packages/workers-utils/tests/config/binding-local-support.test.ts +++ b/packages/workers-utils/tests/config/binding-local-support.test.ts @@ -90,6 +90,12 @@ describe("validateBindingRemoteSetting", () => { expect(() => validateBindingRemoteSetting("flagship", undefined, warn) ).not.toThrow(); + expect(() => + validateBindingRemoteSetting("artifacts", false, warn) + ).not.toThrow(); + expect(() => + validateBindingRemoteSetting("artifacts", undefined, warn) + ).not.toThrow(); expect(warn).not.toHaveBeenCalled(); }); }); diff --git a/packages/workers-utils/tests/print-bindings.test.ts b/packages/workers-utils/tests/print-bindings.test.ts index ccefdc7ff2d..f29177ab10b 100644 --- a/packages/workers-utils/tests/print-bindings.test.ts +++ b/packages/workers-utils/tests/print-bindings.test.ts @@ -80,6 +80,30 @@ describe("printBindings", () => { expect(output).toMatch(/K2 Stream\s+not supported/); }); + test.for([ + { label: "default", remote: undefined, disabled: false, mode: "local" }, + { label: "explicit local", remote: false, disabled: false, mode: "local" }, + { label: "explicit remote", remote: true, disabled: false, mode: "remote" }, + { label: "remote disabled", remote: true, disabled: true, mode: "local" }, + ])( + "labels Artifacts bindings as $mode for $label", + ({ remote, disabled, mode }, { expect }) => { + const output = captureBindings( + { + REPOS: { + type: "artifacts", + namespace: "review", + ...(remote === undefined ? {} : { remote }), + }, + }, + { local: true, remoteBindingsDisabled: disabled } + ); + expect(output).toMatch( + new RegExp(`env\\.REPOS \\(review\\)\\s+Artifacts\\s+${mode}`) + ); + } + ); + test.for([{ class_name: "Sandbox" }, { name: "Sandbox" }])( "prints Durable Object-managed containers using their configured identity %j", (identity, { expect }) => { diff --git a/packages/wrangler/src/__tests__/dev/artifacts-local.test.ts b/packages/wrangler/src/__tests__/dev/artifacts-local.test.ts new file mode 100644 index 00000000000..98c5c1ad879 --- /dev/null +++ b/packages/wrangler/src/__tests__/dev/artifacts-local.test.ts @@ -0,0 +1,206 @@ +import { execFile } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { runInTempDir, seed } from "@cloudflare/workers-utils/test-helpers"; +import { fetch } from "undici"; +import { afterEach, describe, it, vi } from "vitest"; +import { mockConsoleMethods } from "../helpers/mock-console"; +import { runWrangler } from "../helpers/run-wrangler"; +import type { StartDevOptions } from "../../dev"; + +const git = promisify(execFile); +let stopDev: (() => Promise) | undefined; +vi.mock("../../dev/start-dev", async () => { + const actual = await vi.importActual( + "../../dev/start-dev" + ); + return { + ...actual, + async startDev(options: StartDevOptions) { + const result = await actual.startDev(options); + stopDev = () => result.devEnv.teardown(); + return result; + }, + }; +}); + +const worker = `export default { + async fetch(request, env) { + const url = new URL(request.url); + if (url.pathname === "/version") return new Response("initial"); + if (url.pathname === "/create") return Response.json(await env.REPOS.create("demo")); + if (url.pathname === "/list") return Response.json(await env.REPOS.list()); + const repo = await env.REPOS.get("demo"); + if (url.pathname === "/file") { + const file = await repo.readFile({ ref: "main", path: "hello.txt" }); + return new Response(file ?? "missing", { status: file ? 200 : 404 }); + } + return Response.json(await repo.info()); + } +};`; + +type Created = { remote: string; token: string }; + +async function startLocalDev(remote?: false) { + await seed({ + "wrangler.jsonc": JSON.stringify({ + name: "artifacts-local-dev", + main: "index.js", + compatibility_date: "2026-09-03", + artifacts: [ + { + binding: "REPOS", + namespace: "examples", + ...(remote === undefined ? {} : { remote }), + }, + ], + }), + "index.js": worker, + }); + const previousOutput = output.out; + const stopped = runWrangler("dev --port=0 --inspector-port=0", { + WRANGLER_CI_DISABLE_CONFIG_WATCHING: "false", + CLOUDFLARE_API_TOKEN: undefined, + CLOUDFLARE_ACCOUNT_ID: undefined, + }); + + const match = await Promise.race([ + vi.waitUntil( + () => + output.out + .slice(previousOutput.length) + .match(/Ready on (?http:\/\/[^:]+:\d+)/), + { timeout: 15_000 } + ), + stopped.then(() => { + throw new Error("Wrangler dev exited before ready"); + }), + ]).catch((error: unknown) => { + throw new Error( + `Wrangler did not become ready: ${output.out}\n${output.err}`, + { cause: error } + ); + }); + const url = match?.groups?.url; + if (!url) { + throw new Error(`No local dev URL: ${output.out}`); + } + return { url, stopped }; +} + +const output = mockConsoleMethods(); + +async function stopRunningDev(): Promise { + await stopDev?.(); + stopDev = undefined; +} + +async function request(url: string, endpoint: string): Promise { + const response = await fetch(url + endpoint); + if (!response.ok) { + throw new Error(`Request failed: ${response.status}`); + } + return (await response.json()) as T; +} + +describe.sequential("Wrangler dev local Artifacts", () => { + runInTempDir(); + + afterEach(async () => { + await stopRunningDev(); + vi.unstubAllEnvs(); + }); + + it.for([undefined, false] as const)( + "creates, pushes, reads and survives reload (remote=%s)", + async (remote, { expect }) => { + const first = await startLocalDev(remote); + let gitUrl = ""; + try { + const created = await request(first.url, "/create"); + gitUrl = created.remote; + expect(new URL(created.remote).hostname).toBe("127.0.0.1"); + const directory = path.resolve("git-fixture"); + await git("git", ["init", "--initial-branch=main", directory]); + await writeFile(path.join(directory, "hello.txt"), "hello artifacts\n"); + // Do not inherit credential helpers, Cloudflare tokens or Git overrides. + const inherited = Object.fromEntries( + [ + "PATH", + "SystemRoot", + "WINDIR", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + "LANG", + ].map((key) => [key, process.env[key]]) + ); + const env = { + ...inherited, + GIT_AUTHOR_NAME: "Example", + GIT_AUTHOR_EMAIL: "example@example.invalid", + GIT_COMMITTER_NAME: "Example", + GIT_COMMITTER_EMAIL: "example@example.invalid", + GIT_CONFIG_NOSYSTEM: "1", + // Git for Windows cannot use /dev/null as a global config file. + GIT_CONFIG_GLOBAL: path.join(directory, "absent-global-gitconfig"), + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "credential.helper", + GIT_CONFIG_VALUE_0: "", + GIT_TERMINAL_PROMPT: "0", + }; + await git("git", ["add", "hello.txt"], { cwd: directory, env }); + await git("git", ["commit", "-m", "fixture"], { cwd: directory, env }); + await git( + "git", + [ + "-c", + `http.extraHeader=Authorization: Bearer ${created.token}`, + "push", + created.remote, + "main", + ], + { cwd: directory, env } + ); + expect(await (await fetch(first.url + "/file")).text()).toBe( + "hello artifacts\n" + ); + expect( + (await request<{ repos: unknown[] }>(first.url, "/list")).repos + ).toHaveLength(1); + await seed({ "index.js": worker.replace("initial", "reloaded") }); + await vi.waitFor( + async () => { + expect(await (await fetch(first.url + "/version")).text()).toBe( + "reloaded" + ); + }, + { timeout: 15_000 } + ); + expect(await (await fetch(first.url + "/file")).text()).toBe( + "hello artifacts\n" + ); + } finally { + await stopRunningDev(); + await first.stopped; + } + await expect( + fetch(gitUrl + "/info/refs?service=git-upload-pack") + ).rejects.toThrow(); + const second = await startLocalDev(remote); + try { + expect( + (await request<{ name: string }>(second.url, "/info")).name + ).toBe("demo"); + expect(await (await fetch(second.url + "/file")).text()).toBe( + "hello artifacts\n" + ); + } finally { + await stopRunningDev(); + await second.stopped; + } + } + ); +}); diff --git a/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts b/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts index 129d755caca..f8e2e1269f0 100644 --- a/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts +++ b/packages/wrangler/src/__tests__/dev/remote-bindings.test.ts @@ -523,12 +523,14 @@ describe("dev with remote bindings", { sequential: true, retry: 2 }, () => { { binding: "MY_ARTIFACTS", namespace: "default", + remote: true, }, ], }, expectedProxyWorkerBindings: { MY_ARTIFACTS: { namespace: "default", + remote: true, type: "artifacts", }, }, @@ -671,6 +673,36 @@ describe("dev with remote bindings", { sequential: true, retry: 2 }, () => { await wranglerStopped; }); + it("keeps Artifacts local when remote is omitted", async ({ expect }) => { + await seed({ + "wrangler.jsonc": JSON.stringify({ + name: "worker", + main: "index.js", + compatibility_date: "2025-01-01", + artifacts: [{ binding: "MY_ARTIFACTS", namespace: "default" }], + }), + "index.js": `export default { fetch() { return new Response("hello") } }`, + }); + const wranglerStopped = runWrangler("dev --port=0 --inspector-port=0"); + await vi.waitFor(() => expect(std.out).toMatch(/Ready/), { + timeout: devReadyTimeout, + }); + expect(startRemoteProxySessionCallCount).toBe(0); + expect(proxyWorkerBindings).toBeUndefined(); + expect(workerOptions).toEqual([ + expect.objectContaining({ + artifacts: { + MY_ARTIFACTS: { + namespace: "default", + remoteProxyConnectionString: undefined, + }, + }, + }), + ]); + await stopWrangler(); + await wranglerStopped; + }); + it("should not recreate the remote proxy session on reload when auth has not changed", async () => { const { maybeStartOrUpdateRemoteProxySession } = await import("../../api/remoteBindings"); diff --git a/packages/wrangler/src/__tests__/print-bindings.test.ts b/packages/wrangler/src/__tests__/print-bindings.test.ts index 8723079ca5d..68ab3548f5e 100644 --- a/packages/wrangler/src/__tests__/print-bindings.test.ts +++ b/packages/wrangler/src/__tests__/print-bindings.test.ts @@ -176,16 +176,27 @@ describe("printBindings -- Flagship bindings", () => { }); describe("printBindings -- Artifacts bindings", () => { - it("shows Artifacts bindings", ({ expect }) => { - const output = callPrintBindings({ - MY_ARTIFACTS: { - type: "artifacts", - namespace: "default", - }, - }); - - expect(output).toContain("MY_ARTIFACTS"); - expect(output).toContain("Artifacts"); - expect(output).toContain("default"); - }); + it.for([ + { label: "default", remote: undefined, mode: "local" }, + { label: "explicit local", remote: false, mode: "local" }, + { label: "explicit remote", remote: true, mode: "remote" }, + ])( + "shows $label Artifacts bindings as $mode", + ({ remote, mode }, { expect }) => { + const output = callPrintBindings( + { + MY_ARTIFACTS: { + type: "artifacts", + namespace: "default", + ...(remote === undefined ? {} : { remote }), + }, + }, + true + ); + + expect(output).toMatch( + new RegExp(`env\\.MY_ARTIFACTS \\(default\\)\\s+Artifacts\\s+${mode}`) + ); + } + ); }); diff --git a/packages/wrangler/src/dev/miniflare/index.ts b/packages/wrangler/src/dev/miniflare/index.ts index da6c613fc7d..b0ffa13e563 100644 --- a/packages/wrangler/src/dev/miniflare/index.ts +++ b/packages/wrangler/src/dev/miniflare/index.ts @@ -960,7 +960,9 @@ export function buildMiniflareBindingOptions( binding.binding, { namespace: binding.namespace, - remoteProxyConnectionString, + remoteProxyConnectionString: binding.remote + ? remoteProxyConnectionString + : undefined, }, ]) ), diff --git a/turbo.json b/turbo.json index 13cf7874f60..f3b309eff3e 100644 --- a/turbo.json +++ b/turbo.json @@ -13,6 +13,8 @@ "DOCKER_HOST", "LOCAL_TESTS_WITHOUT_DOCKER", "MINIFLARE_CACHE_DIR", + "pnpm_config_pm_on_fail", + "pnpm_config_verify_deps_before_run", "NODE_EXTRA_CA_CERTS", "PWD", "TEST_REPORT_PATH",