[miniflare] Add opt-in local Artifacts simulator [2/3]

.changeset/artifacts-opt-in-simulator.md

@@ -0,0 +1,5 @@
+---
+"miniflare": minor
+---
+
+Add an opt-in local Artifacts binding simulator to Miniflare. Set `dev.remote: false` on an Artifacts binding to use persistent local repositories and authenticated Git smart HTTP backed by host Git 2.32 or newer. Omitted `dev.remote` retains the existing remote binding behavior.

packages/miniflare/scripts/build.mjs

@@ -446,7 +446,13 @@ async function buildPackage() {
 		logLevel: watch ? "info" : "warning",
 		outdir: outPath,
 		outbase: pkgRoot,
-		entryPoints: [indexPath, ...fixtureBuilds],
+		entryPoints: [
+			indexPath,
+			// A separate thread keeps native Git available during synchronous Node
+			// binding-proxy calls, which block Miniflare's main event loop.
+			path.join(pkgRoot, "src/plugins/artifacts/git-sidecar.ts"),
+			...fixtureBuilds,
+		],
 	};
 
 	if (watch) {

packages/miniflare/src/index.ts

@@ -77,6 +77,7 @@ import {
 	WORKER_BINDING_SERVICE_LOOPBACK,
 	WORKFLOWS_PLUGIN_NAME,
 } from "./plugins";
+import { ArtifactsController } from "./plugins/artifacts/controller";
 import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants";
 import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version";
 import { closeBrowserProcess } from "./plugins/browser-rendering/process";
@@ -888,6 +889,7 @@ export class Miniflare {
 	#maybeInspectorProxyController?: InspectorProxyController;
 	#previousRuntimeInspectorPort?: number;
 
+	#artifactsController = new ArtifactsController();
 	#containerPrivilegesCache = new ContainerPrivilegesCache();
 	#hyperdriveProxyController: HyperdriveProxyController =
 		new HyperdriveProxyController();
@@ -2326,6 +2328,7 @@ export class Miniflare {
 				queueConsumers,
 				containerPrivilegesCache: this.#containerPrivilegesCache,
 				hyperdriveProxyController: this.#hyperdriveProxyController,
+				artifactsController: this.#artifactsController,
 			};
 			for (const [key, plugin] of this.#mergedPluginEntries) {
 				const pluginServicesExtensions = await plugin.getServices({
@@ -2341,6 +2344,11 @@ export class Miniflare {
 					} else {
 						pluginServices = pluginServicesExtensions.services;
 						extensions.push(...pluginServicesExtensions.extensions);
+						for (const socket of pluginServicesExtensions.sockets ?? []) {
+							if (!sockets.some((existing) => existing.name === socket.name)) {
+								sockets.push(socket);
+							}
+						}
 					}
 
 					for (const service of pluginServices) {
@@ -2672,11 +2680,13 @@ export class Miniflare {
 
 	async #assembleAndUpdateConfig(reusePorts = false) {
 		this.#hyperdriveProxyController.beginUpdate();
+		this.#artifactsController.beginUpdate();
 		try {
 			await this.#assembleAndUpdateConfigInternal(reusePorts);
 		} finally {
-			// If assembly or runtime startup failed, keep the old listeners and
-			// close only the new candidates. A successful update already committed.
+			// A failed update retains listeners used by the previous runtime and
+			// closes only candidates started during this attempt.
+			await this.#artifactsController.abortUpdate();
 			this.#hyperdriveProxyController.abortUpdate();
 		}
 	}
@@ -2800,6 +2810,7 @@ export class Miniflare {
 			}
 		}
 		this.#hyperdriveProxyController.commitUpdate(activeExternalAddresses);
+		await this.#artifactsController.commitUpdate();
 		// Note: `updateConfig()` doesn't resolve until ports for all required
 		// sockets have been recorded. At this point, `maybeSocketPorts` contains
 		// all of `requiredSockets` as keys.
@@ -3915,6 +3926,14 @@ export class Miniflare {
 		}
 
 		const runtimeCleanupOutcome = await runtimeDisposeOutcome;
+		try {
+			await this.#artifactsController.dispose();
+		} catch (error) {
+			if (!independentCleanupFailed) {
+				independentCleanupFailed = true;
+				independentCleanupError = error;
+			}
+		}
 		this.#devRegistry.unregisterWorkers();
 		try {
 			await Promise.all(

packages/miniflare/src/plugins/artifacts/README.md

@@ -0,0 +1,146 @@
+# Artifacts in Miniflare
+
+The Artifacts plugin implements the local Artifacts binding in workerd. This
+first version requires Git 2.32 or newer installed globally on the host and
+available on PATH. Miniflare checks the Git version at local startup and gives
+installation or upgrade instructions if needed. A Node worker thread runs the
+native Git backend. Set `dev.remote` to `false` explicitly to select local
+storage; an omitted setting retains remote binding behavior in this stage.
+We plan to open-source a JavaScript Git engine in the future; it is not
+included in this version.
+
+## Local startup
+
+```mermaid
+sequenceDiagram
+    participant Dev as Developer tool
+    participant MF as Miniflare
+    participant Plugin as Artifacts plugin
+    participant Controller as ArtifactsController
+    participant Sidecar as Node Git sidecar
+    participant Workerd as workerd
+
+    Dev->>MF: Configure Artifacts binding
+    MF->>Plugin: getBindings()
+    Plugin->>Plugin: Select local and validate namespace
+    Plugin-->>MF: Bind to local namespace entrypoint
+    MF->>Plugin: getServices()
+    Plugin->>Plugin: Verify host Git is available on PATH
+    Plugin->>Controller: get(Git storage path)
+    Controller->>Sidecar: Start worker thread and private HTTP listener
+    Sidecar-->>Controller: Address and secret
+    Controller-->>Plugin: Sidecar endpoint and Git socket port
+    Plugin-->>MF: Binding Worker, DO storage, backend service, Git socket
+    MF->>Workerd: Start configured services and socket
+```
+
+`getBindings()` routes each local binding to a namespace-scoped service.
+`getServices()` creates the binding Worker, namespace Durable Object, persistent
+metadata disk, internal Git backend service, and Git socket. The controller
+reuses a sidecar per Git storage path and closes it when Miniflare is disposed.
+The socket used by Git clients and the sidecar's private HTTP listener are
+**different endpoints**.
+
+## Local binding RPC
+
+For example, `env.REPOS.create("demo")`:
+
+```mermaid
+sequenceDiagram
+    participant App as User Worker
+    participant Entry as Binding Worker
+    participant DO as Namespace DO
+    participant Backend as Backend service
+    participant Sidecar as Node Git sidecar
+    participant Disk as Local disk
+
+    App->>Entry: env.REPOS.create("demo")
+    Entry->>DO: RPC create("demo")
+    DO->>DO: Validate and serialize mutation
+    DO->>Backend: POST internal create request
+    Backend->>Sidecar: Inject private backend header
+    Sidecar->>Disk: git init --bare
+    Sidecar-->>DO: Git state (via backend service)
+    DO->>Disk: Persist metadata and token hash
+    DO-->>Entry: Repository info and initial token
+    Entry-->>App: Create result
+```
+
+The Durable Object stores repository metadata and token hashes on local disk;
+the sidecar stores the bare Git repositories separately. `git-sidecar.ts`
+handles HTTP routing and repository locks, while `git-client.ts` owns
+repository-scoped Git commands and result parsing. Read operations such as
+`readFile()` also use the binding Worker and native Git backend. Node callers
+reach the same binding through Miniflare's Node binding proxy. Namespace and
+repository names are hashed into short, namespace-scoped on-disk paths; this
+leaves room for Git packfiles and workerd SQLite files on platforms with path
+length limits. Older draft storage layouts are not migrated automatically:
+startup rejects a detected old layout rather than silently creating empty
+repositories. Back up any local repositories before resetting the Artifacts
+persistence directory. Do not delete the persistence directory without first
+checking whether it holds work you need.
+
+## Git clone and push
+
+```mermaid
+sequenceDiagram
+    participant Git as Git client
+    participant Socket as Git socket
+    participant Entry as Binding Worker
+    participant DO as Namespace DO
+    participant Backend as Backend service
+    participant Sidecar as Node Git sidecar
+    participant Native as Native Git
+
+    Git->>Socket: Git smart HTTP at /git/:namespace/:repo.git
+    Socket->>Entry: Route request
+    Entry->>DO: fetch(request)
+    DO->>DO: Check endpoint, repository, token, and scope
+    DO->>Backend: Forward request without Authorization header
+    Backend->>Sidecar: Inject private backend header
+    Sidecar->>Native: Stream via git http-backend
+    Native-->>Sidecar: Discovery or pack response
+    Sidecar-->>DO: Git response (via backend service)
+    opt Successful receive-pack push
+        DO->>Backend: Read updated refs
+        Backend->>Sidecar: Internal refs request
+        Sidecar-->>DO: Refs (via backend service)
+        DO->>DO: Persist updated push metadata
+    end
+    DO-->>Entry: Git response
+    Entry-->>Socket: Git response
+    Socket-->>Git: Git response
+```
+
+The binding Worker checks the supported Git endpoints and authenticates
+repository tokens before forwarding. The internal service injects the sidecar
+secret; the client's `Authorization` header is removed before that hop. For a
+successful push, the Durable Object refreshes repository metadata before
+returning the response. The sidecar uses native `git http-backend` for Git smart
+HTTP rather than reproducing Git's pack protocol.
+
+## Explicit remote binding
+
+When `dev.remote` is `true`, this binding uses the remote-bindings proxy instead
+of creating local Artifacts storage or a local Git sidecar for that namespace.
+
+```mermaid
+sequenceDiagram
+    participant App as User Worker
+    participant MF as Miniflare
+    participant Plugin as Artifacts plugin
+    participant Proxy as Remote-bindings proxy
+    participant CF as Cloudflare Artifacts
+
+    MF->>Plugin: getBindings() with remote: true
+    Plugin-->>MF: Bind to remote proxy service
+    MF->>Proxy: Start remote proxy Worker
+    App->>Proxy: env.REPOS method call
+    Proxy->>CF: Remote binding request
+    CF-->>Proxy: Result
+    Proxy-->>App: Result
+```
+
+A local binding does not automatically synchronize with production Artifacts.
+An explicit repository import may contact its specified HTTPS Git remote; that
+is distinct from enabling a remote Artifacts binding.

packages/miniflare/src/plugins/artifacts/controller.ts

@@ -0,0 +1,150 @@
+import { once } from "node:events";
+import path from "node:path";
+import { Worker } from "node:worker_threads";
+import getPort from "get-port";
+import type { GitSidecar } from "./git-sidecar";
+
+export interface LocalArtifactsBackend {
+	sidecar: GitSidecar;
+	port: number;
+}
+
+// Node binding proxies make synchronous calls. The Git backend must run on
+// another thread, otherwise those calls prevent it from answering workerd.
+export class ArtifactsController {
+	#backends = new Map<string, Promise<LocalArtifactsBackend>>();
+	// Keep the old roots alive until workerd accepts the replacement config.
+	#activeRoots = new Set<string>();
+	#pendingRoots?: Set<string>;
+
+	beginUpdate(): void {
+		if (this.#pendingRoots) {
+			throw new Error("Artifacts runtime update already in progress");
+		}
+		this.#pendingRoots = new Set();
+	}
+
+	async commitUpdate(): Promise<void> {
+		const roots = this.#pendingRoots;
+		if (!roots) {
+			throw new Error("No Artifacts runtime update to commit");
+		}
+		this.#pendingRoots = undefined;
+		this.#activeRoots = roots;
+		await this.#closeExcept(roots);
+	}
+
+	async abortUpdate(): Promise<void> {
+		if (!this.#pendingRoots) {
+			return;
+		}
+		this.#pendingRoots = undefined;
+		await this.#closeExcept(this.#activeRoots);
+	}
+
+	get(root: string): Promise<LocalArtifactsBackend> {
+		if (!this.#pendingRoots) {
+			throw new Error("Artifacts backend requested outside runtime update");
+		}
+		this.#pendingRoots.add(root);
+		let backend = this.#backends.get(root);
+		if (!backend) {
+			backend = this.#start(root);
+			this.#backends.set(root, backend);
+			void backend.catch(() => {
+				if (this.#backends.get(root) === backend) {
+					this.#backends.delete(root);
+				}
+			});
+		}
+		return backend;
+	}
+
+	async #start(root: string): Promise<LocalArtifactsBackend> {
+		const port = await getPort({ host: "127.0.0.1" });
+		// __dirname is dist/src in the bundled Miniflare entry point.
+		const worker = new Worker(
+			path.join(__dirname, "plugins/artifacts/git-sidecar.js"),
+			{ workerData: { root } }
+		);
+		try {
+			const endpoint = await waitForSidecarReady(worker);
+			worker.on("error", (error) => {
+				process.emitWarning(
+					`Local Artifacts Git backend failed: ${error.message}`
+				);
+			});
+			return {
+				port,
+				sidecar: {
+					...endpoint,
+					async close() {
+						if (worker.threadId === -1) {
+							return;
+						}
+						const exited = once(worker, "exit");
+						worker.postMessage("close");
+						await exited;
+					},
+				},
+			};
+		} catch (error) {
+			await worker.terminate();
+			throw error;
+		}
+	}
+
+	async #closeExcept(roots: Set<string>): Promise<void> {
+		const retired = [...this.#backends].filter(([root]) => !roots.has(root));
+		for (const [root] of retired) {
+			this.#backends.delete(root);
+		}
+		const results = await Promise.allSettled(
+			retired.map(([, backend]) =>
+				backend.then(
+					({ sidecar }) => sidecar.close(),
+					() => undefined
+				)
+			)
+		);
+		for (const result of results) {
+			if (result.status === "rejected") {
+				process.emitWarning("Unable to stop a retired Artifacts Git backend");
+			}
+		}
+	}
+
+	async dispose(): Promise<void> {
+		this.#pendingRoots = undefined;
+		this.#activeRoots.clear();
+		await this.#closeExcept(this.#activeRoots);
+	}
+}
+
+function waitForSidecarReady(
+	worker: Worker
+): Promise<Pick<GitSidecar, "address" | "secret">> {
+	return new Promise((resolve, reject) => {
+		function cleanup() {
+			worker.off("message", ready);
+			worker.off("error", failed);
+			worker.off("exit", exited);
+		}
+		function ready(value: Pick<GitSidecar, "address" | "secret">) {
+			cleanup();
+			resolve(value);
+		}
+		function failed(error: Error) {
+			cleanup();
+			reject(error);
+		}
+		function exited(code: number) {
+			failed(
+				new Error(`Local Artifacts Git backend exited during startup (${code})`)
+			);
+		}
+		worker.on("message", ready);
+		worker.on("error", failed);
+		worker.on("exit", exited);
+	});
+}