[miniflare] Add opt-in local Artifacts simulator [2/3]

.changeset/artifacts-opt-in-simulator.md

@@ -0,0 +1,5 @@
+---
+"miniflare": minor
+---
+
+Add an opt-in local Artifacts binding simulator to Miniflare. Set `dev.remote: false` on an Artifacts binding to use persistent local repositories and authenticated Git smart HTTP backed by host Git 2.32 or newer. Omitted `dev.remote` retains the existing remote binding behavior.

packages/miniflare/scripts/build.mjs

@@ -446,7 +446,13 @@ async function buildPackage() {
 		logLevel: watch ? "info" : "warning",
 		outdir: outPath,
 		outbase: pkgRoot,
-		entryPoints: [indexPath, ...fixtureBuilds],
+		entryPoints: [
+			indexPath,
+			// A separate thread keeps native Git available during synchronous Node
+			// binding-proxy calls, which block Miniflare's main event loop.
+			path.join(pkgRoot, "src/plugins/artifacts/git-sidecar.ts"),
+			...fixtureBuilds,
+		],
 	};
 
 	if (watch) {

packages/miniflare/src/config/v4-convert.ts

@@ -796,7 +796,11 @@ function addProductBindings(
 		env[name] = {
 			type: "artifacts",
 			namespace: binding.namespace,
-			dev: { remote: isRemote(binding.remoteProxyConnectionString) },
+			// At this stage, only explicit V5 dev.remote: false selects local.
+			// Do not reinterpret an omitted legacy setting as that opt-in.
+			...(binding.remoteProxyConnectionString === undefined
+				? {}
+				: { dev: { remote: isRemote(binding.remoteProxyConnectionString) } }),
 		};
 	}
 	for (const name of Object.keys(worker.workerLoaders ?? {})) {

packages/miniflare/src/index.ts

@@ -77,6 +77,7 @@ import {
 	WORKER_BINDING_SERVICE_LOOPBACK,
 	WORKFLOWS_PLUGIN_NAME,
 } from "./plugins";
+import { ArtifactsController } from "./plugins/artifacts/controller";
 import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants";
 import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version";
 import { closeBrowserProcess } from "./plugins/browser-rendering/process";
@@ -888,6 +889,7 @@ export class Miniflare {
 	#maybeInspectorProxyController?: InspectorProxyController;
 	#previousRuntimeInspectorPort?: number;
 
+	#artifactsController = new ArtifactsController(() => this.#log);
 	#containerPrivilegesCache = new ContainerPrivilegesCache();
 	#hyperdriveProxyController: HyperdriveProxyController =
 		new HyperdriveProxyController();
@@ -2326,6 +2328,7 @@ export class Miniflare {
 				queueConsumers,
 				containerPrivilegesCache: this.#containerPrivilegesCache,
 				hyperdriveProxyController: this.#hyperdriveProxyController,
+				artifactsController: this.#artifactsController,
 			};
 			for (const [key, plugin] of this.#mergedPluginEntries) {
 				const pluginServicesExtensions = await plugin.getServices({
@@ -2341,6 +2344,11 @@ export class Miniflare {
 					} else {
 						pluginServices = pluginServicesExtensions.services;
 						extensions.push(...pluginServicesExtensions.extensions);
+						for (const socket of pluginServicesExtensions.sockets ?? []) {
+							if (!sockets.some((existing) => existing.name === socket.name)) {
+								sockets.push(socket);
+							}
+						}
 					}
 
 					for (const service of pluginServices) {
@@ -2672,11 +2680,13 @@ export class Miniflare {
 
 	async #assembleAndUpdateConfig(reusePorts = false) {
 		this.#hyperdriveProxyController.beginUpdate();
+		this.#artifactsController.beginUpdate();
 		try {
 			await this.#assembleAndUpdateConfigInternal(reusePorts);
 		} finally {
-			// If assembly or runtime startup failed, keep the old listeners and
-			// close only the new candidates. A successful update already committed.
+			// A failed update retains listeners used by the previous runtime and
+			// closes only candidates started during this attempt.
+			await this.#artifactsController.abortUpdate();
 			this.#hyperdriveProxyController.abortUpdate();
 		}
 	}
@@ -2800,6 +2810,7 @@ export class Miniflare {
 			}
 		}
 		this.#hyperdriveProxyController.commitUpdate(activeExternalAddresses);
+		await this.#artifactsController.commitUpdate();
 		// Note: `updateConfig()` doesn't resolve until ports for all required
 		// sockets have been recorded. At this point, `maybeSocketPorts` contains
 		// all of `requiredSockets` as keys.
@@ -3915,6 +3926,14 @@ export class Miniflare {
 		}
 
 		const runtimeCleanupOutcome = await runtimeDisposeOutcome;
+		try {
+			await this.#artifactsController.dispose();
+		} catch (error) {
+			if (!independentCleanupFailed) {
+				independentCleanupFailed = true;
+				independentCleanupError = error;
+			}
+		}
 		this.#devRegistry.unregisterWorkers();
 		try {
 			await Promise.all(

packages/miniflare/src/plugins/artifacts/README.md

@@ -0,0 +1,168 @@
+# Artifacts in Miniflare
+
+The Artifacts plugin implements the local Artifacts binding in workerd. This
+first version requires Git 2.32 or newer installed globally on the host and
+available on PATH. Miniflare checks the Git version at local startup and gives
+installation or upgrade instructions if needed. A Node worker thread runs the
+native Git backend. Set `dev.remote` to `false` explicitly to select local
+storage; an omitted setting retains remote binding behavior in this stage.
+We plan to open-source a JavaScript Git engine in the future; it is not
+included in this version.
+
+## Local startup
+
+```mermaid
+sequenceDiagram
+    participant Dev as Developer tool
+    participant MF as Miniflare
+    participant Plugin as Artifacts plugin
+    participant Controller as ArtifactsController
+    participant Sidecar as Node Git sidecar
+    participant Workerd as workerd
+
+    Dev->>MF: Configure Artifacts binding
+    MF->>Plugin: getBindings()
+    Plugin->>Plugin: Select local and validate namespace
+    Plugin-->>MF: Bind to local namespace entrypoint
+    MF->>Plugin: getServices()
+    Plugin->>Plugin: Verify host Git is available on PATH
+    Plugin->>Controller: get(Git storage path)
+    Controller->>Sidecar: Start worker thread and private HTTP listener
+    Sidecar-->>Controller: Address and secret
+    Controller-->>Plugin: Sidecar endpoint and Git socket port
+    Plugin-->>MF: Binding Worker, DO storage, backend service, Git socket
+    MF->>Workerd: Start configured services and socket
+```
+
+`getBindings()` routes each local binding to a namespace-scoped service.
+`getServices()` creates the binding Worker, namespace Durable Object, persistent
+metadata disk, internal Git backend service, and Git socket. The controller
+reuses a sidecar per Git storage path and closes it when Miniflare is disposed.
+The socket used by Git clients and the sidecar's private HTTP listener are
+**different endpoints**. If the sidecar exits after startup, the controller
+forgets its endpoint and warns; an explicit Miniflare configuration reload
+reconnects, but existing workerd connections do not recover automatically.
+Restart the dev server or reload its configuration if Git operations stop
+working after a sidecar exit.
+
+## Local binding RPC
+
+For example, `env.REPOS.create("demo")`:
+
+```mermaid
+sequenceDiagram
+    participant App as User Worker
+    participant Entry as Binding Worker
+    participant DO as Namespace DO
+    participant Backend as Backend service
+    participant Sidecar as Node Git sidecar
+    participant Disk as Local disk
+
+    App->>Entry: env.REPOS.create("demo")
+    Entry->>DO: RPC create("demo")
+    DO->>DO: Validate and serialize mutation
+    DO->>Backend: POST internal create request
+    Backend->>Sidecar: Inject private backend header
+    Sidecar->>Disk: git init --bare
+    Sidecar-->>DO: Git state (via backend service)
+    DO->>Disk: Persist metadata and token hash
+    DO-->>Entry: Repository info and initial token
+    Entry-->>App: Create result
+```
+
+The Durable Object stores repository metadata and token hashes on local disk;
+the sidecar stores the bare Git repositories separately. `git-sidecar.ts`
+handles HTTP routing and repository locks, while `git-client.ts` owns
+repository-scoped Git commands and result parsing. Read operations such as
+`readFile()` also use the binding Worker and native Git backend. Node callers
+reach the same binding through Miniflare's Node binding proxy. Namespace and
+repository names are hashed into short, namespace-scoped on-disk paths; this
+leaves room for Git packfiles and workerd SQLite files on platforms with path
+length limits. Creation, fork, and import use private staging directories;
+interrupted staging is removed on the next backend startup. If a process dies
+while creating a repository, a leftover creation lock blocks reuse until you
+confirm no other dev server uses the directory and remove the lock named in
+the error. Import URL credentials are passed to Git as an ephemeral HTTPS
+authorization header,
+not recorded in a repository's Git config. Host Git runs with an isolated HOME
+so it cannot read the developer's `.netrc`; SSH/scp-style imports and Git remote
+helpers are rejected rather than accessing host SSH keys or executing helpers.
+The local Worker binding accepts only explicit HTTPS imports; filesystem paths
+are available only to the internal Git client. Repository locks are process-local:
+do not run two dev servers against the same Artifacts persistence directory at
+the same time. Older draft storage layouts are not migrated automatically:
+startup rejects a detected old layout rather than silently creating empty
+repositories. Back up any local repositories before resetting the Artifacts
+persistence directory. Do not delete the persistence directory without first
+checking whether it holds work you need.
+
+## Git clone and push
+
+```mermaid
+sequenceDiagram
+    participant Git as Git client
+    participant Socket as Git socket
+    participant Entry as Binding Worker
+    participant DO as Namespace DO
+    participant Backend as Backend service
+    participant Sidecar as Node Git sidecar
+    participant Native as Native Git
+
+    Git->>Socket: Git smart HTTP at /git/:namespace/:repo.git
+    Socket->>Entry: Route request
+    Entry->>DO: fetch(request)
+    DO->>DO: Check endpoint, repository, token, and scope
+    DO->>Backend: Forward request without Authorization header
+    Backend->>Sidecar: Inject private backend header
+    Sidecar->>Native: Stream via git http-backend
+    Native-->>Sidecar: Discovery or pack response
+    Sidecar-->>DO: Git response (via backend service)
+    opt Successful receive-pack push
+        DO->>Backend: Read updated refs
+        Backend->>Sidecar: Internal refs request
+        Sidecar-->>DO: Refs (via backend service)
+        DO->>DO: Persist updated push metadata
+    end
+    DO-->>Entry: Git response
+    Entry-->>Socket: Git response
+    Socket-->>Git: Git response
+```
+
+The binding Worker checks the supported Git endpoints and authenticates
+repository tokens before forwarding. The internal service injects the sidecar
+secret; the client's `Authorization` header is removed before that hop. For a
+successful push, the Durable Object refreshes repository metadata before
+returning the response. The sidecar uses native `git http-backend` for Git smart
+HTTP rather than reproducing Git's pack protocol. All native Git commands
+(including the HTTP backend) override repository-local hook configuration.
+A failed deletion leaves an inaccessible, persisted tombstone, so retrying
+`delete()` or creating the same name can finish native Git cleanup even after a
+process restart.
+
+## Explicit remote binding
+
+When `dev.remote` is `true`, this binding uses the remote-bindings proxy instead
+of creating local Artifacts storage or a local Git sidecar for that namespace.
+
+```mermaid
+sequenceDiagram
+    participant App as User Worker
+    participant MF as Miniflare
+    participant Plugin as Artifacts plugin
+    participant Proxy as Remote-bindings proxy
+    participant CF as Cloudflare Artifacts
+
+    MF->>Plugin: getBindings() with remote: true
+    Plugin-->>MF: Bind to remote proxy service
+    MF->>Proxy: Start remote proxy Worker
+    App->>Proxy: env.REPOS method call
+    Proxy->>CF: Remote binding request
+    CF-->>Proxy: Result
+    Proxy-->>App: Result
+```
+
+A local binding does not automatically synchronize with production Artifacts.
+An explicit repository import may contact its specified HTTPS Git remote from
+the developer's host, including private-network hosts; it does not follow HTTP
+redirects. This is distinct from enabling a remote Artifacts binding. Do not
+pass an untrusted URL to `import()` when the host can access sensitive services.