feat(models): download model weights before use, with on-screen progress

CLAUDE.md

@@ -90,6 +90,39 @@ once the model/provider is registered, no version bump required
 (`MODEL_ERROR_REASON_CODES` in `ExtractionDeadLetterSchema.ts`). Every other reason
 code stays version-gated (fix the extractor, bump the version, then retry).
 
+### Download-before-use harness
+
+Local model weights must be on disk before generation, and providers differ on
+when that happens: cloud models have nothing to download; HuggingFace ONNX
+auto-fetches on first generation; but node-llama-cpp (GGUF) loads its
+`model_path` directly and never fetches at generation. `ensureModelDownloaded`
+(`src/config/ensureModelDownloaded.ts`) is the single seam that normalizes this.
+It runs `ModelDownloadTask` for the local providers (no-op for cloud, memoized
+per model id so a per-section sweep pays the download once) and skips a bare-path
+GGUF (no `model_url` — the file is assumed on disk).
+
+It takes the running task's `IExecuteContext`: passing the **real** one (not a
+throwaway stub) is what surfaces download progress — the download run-fn's `phase`
+events are forwarded to `context.updateProgress`, which the `@workglow/cli`
+progress UI (`withCli`) renders, so a multi-GB GGUF/ONNX fetch shows a live
+percentage instead of a silent hang (and `context.signal` aborts it on Ctrl-C).
+`prefetchModel(model, context)` is the best-effort wrapper the CLI-task boundaries
+call: the AI form processors (`processFormS1` / `processForm424` /
+`processMergerProxy` / `processRedemption8K` / `processLoi8K`, via a `context`
+threaded through `storageArgs`) prefetch once after resolving their model, and the
+eval loops prefetch before their timed sections (so download time isn't charged to
+a model's measured latency). `runStructured` keeps a context-less
+`ensureModelDownloaded` call as a per-section correctness safety-net — it downloads
+silently if a model was never prefetched (e.g. a sub-extractor's distinct model),
+but the progress-bearing fetch lives at the task boundary.
+
+To make GGUF weights fetchable rather than pre-staged, a `gguf:` id may be a
+**remote URI** — a node-llama-cpp HuggingFace URI (`gguf:hf:org/repo:Q4_K_M`) or
+an `https://` URL — which `secModelRecord` turns into a `model_url` (download
+source) plus a local `model_path` / `models_dir` under the GGUF models dir. A
+plain `gguf:` path (`gguf:Model-Q4.gguf`, `gguf:/abs/Model.gguf`) stays a
+load-directly local file, unchanged.
+
 ### AI SPAC content classifier (SIC-miscoded SPACs)
 
 Deterministic SPAC classification keys off the SGML-header SIC (`6770` →
@@ -245,18 +278,22 @@ sonnet takes ~20s each, and a local HFT model minutes.
 
 PrismML **Bonsai 27B** (Qwen3.6-based, Apache-2.0) runs through the existing
 node-llama-cpp path — there is **no special model id or route**; it is just a
-`gguf:` model like any other local GGUF. Download a quant from HuggingFace
-(`prism-ml/Ternary-Bonsai-27B-gguf`, or the 1-bit `prism-ml/Bonsai-27B-gguf`)
-into the GGUF models dir (`$SEC_GGUF_DIR`, else `$SEC_RAW_DATA_FOLDER/gguf`, else
-`./models`) and pass it as a `gguf:` candidate:
+`gguf:` model like any other local GGUF. Point a `gguf:` candidate at a HuggingFace
+quant URI and the download-before-use harness fetches it into the GGUF models dir
+(`$SEC_GGUF_DIR`, else `$SEC_RAW_DATA_FOLDER/gguf`, else `./models`) on first use;
+or pre-stage the file yourself and pass its local path:
 
 ```bash
-# one-time: fetch a quant (e.g. the ternary Q2_0 ~ a few GB) into the models dir
+# Remote URI — the harness downloads it before the run (into the GGUF models dir)
+sec eval s1 --reference claude-sonnet-5 \
+  --models "gguf:hf:prism-ml/Ternary-Bonsai-27B-gguf:Q2_0" \
+  --extractors "management,beneficial-ownership,related-party"
+
+# Or pre-stage the quant and pass its local filename / absolute path instead
 huggingface-cli download prism-ml/Ternary-Bonsai-27B-gguf \
   Ternary-Bonsai-27B-Q2_0.gguf --local-dir "${SEC_GGUF_DIR:-./models}"
-
-# score Bonsai against the opus oracle on the committed real S-1 sections
-sec eval s1 --models "gguf:Ternary-Bonsai-27B-Q2_0.gguf" \
+sec eval s1 --reference claude-sonnet-5 \
+  --models "gguf:Ternary-Bonsai-27B-Q2_0.gguf" \
   --extractors "management,beneficial-ownership,related-party"
 # (an absolute path also works: --models "gguf:/abs/path/Ternary-Bonsai-27B-Q2_0.gguf")
 ```

src/config/ensureModelDownloaded.test.ts

@@ -0,0 +1,183 @@
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import type { IExecuteContext, ModelConfig } from "workglow";
+import {
+  AiProviderRegistry,
+  getAiProviderRegistry,
+  setAiProviderRegistry,
+} from "workglow";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+  ensureModelDownloaded,
+  resetEnsuredModelsForTesting,
+} from "./ensureModelDownloaded";
+
+/** A throwaway execute context — these tests never reach a real provider. */
+const ctx = (): IExecuteContext =>
+  ({
+    signal: new AbortController().signal,
+    updateProgress: async () => {},
+    own: <T>(v: T): T => v,
+    registry: { has: () => false, get: () => { throw new Error("x"); } },
+    resourceScope: { register: () => {}, dispose: async () => {} },
+  }) as unknown as IExecuteContext;
+
+/**
+ * No AI providers are registered in this suite, so any model that actually
+ * dispatches `ModelDownloadTask` rejects fast with "No run function found …
+ * model.download". We use that as the observable signal that a download was
+ * *attempted*; a clean resolve is the signal that it was *skipped* (a no-op).
+ */
+const cloud = (id: string, provider: string): ModelConfig =>
+  ({ model_id: id, provider, provider_config: { model_name: id } }) as unknown as ModelConfig;
+
+const llamaCpp = (id: string, provider_config: Record<string, unknown>): ModelConfig =>
+  ({ model_id: id, provider: "LOCAL_LLAMACPP", provider_config }) as unknown as ModelConfig;
+
+const hft = (id: string): ModelConfig =>
+  ({
+    model_id: id,
+    provider: "HF_TRANSFORMERS_ONNX",
+    provider_config: { model_path: id },
+  }) as unknown as ModelConfig;
+
+describe("ensureModelDownloaded", () => {
+  beforeEach(() => {
+    resetEnsuredModelsForTesting();
+  });
+
+  it("is a no-op for cloud providers (nothing to download)", async () => {
+    await expect(ensureModelDownloaded(cloud("claude-sonnet-5", "ANTHROPIC"), ctx())).resolves.toBeUndefined();
+    await expect(ensureModelDownloaded(cloud("gpt-5.5", "OPENAI"), ctx())).resolves.toBeUndefined();
+    await expect(
+      ensureModelDownloaded(cloud("gemini-3-flash-preview", "GOOGLE_GEMINI"), ctx())
+    ).resolves.toBeUndefined();
+    await expect(ensureModelDownloaded(cloud("grok-4.5", "XAI"), ctx())).resolves.toBeUndefined();
+  });
+
+  it("skips a bare-path GGUF (no model_url): the file is assumed on disk", async () => {
+    await expect(
+      ensureModelDownloaded(llamaCpp("gguf:/models/x.gguf", { model_path: "/models/x.gguf" }), ctx())
+    ).resolves.toBeUndefined();
+  });
+
+  it("attempts a download for a HuggingFace ONNX model", async () => {
+    await expect(ensureModelDownloaded(hft("onnx-community/x"), ctx())).rejects.toThrow();
+  });
+
+  it("attempts a download for a GGUF model that has a model_url", async () => {
+    await expect(
+      ensureModelDownloaded(
+        llamaCpp("gguf:hf:org/repo:Q4_K_M", {
+          model_path: "/models/repo-Q4_K_M.gguf",
+          model_url: "hf:org/repo:Q4_K_M",
+          models_dir: "/models",
+        }),
+        ctx()
+      )
+    ).rejects.toThrow();
+  });
+
+  it("downloads each model at most once (memoized across calls)", async () => {
+    const id = "gguf:memo.gguf";
+    // First mark the id ready via the bare-path (no-op) branch.
+    await ensureModelDownloaded(llamaCpp(id, { model_path: "/models/memo.gguf" }), ctx());
+    // A later record with the SAME id would normally attempt a download and
+    // reject — but the memo short-circuits it, so it resolves cleanly.
+    await expect(
+      ensureModelDownloaded(
+        llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }),
+        ctx()
+      )
+    ).resolves.toBeUndefined();
+    // Sanity: without the memo, the url-bearing record does attempt (and reject).
+    resetEnsuredModelsForTesting();
+    await expect(
+      ensureModelDownloaded(
+        llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }),
+        ctx()
+      )
+    ).rejects.toThrow();
+  });
+
+  describe("progress + abort forwarding", () => {
+    // Swap in a throwaway provider registry so a fake download run-fn doesn't leak
+    // into sibling suites, mirroring registerModels.test's model-repo swap.
+    let original: AiProviderRegistry;
+    beforeEach(() => {
+      original = getAiProviderRegistry();
+      setAiProviderRegistry(new AiProviderRegistry());
+      resetEnsuredModelsForTesting();
+    });
+    afterEach(() => {
+      setAiProviderRegistry(original);
+    });
+
+    it("forwards the download run-fn's progress to context.updateProgress (and memoizes)", async () => {
+      let runFnCalls = 0;
+      // Real provider run-fns are plain async fns that push events via `emit`
+      // (see LlamaCpp_Download), not generators.
+      getAiProviderRegistry().registerRunFn("HF_TRANSFORMERS_ONNX", {
+        serves: ["model.download"],
+        runFn: async (input: any, _model: any, _signal: any, emit: any) => {
+          runFnCalls += 1;
+          emit({ type: "phase", message: "Downloading model", progress: 42 });
+          emit({ type: "finish", data: { model: input.model } });
+        },
+      } as any);
+
+      const progress: Array<[number | undefined, string | undefined]> = [];
+      const context = {
+        signal: new AbortController().signal,
+        updateProgress: async (p: number | undefined, m?: string) => {
+          progress.push([p, m]);
+        },
+        own: <T>(v: T): T => v,
+        registry: { has: () => false, get: () => { throw new Error("x"); } },
+        resourceScope: { register: () => {}, dispose: async () => {} },
+      } as unknown as IExecuteContext;
+
+      const model = hft("onnx-community/progress");
+      await ensureModelDownloaded(model, context);
+      // The download's phase event reached the running task's progress sink — this
+      // is what the withCli UI renders on screen.
+      expect(runFnCalls).toBe(1);
+      expect(progress).toContainEqual([42, "Downloading model"]);
+
+      // Memoized: a second call does not re-invoke the download run-fn.
+      await ensureModelDownloaded(model, context);
+      expect(runFnCalls).toBe(1);
+    });
+
+    it("memoizes a model identified by `model` (no model_id)", async () => {
+      let runFnCalls = 0;
+      getAiProviderRegistry().registerRunFn("HF_TRANSFORMERS_ONNX", {
+        serves: ["model.download"],
+        runFn: async (input: any, _m: any, _s: any, emit: any) => {
+          runFnCalls += 1;
+          emit({ type: "finish", data: { model: input.model } });
+        },
+      } as any);
+      const context = {
+        signal: new AbortController().signal,
+        updateProgress: async () => {},
+        own: <T>(v: T): T => v,
+        registry: { has: () => false, get: () => { throw new Error("x"); } },
+        resourceScope: { register: () => {}, dispose: async () => {} },
+      } as unknown as IExecuteContext;
+      // No model_id — identity comes from `model` (mirrors resolveModelId's fallback).
+      const model = {
+        model: "onnx-community/no-id",
+        provider: "HF_TRANSFORMERS_ONNX",
+        provider_config: { model_path: "onnx-community/no-id" },
+      } as unknown as ModelConfig;
+      await ensureModelDownloaded(model, context);
+      await ensureModelDownloaded(model, context);
+      expect(runFnCalls).toBe(1);
+    });
+  });
+});

src/config/ensureModelDownloaded.ts

@@ -0,0 +1,131 @@
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import type { IExecuteContext, ModelConfig } from "workglow";
+import { ModelDownloadTask } from "workglow";
+
+/**
+ * Providers whose weights are fetched from a remote source and cached to disk by
+ * a `model.download` run-fn — the local providers. The cloud providers
+ * (`ANTHROPIC`, `OPENAI`, `GOOGLE_GEMINI`, `XAI`) register no such run-fn, so a
+ * `ModelDownloadTask` for them would throw "no run-fn for provider serving
+ * model.download"; downloading is therefore a no-op for anything not listed here.
+ */
+const DOWNLOADABLE_PROVIDERS = new Set<string>(["HF_TRANSFORMERS_ONNX", "LOCAL_LLAMACPP"]);
+const LLAMACPP_PROVIDER = "LOCAL_LLAMACPP";
+
+/**
+ * Model ids downloaded (or confirmed ready) in this process. A multi-section run
+ * or an eval sweep drives the same model many times; the download run-fn is
+ * idempotent but not free (it re-scans/verifies on-disk files and re-emits
+ * progress), so we run it at most once per model.
+ */
+const ensured = new Set<string>();
+
+/** @internal Reset the memo — for tests only. */
+export function resetEnsuredModelsForTesting(): void {
+  ensured.clear();
+}
+
+/**
+ * Stable per-model memo key. Mirrors `resolveModelId` (`model_id ?? model`) — not
+ * every ModelConfig carries `model_id` (some identify via `model`) — and falls
+ * back to the provider_config download/load target so a record with neither still
+ * memoizes on something unique (`model_url` for a remote GGUF, `model_path` for a
+ * local one / HFT repo id). Empty only for a truly anonymous record, which then
+ * downloads every call rather than being wrongly deduped against another.
+ */
+function modelKey(model: ModelConfig): string {
+  const ref = model as {
+    model_id?: unknown;
+    model?: unknown;
+    provider_config?: { model_url?: unknown; model_path?: unknown };
+  };
+  const candidates = [
+    ref.model_id,
+    ref.model,
+    ref.provider_config?.model_url,
+    ref.provider_config?.model_path,
+  ];
+  const found = candidates.find((c): c is string => typeof c === "string" && c.length > 0);
+  return found ?? "";
+}
+
+/**
+ * Ensure a model's weights are present locally before it is used for generation.
+ *
+ * Providers differ in when weights arrive: cloud models have nothing to download
+ * (no-op); HuggingFace ONNX auto-downloads on first generation anyway, so this
+ * merely fetches it ahead of the timed work; and node-llama-cpp (GGUF) loads its
+ * `model_path` directly and does **not** fetch on generation — so a GGUF model
+ * configured with a `model_url` only lands on disk if `ModelDownloadTask` runs
+ * first. This helper is the seam every SEC model consumer calls to make that
+ * download happen uniformly, whatever the env-configured provider.
+ *
+ * A GGUF record with only a local `model_path` (no `model_url`) is assumed already
+ * on disk — the provider loads it directly and node-llama-cpp's downloader has no
+ * URI to fetch — so download is skipped; a missing file surfaces at load time with
+ * the provider's own error.
+ *
+ * `context` is the running task's {@link IExecuteContext}. Passing the real one
+ * (rather than a throwaway stub) is what surfaces download progress: the download
+ * run-fn's `phase` events are forwarded to `context.updateProgress`, which the
+ * CLI progress UI (`withCli`) renders — so a multi-GB GGUF/ONNX fetch shows a live
+ * percentage instead of a silent hang. `context.signal` also aborts a long
+ * download on Ctrl-C. Memoized per model id, so a per-section sweep pays it once.
+ */
+export async function ensureModelDownloaded(
+  model: ModelConfig,
+  context: IExecuteContext
+): Promise<void> {
+  const provider = (model as { provider?: string }).provider;
+  if (!provider || !DOWNLOADABLE_PROVIDERS.has(provider)) return;
+
+  const modelId = modelKey(model);
+  if (modelId && ensured.has(modelId)) return;
+
+  if (provider === LLAMACPP_PROVIDER) {
+    const config = (model as { provider_config?: { model_url?: string } }).provider_config;
+    if (!config?.model_url) {
+      // Bare local GGUF path: nothing to fetch. Mark ready so we don't re-check.
+      if (modelId) ensured.add(modelId);
+      return;
+    }
+  }
+
+  const input = { model };
+  const task = new ModelDownloadTask({ defaults: input } as any);
+  // Drive the download through its `run()` lifecycle. `run` routes the download
+  // run-fn's `phase` events to `config.updateProgress`, which we forward to the
+  // caller's `context.updateProgress` so a multi-GB fetch renders a live
+  // percentage in the CLI task UI; `signal` aborts it on Ctrl-C.
+  await task.run(input as any, {
+    updateProgress: (_t, progress, message) => context.updateProgress(progress, message),
+    signal: context.signal,
+  });
+  if (modelId) ensured.add(modelId);
+}
+
+/**
+ * Best-effort prefetch used at the CLI-task boundary (form processors, eval
+ * sweeps) to surface download progress before the work begins. No-ops when there
+ * is no model or no context (a direct/test caller), and swallows failures — the
+ * downstream generation path re-attempts via {@link ensureModelDownloaded} and
+ * records the failure in its own way (dead-letter or failed eval run), so a
+ * prefetch problem must never abort the run. Whether it downloads with a visible
+ * progress bar is decided entirely by whether a real `context` is threaded in.
+ */
+export async function prefetchModel(
+  model: ModelConfig | null | undefined,
+  context: IExecuteContext | undefined
+): Promise<void> {
+  if (!model || !context) return;
+  try {
+    await ensureModelDownloaded(model, context);
+  } catch {
+    // Downstream generation re-attempts the download and records any failure.
+  }
+}