From 69d78828f5c474e6a7ad1ea0bb994fb52898fdd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 00:59:05 +0000 Subject: [PATCH 1/4] feat(models): download model weights before use across commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local model weights must be present on disk before generation, but the providers differ on when that happens: cloud models have nothing to download, HuggingFace ONNX auto-fetches on first generation, and node-llama-cpp (GGUF) loads its model_path directly and never fetches at generation. Add ensureModelDownloaded as the single seam that normalizes this — it runs ModelDownloadTask for the local providers (no-op for cloud, memoized per model id) and skips a bare-path GGUF whose file is assumed on disk. Wire it into runStructured (the chokepoint every extractor funnels through) so normal runs fetch weights before use, and prefetch in the eval loops before their timed sections so download time isn't charged to a model's measured latency. Make GGUF ids fetchable rather than pre-staged: a gguf: id may now be a remote URI (gguf:hf:org/repo:quant or an https URL), which secModelRecord turns into a model_url download source plus a local model_path/models_dir under the GGUF models dir. Plain gguf: paths stay load-directly local files, unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011HoQKRtpU7mGpK8Dah9ySm --- CLAUDE.md | 41 ++++++-- src/config/ensureModelDownloaded.test.ts | 88 ++++++++++++++++++ src/config/ensureModelDownloaded.ts | 93 +++++++++++++++++++ src/config/registerModels.test.ts | 49 ++++++++++ src/config/registerModels.ts | 75 ++++++++++++--- src/eval/runExtractionEval.ts | 11 +++ src/eval/runOracleEval.ts | 14 +++ src/eval/runUnitTermsEval.ts | 9 ++ .../s1/sectionExtractors.ts | 5 + 9 files changed, 366 insertions(+), 19 deletions(-) create mode 100644 src/config/ensureModelDownloaded.test.ts create mode 100644 src/config/ensureModelDownloaded.ts diff --git a/CLAUDE.md b/CLAUDE.md index c081a0c6..fcc80e25 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,6 +90,27 @@ once the model/provider is registered, no version bump required (`MODEL_ERROR_REASON_CODES` in `ExtractionDeadLetterSchema.ts`). Every other reason code stays version-gated (fix the extractor, bump the version, then retry). +### Download-before-use harness + +Local model weights must be on disk before generation, and providers differ on +when that happens: cloud models have nothing to download; HuggingFace ONNX +auto-fetches on first generation; but node-llama-cpp (GGUF) loads its +`model_path` directly and never fetches at generation. `ensureModelDownloaded` +(`src/config/ensureModelDownloaded.ts`) is the single seam that normalizes this — +every extraction (through `runStructured`) and every eval sweep calls it before +using a model. It runs `ModelDownloadTask` for the local providers (no-op for +cloud, memoized per model id so a per-section sweep pays the download once) and +skips a bare-path GGUF (no `model_url` — the file is assumed on disk). The eval +loops prefetch before their timed sections so download time isn't charged to a +model's measured latency. + +To make GGUF weights fetchable rather than pre-staged, a `gguf:` id may be a +**remote URI** — a node-llama-cpp HuggingFace URI (`gguf:hf:org/repo:Q4_K_M`) or +an `https://` URL — which `secModelRecord` turns into a `model_url` (download +source) plus a local `model_path` / `models_dir` under the GGUF models dir. A +plain `gguf:` path (`gguf:Model-Q4.gguf`, `gguf:/abs/Model.gguf`) stays a +load-directly local file, unchanged. + ### AI SPAC content classifier (SIC-miscoded SPACs) Deterministic SPAC classification keys off the SGML-header SIC (`6770` → @@ -245,18 +266,22 @@ sonnet takes ~20s each, and a local HFT model minutes. PrismML **Bonsai 27B** (Qwen3.6-based, Apache-2.0) runs through the existing node-llama-cpp path — there is **no special model id or route**; it is just a -`gguf:` model like any other local GGUF. Download a quant from HuggingFace -(`prism-ml/Ternary-Bonsai-27B-gguf`, or the 1-bit `prism-ml/Bonsai-27B-gguf`) -into the GGUF models dir (`$SEC_GGUF_DIR`, else `$SEC_RAW_DATA_FOLDER/gguf`, else -`./models`) and pass it as a `gguf:` candidate: +`gguf:` model like any other local GGUF. Point a `gguf:` candidate at a HuggingFace +quant URI and the download-before-use harness fetches it into the GGUF models dir +(`$SEC_GGUF_DIR`, else `$SEC_RAW_DATA_FOLDER/gguf`, else `./models`) on first use; +or pre-stage the file yourself and pass its local path: ```bash -# one-time: fetch a quant (e.g. the ternary Q2_0 ~ a few GB) into the models dir +# Remote URI — the harness downloads it before the run (into the GGUF models dir) +sec eval s1 --reference claude-sonnet-5 \ + --models "gguf:hf:prism-ml/Ternary-Bonsai-27B-gguf:Q2_0" \ + --extractors "management,beneficial-ownership,related-party" + +# Or pre-stage the quant and pass its local filename / absolute path instead huggingface-cli download prism-ml/Ternary-Bonsai-27B-gguf \ Ternary-Bonsai-27B-Q2_0.gguf --local-dir "${SEC_GGUF_DIR:-./models}" - -# score Bonsai against the opus oracle on the committed real S-1 sections -sec eval s1 --models "gguf:Ternary-Bonsai-27B-Q2_0.gguf" \ +sec eval s1 --reference claude-sonnet-5 \ + --models "gguf:Ternary-Bonsai-27B-Q2_0.gguf" \ --extractors "management,beneficial-ownership,related-party" # (an absolute path also works: --models "gguf:/abs/path/Ternary-Bonsai-27B-Q2_0.gguf") ``` diff --git a/src/config/ensureModelDownloaded.test.ts b/src/config/ensureModelDownloaded.test.ts new file mode 100644 index 00000000..e790ac83 --- /dev/null +++ b/src/config/ensureModelDownloaded.test.ts @@ -0,0 +1,88 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { ModelConfig } from "workglow"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + ensureModelDownloaded, + resetEnsuredModelsForTesting, +} from "./ensureModelDownloaded"; + +/** + * No AI providers are registered in this suite, so any model that actually + * dispatches `ModelDownloadTask` rejects fast with "No run function found … + * model.download". We use that as the observable signal that a download was + * *attempted*; a clean resolve is the signal that it was *skipped* (a no-op). + */ +const cloud = (id: string, provider: string): ModelConfig => + ({ model_id: id, provider, provider_config: { model_name: id } }) as unknown as ModelConfig; + +const llamaCpp = (id: string, provider_config: Record): ModelConfig => + ({ model_id: id, provider: "LOCAL_LLAMACPP", provider_config }) as unknown as ModelConfig; + +const hft = (id: string): ModelConfig => + ({ + model_id: id, + provider: "HF_TRANSFORMERS_ONNX", + provider_config: { model_path: id }, + }) as unknown as ModelConfig; + +describe("ensureModelDownloaded", () => { + beforeEach(() => { + resetEnsuredModelsForTesting(); + }); + + it("is a no-op for cloud providers (nothing to download)", async () => { + await expect(ensureModelDownloaded(cloud("claude-sonnet-5", "ANTHROPIC"))).resolves.toBeUndefined(); + await expect(ensureModelDownloaded(cloud("gpt-5.5", "OPENAI"))).resolves.toBeUndefined(); + await expect( + ensureModelDownloaded(cloud("gemini-3-flash-preview", "GOOGLE_GEMINI")) + ).resolves.toBeUndefined(); + await expect(ensureModelDownloaded(cloud("grok-4.5", "XAI"))).resolves.toBeUndefined(); + }); + + it("skips a bare-path GGUF (no model_url): the file is assumed on disk", async () => { + await expect( + ensureModelDownloaded(llamaCpp("gguf:/models/x.gguf", { model_path: "/models/x.gguf" })) + ).resolves.toBeUndefined(); + }); + + it("attempts a download for a HuggingFace ONNX model", async () => { + await expect(ensureModelDownloaded(hft("onnx-community/x"))).rejects.toThrow(/model\.download/); + }); + + it("attempts a download for a GGUF model that has a model_url", async () => { + await expect( + ensureModelDownloaded( + llamaCpp("gguf:hf:org/repo:Q4_K_M", { + model_path: "/models/repo-Q4_K_M.gguf", + model_url: "hf:org/repo:Q4_K_M", + models_dir: "/models", + }) + ) + ).rejects.toThrow(/model\.download/); + }); + + it("downloads each model at most once (memoized across calls)", async () => { + const id = "gguf:memo.gguf"; + // First mark the id ready via the bare-path (no-op) branch. + await ensureModelDownloaded(llamaCpp(id, { model_path: "/models/memo.gguf" })); + // A later record with the SAME id would normally attempt a download and + // reject — but the memo short-circuits it, so it resolves cleanly. + await expect( + ensureModelDownloaded( + llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }) + ) + ).resolves.toBeUndefined(); + // Sanity: without the memo, the url-bearing record does attempt (and reject). + resetEnsuredModelsForTesting(); + await expect( + ensureModelDownloaded( + llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }) + ) + ).rejects.toThrow(/model\.download/); + }); +}); diff --git a/src/config/ensureModelDownloaded.ts b/src/config/ensureModelDownloaded.ts new file mode 100644 index 00000000..22b72b1e --- /dev/null +++ b/src/config/ensureModelDownloaded.ts @@ -0,0 +1,93 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { IExecuteContext, ModelConfig } from "workglow"; +import { ModelDownloadTask } from "workglow"; + +/** + * Providers whose weights are fetched from a remote source and cached to disk by + * a `model.download` run-fn — the local providers. The cloud providers + * (`ANTHROPIC`, `OPENAI`, `GOOGLE_GEMINI`, `XAI`) register no such run-fn, so a + * `ModelDownloadTask` for them would throw "no run-fn for provider serving + * model.download"; downloading is therefore a no-op for anything not listed here. + */ +const DOWNLOADABLE_PROVIDERS = new Set(["HF_TRANSFORMERS_ONNX", "LOCAL_LLAMACPP"]); +const LLAMACPP_PROVIDER = "LOCAL_LLAMACPP"; + +/** + * Model ids downloaded (or confirmed ready) in this process. A multi-section run + * or an eval sweep drives the same model many times; the download run-fn is + * idempotent but not free (it re-scans/verifies on-disk files and re-emits + * progress), so we run it at most once per model. + */ +const ensured = new Set(); + +/** @internal Reset the memo — for tests only. */ +export function resetEnsuredModelsForTesting(): void { + ensured.clear(); +} + +/** + * Minimal execution context for driving {@link ModelDownloadTask.execute} outside + * a full task-graph run. Mirrors the stub used elsewhere for one-shot CLI task + * execution — download only needs `signal` / `updateProgress` / `own`, with + * defensive `registry` / `resourceScope` shims. + */ +function makeStubContext(): IExecuteContext { + return { + signal: new AbortController().signal, + updateProgress: async () => {}, + own: (value: T): T => value, + registry: { + has: () => false, + get: () => { + throw new Error("not registered"); + }, + } as any, + resourceScope: { register: () => {}, dispose: async () => {} } as any, + } as IExecuteContext; +} + +/** + * Ensure a model's weights are present locally before it is used for generation. + * + * Providers differ in when weights arrive: cloud models have nothing to download + * (no-op); HuggingFace ONNX auto-downloads on first generation anyway, so this + * merely fetches it ahead of the timed work; and node-llama-cpp (GGUF) loads its + * `model_path` directly and does **not** fetch on generation — so a GGUF model + * configured with a `model_url` only lands on disk if `ModelDownloadTask` runs + * first. This helper is the seam every SEC model consumer calls to make that + * download happen uniformly, whatever the env-configured provider. + * + * A GGUF record with only a local `model_path` (no `model_url`) is assumed already + * on disk — the provider loads it directly and node-llama-cpp's downloader has no + * URI to fetch — so download is skipped; a missing file surfaces at load time with + * the provider's own error. + * + * Uses {@link ModelDownloadTask} (not the provider run-fn directly) so provider + * resolution and progress handling stay in the task layer. Memoized per model id. + */ +export async function ensureModelDownloaded(model: ModelConfig): Promise { + const provider = (model as { provider?: string }).provider; + if (!provider || !DOWNLOADABLE_PROVIDERS.has(provider)) return; + + const modelId = (model as { model_id?: string }).model_id ?? ""; + if (modelId && ensured.has(modelId)) return; + + if (provider === LLAMACPP_PROVIDER) { + const config = (model as { provider_config?: { model_url?: string } }).provider_config; + if (!config?.model_url) { + // Bare local GGUF path: nothing to fetch. Mark ready so we don't re-check. + if (modelId) ensured.add(modelId); + return; + } + } + + const input = { model }; + const task = new ModelDownloadTask({ defaults: input } as any); + await task.execute(input as any, makeStubContext()); + if (modelId) ensured.add(modelId); +} diff --git a/src/config/registerModels.test.ts b/src/config/registerModels.test.ts index 46e66412..fb190f7e 100644 --- a/src/config/registerModels.test.ts +++ b/src/config/registerModels.test.ts @@ -16,6 +16,7 @@ import { anthropicModelRecord, geminiModelRecord, hftModelRecord, + llamaCppModelRecord, openAiModelRecord, registerModelIds, registerSecModels, @@ -114,6 +115,54 @@ describe("registerSecModels", () => { expect((await repo.findByName("onnx-community/tiny"))?.provider).toBe("HF_TRANSFORMERS_ONNX"); }); + describe("llamaCppModelRecord GGUF id parsing", () => { + const savedGgufEnv = { + SEC_GGUF_DIR: process.env.SEC_GGUF_DIR, + SEC_RAW_DATA_FOLDER: process.env.SEC_RAW_DATA_FOLDER, + }; + beforeEach(() => { + process.env.SEC_GGUF_DIR = "/models/gguf"; + delete process.env.SEC_RAW_DATA_FOLDER; + }); + afterEach(() => { + for (const [key, value] of Object.entries(savedGgufEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + + it("keeps a relative local path under the models dir, no download url", () => { + const config = llamaCppModelRecord("gguf:Bonsai-27B-Q2_0.gguf").provider_config; + expect(config.model_path).toBe("/models/gguf/Bonsai-27B-Q2_0.gguf"); + expect(config.model_url).toBeUndefined(); + expect(config.models_dir).toBeUndefined(); + }); + + it("keeps an absolute local path as-is", () => { + const config = llamaCppModelRecord("gguf:/abs/model.gguf").provider_config; + expect(config.model_path).toBe("/abs/model.gguf"); + expect(config.model_url).toBeUndefined(); + }); + + it("turns an hf: URI into a download source + cache target", () => { + const config = llamaCppModelRecord( + "gguf:hf:bartowski/SmolLM2-135M-Instruct-GGUF:Q4_K_M" + ).provider_config; + expect(config.model_url).toBe("hf:bartowski/SmolLM2-135M-Instruct-GGUF:Q4_K_M"); + expect(config.models_dir).toBe("/models/gguf"); + expect(config.model_path).toBe("/models/gguf/SmolLM2-135M-Instruct-GGUF-Q4_K_M.gguf"); + }); + + it("turns an https URL into a download source + cache target", () => { + const config = llamaCppModelRecord( + "gguf:https://host.example/a/b/model.gguf" + ).provider_config; + expect(config.model_url).toBe("https://host.example/a/b/model.gguf"); + expect(config.models_dir).toBe("/models/gguf"); + expect(config.model_path).toBe("/models/gguf/model.gguf"); + }); + }); + it("registers the SEC_LOI_MODEL override id", async () => { process.env.SEC_LOI_MODEL = "claude-haiku-4-5"; await registerSecModels(); diff --git a/src/config/registerModels.ts b/src/config/registerModels.ts index 4be17489..fefdf101 100644 --- a/src/config/registerModels.ts +++ b/src/config/registerModels.ts @@ -26,10 +26,12 @@ const LLAMACPP_PROVIDER = "LOCAL_LLAMACPP"; /** * Prefix that routes a model id to the node-llama-cpp (GGUF) provider. The rest - * of the id is a path to a local `.gguf` file — absolute (`gguf:/abs/x.gguf`) or - * relative to the GGUF models dir (`gguf:Qwen3.6-35B-A3B-UD-Q4_K_M.gguf`). We key - * on an explicit prefix rather than the id shape because a GGUF repo id - * (`org/name`) is indistinguishable from an ONNX one. + * of the id is either a **local path** to a `.gguf` file — absolute + * (`gguf:/abs/x.gguf`) or relative to the GGUF models dir + * (`gguf:Qwen3.6-35B-A3B-UD-Q4_K_M.gguf`) — or a **remote URI** the download + * harness fetches: a node-llama-cpp HuggingFace URI (`gguf:hf:org/repo:Q4_K_M`) + * or an `https://` URL. We key on an explicit prefix rather than the id shape + * because a GGUF repo id (`org/name`) is indistinguishable from an ONNX one. */ const GGUF_ID_PREFIX = "gguf:"; @@ -281,20 +283,71 @@ const LLAMACPP_CAPABILITIES: readonly string[] = [ ]; /** - * Builds a node-llama-cpp {@link ModelRecord} from a `gguf:`-prefixed id whose - * remainder is a path to a local `.gguf` file (absolute, or relative to - * {@link ggufModelsDir}). `gpu_layers` is set high (offload every layer to the + * Builds a node-llama-cpp {@link ModelRecord} from a `gguf:`-prefixed id. The + * remainder is either a **local path** to a `.gguf` file (absolute, or relative + * to {@link ggufModelsDir}) or a **remote URI** (`hf:org/repo:quant` or an + * `https://` URL) that carries a `model_url` for the download harness to fetch — + * see {@link ggufPathConfig}. `gpu_layers` is set high (offload every layer to the * GPU — Metal on Apple Silicon; node-llama-cpp clamps to the model's layer * count). Do NOT use `-1`: that is llama.cpp's "all" sentinel but node-llama-cpp * treats it as zero, silently running the whole model on CPU (~20x slower). - * `context_size` is sized for large S-1 sections. The file must already exist on - * disk — the provider loads `model_path` directly, no download at generation. + * `context_size` is sized for large S-1 sections. A bare local path must already + * exist on disk — the provider loads `model_path` directly and does not fetch at + * generation; the download harness (`ensureModelDownloaded`) fetches a `model_url` + * ahead of use. */ const GGUF_GPU_LAYERS_ALL = 999; +/** A `gguf:` remainder that names a remote source the download harness can fetch. */ +function isRemoteGgufUri(rawPath: string): boolean { + return rawPath.startsWith("hf:") || /^https?:\/\//i.test(rawPath); +} + +/** + * Local `.gguf` filename to cache a remote GGUF under, derived from its URI. This + * is only a fallback for the required `model_path` field: once the download runs, + * the provider keys on `model_url` and resolves the real on-disk path itself, so + * the exact name here does not affect which file generation loads — it just gives + * the cache target a stable, human-legible name. + * + * `hf:org/repo:Q4_K_M` → `repo-Q4_K_M.gguf`; `hf:org/repo/file.gguf` → `file.gguf`; + * `https://host/a/b/model.gguf` → `model.gguf`. + */ +function ggufCacheFileName(uri: string): string { + let rest = uri.replace(/^https?:\/\//i, "").replace(/^hf:/i, ""); + // A trailing `:quant` (HF quant selector) is a filename hint, not a path segment. + let quant: string | undefined; + const quantMatch = rest.match(/:([^/:]+)$/); + if (quantMatch) { + quant = quantMatch[1]; + rest = rest.slice(0, rest.length - quant.length - 1); + } + const last = rest.split(/[/?#]/).filter(Boolean).pop() ?? "model"; + const base = last.toLowerCase().endsWith(".gguf") ? last.slice(0, -".gguf".length) : last; + const name = [base, quant].filter(Boolean).join("-") || "model"; + return `${name}.gguf`; +} + +/** + * Resolves a `gguf:` id remainder into node-llama-cpp path config. A remote URI + * ({@link isRemoteGgufUri}) becomes `model_url` (the download source) plus a local + * `model_path` / `models_dir` under {@link ggufModelsDir} so the harness fetches + * it there; a plain path stays `model_path`-only (assumed already on disk). + */ +function ggufPathConfig(rawPath: string): Record { + if (!isRemoteGgufUri(rawPath)) { + return { model_path: isAbsolute(rawPath) ? rawPath : join(ggufModelsDir(), rawPath) }; + } + const dir = ggufModelsDir(); + return { + model_path: join(dir, ggufCacheFileName(rawPath)), + model_url: rawPath, + models_dir: dir, + }; +} + export function llamaCppModelRecord(modelId: string): ModelRecord { const rawPath = modelId.slice(GGUF_ID_PREFIX.length); - const modelPath = isAbsolute(rawPath) ? rawPath : join(ggufModelsDir(), rawPath); return { model_id: modelId, provider: LLAMACPP_PROVIDER, @@ -302,7 +355,7 @@ export function llamaCppModelRecord(modelId: string): ModelRecord { description: `node-llama-cpp GGUF ${rawPath}`, capabilities: [...LLAMACPP_CAPABILITIES], provider_config: { - model_path: modelPath, + ...ggufPathConfig(rawPath), gpu_layers: GGUF_GPU_LAYERS_ALL, context_size: ggufContextSize(), flash_attention: true, diff --git a/src/eval/runExtractionEval.ts b/src/eval/runExtractionEval.ts index 48401db4..b987e6e0 100644 --- a/src/eval/runExtractionEval.ts +++ b/src/eval/runExtractionEval.ts @@ -6,6 +6,7 @@ import type { ModelConfig } from "workglow"; import { getGlobalModelRepository } from "workglow"; +import { ensureModelDownloaded } from "../config/ensureModelDownloaded"; import { registerModelIds } from "../config/registerModels"; import { EVAL_EXTRACTORS, EVAL_FIXTURES, type EvalFixture } from "./fixtures"; import { estimateCost, type CostEstimate } from "./modelPricing"; @@ -182,6 +183,16 @@ export async function runExtractionEval(opts: RunEvalOptions): Promise(); const push = (r: OracleRunResult): void => { diff --git a/src/eval/runUnitTermsEval.ts b/src/eval/runUnitTermsEval.ts index 06ebc4ff..28950c3e 100644 --- a/src/eval/runUnitTermsEval.ts +++ b/src/eval/runUnitTermsEval.ts @@ -6,6 +6,7 @@ import type { ModelConfig } from "workglow"; import { getGlobalModelRepository } from "workglow"; +import { ensureModelDownloaded } from "../config/ensureModelDownloaded"; import { registerModelIds } from "../config/registerModels"; import { cikFromFilingName, @@ -124,6 +125,14 @@ export async function runUnitTermsEval(opts: RunUnitTermsOptions): Promise> { + // Local providers (GGUF especially) must have their weights on disk before + // generation — cloud models no-op here. Memoized, so the per-section sweep pays + // the download once. + await ensureModelDownloaded(model); const grammarConstrained = (model as { provider?: string }).provider === "LOCAL_LLAMACPP"; const input = { model, From 1f18fa365c6c5cdb14dc988ae24055329836c57f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 02:33:10 +0000 Subject: [PATCH 2/4] feat(models): render model-download progress through the CLI task UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The download seam used a throwaway execute context, which silently discarded the download run-fn's progress events — a multi-GB GGUF/ONNX fetch looked like a hang. Thread the running task's real IExecuteContext into ensureModelDownloaded and on to ModelDownloadTask.execute: the download's phase events are forwarded to context.updateProgress, which the @workglow/cli progress UI (withCli) already renders, and context.signal now aborts a long download on Ctrl-C. Add prefetchModel, the best-effort wrapper the CLI-task boundaries call to fetch weights (with visible progress) before the work begins: - eval sweeps pass their task context so `sec eval` shows download progress; - the AI form processors (S-1, 424, merger-proxy, redemption, LOI) prefetch once after resolving their model, via a context threaded through the shared storageArgs in ProcessAccessionDocFormTask. runStructured keeps a context-less ensureModelDownloaded call as a per-section correctness safety-net (it downloads silently only if a model was never prefetched, e.g. a sub-extractor's distinct model); the progress-bearing fetch lives at the task boundary. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011HoQKRtpU7mGpK8Dah9ySm --- CLAUDE.md | 26 +++-- src/config/ensureModelDownloaded.test.ts | 96 ++++++++++++++++--- src/config/ensureModelDownloaded.ts | 57 ++++++----- src/eval/runExtractionEval.ts | 24 ++--- src/eval/runOracleEval.ts | 21 ++-- src/eval/runUnitTermsEval.ts | 18 ++-- .../miscellaneous-filings/Form_8_K.storage.ts | 6 +- src/sec/forms/miscellaneous-filings/loi8k.ts | 5 +- .../miscellaneous-filings/redemption8k.ts | 5 +- .../Form_DEFM14A.storage.ts | 5 +- .../Form_424.storage.ts | 5 +- .../Form_S_1.storage.ts | 7 +- .../s1/sectionExtractors.ts | 14 ++- src/task/eval/EvalExtractTask.ts | 1 + src/task/eval/EvalS1Task.ts | 1 + src/task/eval/EvalUnitTermsTask.ts | 1 + src/task/forms/ProcessAccessionDocFormTask.ts | 4 + 17 files changed, 210 insertions(+), 86 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index fcc80e25..f0d0be18 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,13 +96,25 @@ Local model weights must be on disk before generation, and providers differ on when that happens: cloud models have nothing to download; HuggingFace ONNX auto-fetches on first generation; but node-llama-cpp (GGUF) loads its `model_path` directly and never fetches at generation. `ensureModelDownloaded` -(`src/config/ensureModelDownloaded.ts`) is the single seam that normalizes this — -every extraction (through `runStructured`) and every eval sweep calls it before -using a model. It runs `ModelDownloadTask` for the local providers (no-op for -cloud, memoized per model id so a per-section sweep pays the download once) and -skips a bare-path GGUF (no `model_url` — the file is assumed on disk). The eval -loops prefetch before their timed sections so download time isn't charged to a -model's measured latency. +(`src/config/ensureModelDownloaded.ts`) is the single seam that normalizes this. +It runs `ModelDownloadTask` for the local providers (no-op for cloud, memoized +per model id so a per-section sweep pays the download once) and skips a bare-path +GGUF (no `model_url` — the file is assumed on disk). + +It takes the running task's `IExecuteContext`: passing the **real** one (not a +throwaway stub) is what surfaces download progress — the download run-fn's `phase` +events are forwarded to `context.updateProgress`, which the `@workglow/cli` +progress UI (`withCli`) renders, so a multi-GB GGUF/ONNX fetch shows a live +percentage instead of a silent hang (and `context.signal` aborts it on Ctrl-C). +`prefetchModel(model, context)` is the best-effort wrapper the CLI-task boundaries +call: the AI form processors (`processFormS1` / `processForm424` / +`processMergerProxy` / `processRedemption8K` / `processLoi8K`, via a `context` +threaded through `storageArgs`) prefetch once after resolving their model, and the +eval loops prefetch before their timed sections (so download time isn't charged to +a model's measured latency). `runStructured` keeps a context-less +`ensureModelDownloaded` call as a per-section correctness safety-net — it downloads +silently if a model was never prefetched (e.g. a sub-extractor's distinct model), +but the progress-bearing fetch lives at the task boundary. To make GGUF weights fetchable rather than pre-staged, a `gguf:` id may be a **remote URI** — a node-llama-cpp HuggingFace URI (`gguf:hf:org/repo:Q4_K_M`) or diff --git a/src/config/ensureModelDownloaded.test.ts b/src/config/ensureModelDownloaded.test.ts index e790ac83..9286e734 100644 --- a/src/config/ensureModelDownloaded.test.ts +++ b/src/config/ensureModelDownloaded.test.ts @@ -4,13 +4,28 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type { ModelConfig } from "workglow"; -import { beforeEach, describe, expect, it } from "vitest"; +import type { IExecuteContext, ModelConfig } from "workglow"; +import { + AiProviderRegistry, + getAiProviderRegistry, + setAiProviderRegistry, +} from "workglow"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { ensureModelDownloaded, resetEnsuredModelsForTesting, } from "./ensureModelDownloaded"; +/** A throwaway execute context — these tests never reach a real provider. */ +const ctx = (): IExecuteContext => + ({ + signal: new AbortController().signal, + updateProgress: async () => {}, + own: (v: T): T => v, + registry: { has: () => false, get: () => { throw new Error("x"); } }, + resourceScope: { register: () => {}, dispose: async () => {} }, + }) as unknown as IExecuteContext; + /** * No AI providers are registered in this suite, so any model that actually * dispatches `ModelDownloadTask` rejects fast with "No run function found … @@ -36,22 +51,22 @@ describe("ensureModelDownloaded", () => { }); it("is a no-op for cloud providers (nothing to download)", async () => { - await expect(ensureModelDownloaded(cloud("claude-sonnet-5", "ANTHROPIC"))).resolves.toBeUndefined(); - await expect(ensureModelDownloaded(cloud("gpt-5.5", "OPENAI"))).resolves.toBeUndefined(); + await expect(ensureModelDownloaded(cloud("claude-sonnet-5", "ANTHROPIC"), ctx())).resolves.toBeUndefined(); + await expect(ensureModelDownloaded(cloud("gpt-5.5", "OPENAI"), ctx())).resolves.toBeUndefined(); await expect( - ensureModelDownloaded(cloud("gemini-3-flash-preview", "GOOGLE_GEMINI")) + ensureModelDownloaded(cloud("gemini-3-flash-preview", "GOOGLE_GEMINI"), ctx()) ).resolves.toBeUndefined(); - await expect(ensureModelDownloaded(cloud("grok-4.5", "XAI"))).resolves.toBeUndefined(); + await expect(ensureModelDownloaded(cloud("grok-4.5", "XAI"), ctx())).resolves.toBeUndefined(); }); it("skips a bare-path GGUF (no model_url): the file is assumed on disk", async () => { await expect( - ensureModelDownloaded(llamaCpp("gguf:/models/x.gguf", { model_path: "/models/x.gguf" })) + ensureModelDownloaded(llamaCpp("gguf:/models/x.gguf", { model_path: "/models/x.gguf" }), ctx()) ).resolves.toBeUndefined(); }); it("attempts a download for a HuggingFace ONNX model", async () => { - await expect(ensureModelDownloaded(hft("onnx-community/x"))).rejects.toThrow(/model\.download/); + await expect(ensureModelDownloaded(hft("onnx-community/x"), ctx())).rejects.toThrow(); }); it("attempts a download for a GGUF model that has a model_url", async () => { @@ -61,28 +76,81 @@ describe("ensureModelDownloaded", () => { model_path: "/models/repo-Q4_K_M.gguf", model_url: "hf:org/repo:Q4_K_M", models_dir: "/models", - }) + }), + ctx() ) - ).rejects.toThrow(/model\.download/); + ).rejects.toThrow(); }); it("downloads each model at most once (memoized across calls)", async () => { const id = "gguf:memo.gguf"; // First mark the id ready via the bare-path (no-op) branch. - await ensureModelDownloaded(llamaCpp(id, { model_path: "/models/memo.gguf" })); + await ensureModelDownloaded(llamaCpp(id, { model_path: "/models/memo.gguf" }), ctx()); // A later record with the SAME id would normally attempt a download and // reject — but the memo short-circuits it, so it resolves cleanly. await expect( ensureModelDownloaded( - llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }) + llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }), + ctx() ) ).resolves.toBeUndefined(); // Sanity: without the memo, the url-bearing record does attempt (and reject). resetEnsuredModelsForTesting(); await expect( ensureModelDownloaded( - llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }) + llamaCpp(id, { model_path: "/models/memo.gguf", model_url: "hf:org/memo:Q4" }), + ctx() ) - ).rejects.toThrow(/model\.download/); + ).rejects.toThrow(); + }); + + describe("progress + abort forwarding", () => { + // Swap in a throwaway provider registry so a fake download run-fn doesn't leak + // into sibling suites, mirroring registerModels.test's model-repo swap. + let original: AiProviderRegistry; + beforeEach(() => { + original = getAiProviderRegistry(); + setAiProviderRegistry(new AiProviderRegistry()); + resetEnsuredModelsForTesting(); + }); + afterEach(() => { + setAiProviderRegistry(original); + }); + + it("forwards the download run-fn's progress to context.updateProgress (and memoizes)", async () => { + let runFnCalls = 0; + // Real provider run-fns are plain async fns that push events via `emit` + // (see LlamaCpp_Download), not generators. + getAiProviderRegistry().registerRunFn("HF_TRANSFORMERS_ONNX", { + serves: ["model.download"], + runFn: async (input: any, _model: any, _signal: any, emit: any) => { + runFnCalls += 1; + emit({ type: "phase", message: "Downloading model", progress: 42 }); + emit({ type: "finish", data: { model: input.model } }); + }, + } as any); + + const progress: Array<[number | undefined, string | undefined]> = []; + const context = { + signal: new AbortController().signal, + updateProgress: async (p: number | undefined, m?: string) => { + progress.push([p, m]); + }, + own: (v: T): T => v, + registry: { has: () => false, get: () => { throw new Error("x"); } }, + resourceScope: { register: () => {}, dispose: async () => {} }, + } as unknown as IExecuteContext; + + const model = hft("onnx-community/progress"); + await ensureModelDownloaded(model, context); + // The download's phase event reached the running task's progress sink — this + // is what the withCli UI renders on screen. + expect(runFnCalls).toBe(1); + expect(progress).toContainEqual([42, "Downloading model"]); + + // Memoized: a second call does not re-invoke the download run-fn. + await ensureModelDownloaded(model, context); + expect(runFnCalls).toBe(1); + }); }); }); diff --git a/src/config/ensureModelDownloaded.ts b/src/config/ensureModelDownloaded.ts index 22b72b1e..05d5e0c1 100644 --- a/src/config/ensureModelDownloaded.ts +++ b/src/config/ensureModelDownloaded.ts @@ -30,27 +30,6 @@ export function resetEnsuredModelsForTesting(): void { ensured.clear(); } -/** - * Minimal execution context for driving {@link ModelDownloadTask.execute} outside - * a full task-graph run. Mirrors the stub used elsewhere for one-shot CLI task - * execution — download only needs `signal` / `updateProgress` / `own`, with - * defensive `registry` / `resourceScope` shims. - */ -function makeStubContext(): IExecuteContext { - return { - signal: new AbortController().signal, - updateProgress: async () => {}, - own: (value: T): T => value, - registry: { - has: () => false, - get: () => { - throw new Error("not registered"); - }, - } as any, - resourceScope: { register: () => {}, dispose: async () => {} } as any, - } as IExecuteContext; -} - /** * Ensure a model's weights are present locally before it is used for generation. * @@ -67,10 +46,17 @@ function makeStubContext(): IExecuteContext { * URI to fetch — so download is skipped; a missing file surfaces at load time with * the provider's own error. * - * Uses {@link ModelDownloadTask} (not the provider run-fn directly) so provider - * resolution and progress handling stay in the task layer. Memoized per model id. + * `context` is the running task's {@link IExecuteContext}. Passing the real one + * (rather than a throwaway stub) is what surfaces download progress: the download + * run-fn's `phase` events are forwarded to `context.updateProgress`, which the + * CLI progress UI (`withCli`) renders — so a multi-GB GGUF/ONNX fetch shows a live + * percentage instead of a silent hang. `context.signal` also aborts a long + * download on Ctrl-C. Memoized per model id, so a per-section sweep pays it once. */ -export async function ensureModelDownloaded(model: ModelConfig): Promise { +export async function ensureModelDownloaded( + model: ModelConfig, + context: IExecuteContext +): Promise { const provider = (model as { provider?: string }).provider; if (!provider || !DOWNLOADABLE_PROVIDERS.has(provider)) return; @@ -88,6 +74,27 @@ export async function ensureModelDownloaded(model: ModelConfig): Promise { const input = { model }; const task = new ModelDownloadTask({ defaults: input } as any); - await task.execute(input as any, makeStubContext()); + await task.execute(input as any, context); if (modelId) ensured.add(modelId); } + +/** + * Best-effort prefetch used at the CLI-task boundary (form processors, eval + * sweeps) to surface download progress before the work begins. No-ops when there + * is no model or no context (a direct/test caller), and swallows failures — the + * downstream generation path re-attempts via {@link ensureModelDownloaded} and + * records the failure in its own way (dead-letter or failed eval run), so a + * prefetch problem must never abort the run. Whether it downloads with a visible + * progress bar is decided entirely by whether a real `context` is threaded in. + */ +export async function prefetchModel( + model: ModelConfig | null | undefined, + context: IExecuteContext | undefined +): Promise { + if (!model || !context) return; + try { + await ensureModelDownloaded(model, context); + } catch { + // Downstream generation re-attempts the download and records any failure. + } +} diff --git a/src/eval/runExtractionEval.ts b/src/eval/runExtractionEval.ts index b987e6e0..00b470f9 100644 --- a/src/eval/runExtractionEval.ts +++ b/src/eval/runExtractionEval.ts @@ -4,9 +4,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type { ModelConfig } from "workglow"; +import type { IExecuteContext, ModelConfig } from "workglow"; import { getGlobalModelRepository } from "workglow"; -import { ensureModelDownloaded } from "../config/ensureModelDownloaded"; +import { prefetchModel } from "../config/ensureModelDownloaded"; import { registerModelIds } from "../config/registerModels"; import { EVAL_EXTRACTORS, EVAL_FIXTURES, type EvalFixture } from "./fixtures"; import { estimateCost, type CostEstimate } from "./modelPricing"; @@ -55,6 +55,13 @@ export interface RunEvalOptions { readonly onProgress?: (done: number, total: number, message: string) => void; /** When aborted, the sweep stops after the current run and reports what ran. */ readonly signal?: AbortSignal; + /** + * The running task's execute context. When present, a local model's weights are + * prefetched through it so the download's progress renders in the CLI task UI + * (and Ctrl-C aborts the fetch). Omitted by direct callers (tests); the download + * then falls back to the per-section safety-net in `runStructured`. + */ + readonly context?: IExecuteContext; } /** Extractor ids that actually have at least one committed fixture. */ @@ -184,15 +191,10 @@ export async function runExtractionEval(opts: RunEvalOptions): Promise void; /** When aborted, the sweep stops after the current section and reports what ran. */ readonly signal?: AbortSignal; + /** + * The running task's execute context. When present, participating local models + * are prefetched through it so download progress renders in the CLI task UI. + * Omitted by direct callers (tests); `runStructured`'s safety-net then downloads. + */ + readonly context?: IExecuteContext; } async function runSection( @@ -189,13 +195,10 @@ export async function runOracleEval(opts: RunOracleOptions): Promise void; readonly signal?: AbortSignal; + /** + * The running task's execute context. When present, a local model's weights are + * prefetched through it so download progress renders in the CLI task UI. Omitted + * by direct callers (tests); `runStructured`'s safety-net then downloads. + */ + readonly context?: IExecuteContext; } export interface UnitTermsReport extends EvalReport { @@ -126,13 +132,7 @@ export async function runUnitTermsEval(opts: RunUnitTermsOptions): Promise { const eventRepo = new Form8KEventRepo(); const isAmendment = form === "8-K/A"; @@ -151,6 +153,7 @@ export async function processForm8K({ itemCodes, fullSubmissionText, model, + context, }); await processLoi8K({ cik, @@ -161,6 +164,7 @@ export async function processForm8K({ fullSubmissionText, event_date: effectiveReportDate || filing_date, model, + context, }); } } diff --git a/src/sec/forms/miscellaneous-filings/loi8k.ts b/src/sec/forms/miscellaneous-filings/loi8k.ts index 91314880..d6e1cc9e 100644 --- a/src/sec/forms/miscellaneous-filings/loi8k.ts +++ b/src/sec/forms/miscellaneous-filings/loi8k.ts @@ -3,8 +3,9 @@ * Copyright 2026 Steven Roussey * SPDX-License-Identifier: Apache-2.0 */ -import type { ModelConfig } from "workglow"; +import type { IExecuteContext, ModelConfig } from "workglow"; import { globalServiceRegistry, renderMarkdown } from "workglow"; +import { prefetchModel } from "../../../config/ensureModelDownloaded"; import { parseEdgarHtml } from "../../html/parseEdgarHtml"; import { parseEightKSubmission } from "../registration-statements/s1/parseSubmission"; import { makeRunSection } from "../registration-statements/s1/sectionRunner"; @@ -45,6 +46,7 @@ export interface ProcessLoi8KArgs { /** Event-date fallback when the narrative states no LOI date (report_date ?? filing_date). */ readonly event_date?: string; readonly model?: ModelConfig; + readonly context?: IExecuteContext; } /** Renders an EDGAR HTML body to plain markdown text (source-span verifiable). */ @@ -131,6 +133,7 @@ export async function processLoi8K(args: ProcessLoi8KArgs): Promise { await recordLoiRun(false, `MODEL_RESOLUTION_ERROR: ${message}`); return; } + await prefetchModel(model, args.context); let text: string; let dropped = 0; diff --git a/src/sec/forms/miscellaneous-filings/redemption8k.ts b/src/sec/forms/miscellaneous-filings/redemption8k.ts index 21a0b7db..ff940fdd 100644 --- a/src/sec/forms/miscellaneous-filings/redemption8k.ts +++ b/src/sec/forms/miscellaneous-filings/redemption8k.ts @@ -3,8 +3,9 @@ * Copyright 2026 Steven Roussey * SPDX-License-Identifier: Apache-2.0 */ -import type { ModelConfig } from "workglow"; +import type { IExecuteContext, ModelConfig } from "workglow"; import { globalServiceRegistry, renderMarkdown } from "workglow"; +import { prefetchModel } from "../../../config/ensureModelDownloaded"; import { parseEdgarHtml } from "../../html/parseEdgarHtml"; import { parseEightKSubmission } from "../registration-statements/s1/parseSubmission"; import { makeRunSection } from "../registration-statements/s1/sectionRunner"; @@ -53,6 +54,7 @@ export interface ProcessRedemption8KArgs { readonly itemCodes: readonly string[]; readonly fullSubmissionText: string; readonly model?: ModelConfig; + readonly context?: IExecuteContext; } /** Renders an EDGAR HTML body to plain markdown text (source-span verifiable). */ @@ -137,6 +139,7 @@ export async function processRedemption8K(args: ProcessRedemption8KArgs): Promis await recordRedemptionRun(false, `MODEL_RESOLUTION_ERROR: ${message}`); return; } + await prefetchModel(model, args.context); // Parsing/rendering filer-supplied HTML must not abort the filing (its 8-K // events and milestone deals already wrote); a malformed body dead-letters the diff --git a/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts b/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts index 1d361c7f..fe01d003 100644 --- a/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts +++ b/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts @@ -4,7 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { globalServiceRegistry, type ModelConfig } from "workglow"; +import { globalServiceRegistry, type IExecuteContext, type ModelConfig } from "workglow"; +import { prefetchModel } from "../../../config/ensureModelDownloaded"; import { buildEntityObserver } from "../../../resolver/buildEntityObserver"; import { CanonicalCompanyRepo } from "../../../storage/canonical/CanonicalCompanyRepo"; import { COMPONENT_VERSION_REPOSITORY_TOKEN } from "../../../storage/versioning/ComponentVersionSchema"; @@ -48,6 +49,7 @@ export interface ProcessMergerProxyArgs { readonly form: string; readonly formMergerProxy: FormS1Parsed; readonly model?: ModelConfig; + readonly context?: IExecuteContext; } /** @@ -99,6 +101,7 @@ export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise< modelError = err instanceof Error ? err.message : String(err); } const model_id = model ? resolveModelId(model) : null; + await prefetchModel(model, args.context); const recordMergerProxyRun = async (success: boolean, error: string | null): Promise => { try { diff --git a/src/sec/forms/registration-statements/Form_424.storage.ts b/src/sec/forms/registration-statements/Form_424.storage.ts index 55375f75..b4b9b546 100644 --- a/src/sec/forms/registration-statements/Form_424.storage.ts +++ b/src/sec/forms/registration-statements/Form_424.storage.ts @@ -4,7 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { globalServiceRegistry, type ModelConfig } from "workglow"; +import { globalServiceRegistry, type IExecuteContext, type ModelConfig } from "workglow"; +import { prefetchModel } from "../../../config/ensureModelDownloaded"; import { buildEntityObserver } from "../../../resolver/buildEntityObserver"; import { ExtractionDeadLetterRepo } from "../../../storage/dead-letter/ExtractionDeadLetterRepo"; import { ObservationProvenanceRepo } from "../../../storage/provenance/ObservationProvenanceRepo"; @@ -49,6 +50,7 @@ export interface ProcessForm424Args { readonly form: string; readonly form424: FormS1Parsed; readonly model?: ModelConfig; + readonly context?: IExecuteContext; } /** @@ -167,6 +169,7 @@ export async function processForm424(args: ProcessForm424Args): Promise { return; } const model_id = resolveModelId(model); + await prefetchModel(model, args.context); // Mirror the S-1 PARSE_ERROR containment: a converter throw dead-letters the // offering sections so the filing stays on the retry worklist. diff --git a/src/sec/forms/registration-statements/Form_S_1.storage.ts b/src/sec/forms/registration-statements/Form_S_1.storage.ts index de9d0db7..adac66d5 100644 --- a/src/sec/forms/registration-statements/Form_S_1.storage.ts +++ b/src/sec/forms/registration-statements/Form_S_1.storage.ts @@ -4,7 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { globalServiceRegistry, type ModelConfig } from "workglow"; +import { globalServiceRegistry, type IExecuteContext, type ModelConfig } from "workglow"; +import { prefetchModel } from "../../../config/ensureModelDownloaded"; import { CompanyObservationRepo } from "../../../storage/observation/CompanyObservationRepo"; import { buildEntityObserver } from "../../../resolver/buildEntityObserver"; import { COMPONENT_VERSION_REPOSITORY_TOKEN } from "../../../storage/versioning/ComponentVersionSchema"; @@ -79,6 +80,7 @@ export interface ProcessFormS1Args { readonly form: string; readonly formS1: FormS1Parsed; readonly model?: ModelConfig; + readonly context?: IExecuteContext; } export async function processFormS1(args: ProcessFormS1Args): Promise { @@ -96,6 +98,9 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { modelError = err instanceof Error ? err.message : String(err); } const model_id = model ? resolveModelId(model) : null; + // Fetch a local model's weights up front so the download's progress renders in + // the CLI task UI before the (silent) per-section extraction begins. + await prefetchModel(model, args.context); const versionRegistry = new VersionRegistry( globalServiceRegistry.get(COMPONENT_VERSION_REPOSITORY_TOKEN) diff --git a/src/sec/forms/registration-statements/s1/sectionExtractors.ts b/src/sec/forms/registration-statements/s1/sectionExtractors.ts index 69a60cab..10a6de2f 100644 --- a/src/sec/forms/registration-statements/s1/sectionExtractors.ts +++ b/src/sec/forms/registration-statements/s1/sectionExtractors.ts @@ -358,10 +358,14 @@ async function runStructured( prompt: string, outputSchema: object ): Promise> { - // Local providers (GGUF especially) must have their weights on disk before - // generation — cloud models no-op here. Memoized, so the per-section sweep pays - // the download once. - await ensureModelDownloaded(model); + const context = makeExecuteContext(); + // Correctness safety-net: local providers (GGUF especially) must have their + // weights on disk before generation — cloud models no-op here. Memoized, so the + // per-section sweep pays the download once; a form/eval run that prefetched with + // a real context (for visible progress) already satisfied this. This call uses + // the bare structured-generation context, so it downloads silently if it ever + // has to — the progress-bearing prefetch lives at the CLI-task boundary. + await ensureModelDownloaded(model, context); const grammarConstrained = (model as { provider?: string }).provider === "LOCAL_LLAMACPP"; const input = { model, @@ -371,7 +375,7 @@ async function runStructured( maxRetries: 1, }; const task = new StructuredGenerationTask({ defaults: input } as any); - const result = await task.execute(input as any, makeExecuteContext()); + const result = await task.execute(input as any, context); return (result?.object as Record | undefined) ?? {}; } diff --git a/src/task/eval/EvalExtractTask.ts b/src/task/eval/EvalExtractTask.ts index 0df5fc62..5b779721 100644 --- a/src/task/eval/EvalExtractTask.ts +++ b/src/task/eval/EvalExtractTask.ts @@ -72,6 +72,7 @@ export class EvalExtractTask extends Task { const pct = total === 0 ? 100 : Math.floor((done / total) * 100); void context.updateProgress(pct, message); diff --git a/src/task/eval/EvalS1Task.ts b/src/task/eval/EvalS1Task.ts index 26f0a47a..e557788a 100644 --- a/src/task/eval/EvalS1Task.ts +++ b/src/task/eval/EvalS1Task.ts @@ -71,6 +71,7 @@ export class EvalS1Task extends Task { extractors: input.extractors, dir: input.dir, signal: context.signal, + context, onProgress: (done, total, message) => { const pct = total === 0 ? 100 : Math.floor((done / total) * 100); void context.updateProgress(pct, message); diff --git a/src/task/eval/EvalUnitTermsTask.ts b/src/task/eval/EvalUnitTermsTask.ts index 6b9f9eb5..7ec02598 100644 --- a/src/task/eval/EvalUnitTermsTask.ts +++ b/src/task/eval/EvalUnitTermsTask.ts @@ -56,6 +56,7 @@ export class EvalUnitTermsTask extends Task { const pct = total === 0 ? 100 : Math.floor((done / total) * 100); void context.updateProgress(pct, message); diff --git a/src/task/forms/ProcessAccessionDocFormTask.ts b/src/task/forms/ProcessAccessionDocFormTask.ts index f21b9c36..f9f82a73 100644 --- a/src/task/forms/ProcessAccessionDocFormTask.ts +++ b/src/task/forms/ProcessAccessionDocFormTask.ts @@ -327,6 +327,10 @@ export class ProcessAccessionDocFormTask extends Task< accession_number: accessionNumber, filing_date: filing_date ?? "", primary_doc: fileName, + // Threaded to the AI form processors so a local model's download renders + // its progress in this task's CLI UI (via `prefetchModel`). Non-AI + // processors ignore it (spread bypasses excess-property checks). + context, }; switch (form) { From 2b199cb1ca7587ebff47684a9f60e29aad0f6665 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 04:35:38 +0000 Subject: [PATCH 3/4] feat(s1): surface per-section generation progress on the task row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread the running task's IExecuteContext down the extraction stack (processFormS1 / runOfferingSections / the 424, merger-proxy, redemption and LOI processors → extract* → runGuardedExtraction → runStructured), so each section's AI generation reports progress on the CLI task row instead of running silently. The context alone is not enough: StructuredGenerationTask.execute() keeps only the finish event and drops the phase events, so a threaded context would still see nothing. runStructured now drives the task's executeStream itself — identical result (same retry/validation loop, same finish) — and forwards the Preparing/Generating phases to context.updateProgress. Each section's row now shows the model actively working; the download progress already added shows alongside it. context is optional throughout: the eval sweeps and unit tests call the extract* functions with the two-arg (no-context) signature and fall back to the self-contained stub, so behavior there is unchanged. All side effects are preserved, so the form-pipeline tests that pin processFormS1/ProcessAccessionDocFormTask stay green. Adds a regression test asserting a threaded context receives the phase progress. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011HoQKRtpU7mGpK8Dah9ySm --- src/sec/forms/miscellaneous-filings/loi8k.ts | 2 +- .../miscellaneous-filings/redemption8k.ts | 2 +- .../Form_DEFM14A.storage.ts | 2 +- .../Form_424.storage.ts | 1 + .../Form_S_1.storage.ts | 13 ++- .../s1/offeringSections.ts | 13 ++- .../s1/sectionExtractors.test.ts | 45 ++++++++ .../s1/sectionExtractors.ts | 108 ++++++++++++------ 8 files changed, 138 insertions(+), 48 deletions(-) diff --git a/src/sec/forms/miscellaneous-filings/loi8k.ts b/src/sec/forms/miscellaneous-filings/loi8k.ts index d6e1cc9e..6c7612cf 100644 --- a/src/sec/forms/miscellaneous-filings/loi8k.ts +++ b/src/sec/forms/miscellaneous-filings/loi8k.ts @@ -215,7 +215,7 @@ export async function processLoi8K(args: ProcessLoi8KArgs): Promise { verifyRow: (t, r) => verifyRowSpan(t, r.source_span), unverifiedAllDetail: "LOI source_span not present in narrative text", extract: async (t) => { - const row = await extractLoi(t, model); + const row = await extractLoi(t, model, args.context); return row === null ? [] : [row]; }, persist: async (rows) => { diff --git a/src/sec/forms/miscellaneous-filings/redemption8k.ts b/src/sec/forms/miscellaneous-filings/redemption8k.ts index ff940fdd..6db6e251 100644 --- a/src/sec/forms/miscellaneous-filings/redemption8k.ts +++ b/src/sec/forms/miscellaneous-filings/redemption8k.ts @@ -229,7 +229,7 @@ export async function processRedemption8K(args: ProcessRedemption8KArgs): Promis verifyRow: (t, r) => verifyRowSpan(t, r.source_span), unverifiedAllDetail: "redemption source_span not present in narrative text", extract: async (t) => { - const row = await extractRedemption(t, model); + const row = await extractRedemption(t, model, args.context); return row === null ? [] : [row]; }, persist: async (rows) => { diff --git a/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts b/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts index fe01d003..287428e3 100644 --- a/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts +++ b/src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts @@ -186,7 +186,7 @@ export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise< verifyRow: (text, r) => verifyRowSpan(text, r.source_span), unverifiedAllDetail: "merger deal source_span not present in section text", extract: async (text) => { - const deal = await extractMergerDeal(text, model); + const deal = await extractMergerDeal(text, model, args.context); return deal === null ? [] : [deal]; }, persist: async (rows) => { diff --git a/src/sec/forms/registration-statements/Form_424.storage.ts b/src/sec/forms/registration-statements/Form_424.storage.ts index b4b9b546..623d2648 100644 --- a/src/sec/forms/registration-statements/Form_424.storage.ts +++ b/src/sec/forms/registration-statements/Form_424.storage.ts @@ -207,6 +207,7 @@ export async function processForm424(args: ProcessForm424Args): Promise { model_id, activeUnderwriterFamilyVersion, byName, + context: args.context, }); await recordSpacIpoEventIfEligible(); diff --git a/src/sec/forms/registration-statements/Form_S_1.storage.ts b/src/sec/forms/registration-statements/Form_S_1.storage.ts index adac66d5..be0ea9f4 100644 --- a/src/sec/forms/registration-statements/Form_S_1.storage.ts +++ b/src/sec/forms/registration-statements/Form_S_1.storage.ts @@ -336,7 +336,7 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { unverifiedAllDetail: "the confident SPAC classification had source_span not present in section text", extract: async (text) => { - const c = await extractSpacClassification(text, classifierModelResolved); + const c = await extractSpacClassification(text, classifierModelResolved, args.context); return c === null ? [] : [c]; }, persist: async () => { @@ -395,7 +395,7 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { verifyRow: (text, r) => verifyRowSpan(text, r.source_span), unverifiedAllDetail: "the confident SPAC profile had source_span not present in section text", extract: async (text) => { - const p = await extractSpacProfile(text, model); + const p = await extractSpacProfile(text, model, args.context); return p === null ? [] : [p]; }, persist: async (rows) => { @@ -435,7 +435,7 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { "all $T confident management rows had source_span not present in section text", unverifiedPartialDetail: "$N of $T confident management rows had source_span not present in section text", - extract: (text) => extractManagement(text, model), + extract: (text) => extractManagement(text, model, args.context), persist: async (rows) => { for (const r of rows) { const name = splitPersonName(r.full_name); @@ -481,7 +481,7 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { "all $T confident ownership rows had source_span not present in section text", unverifiedPartialDetail: "$N of $T confident ownership rows had source_span not present in section text", - extract: (text) => extractBeneficialOwnership(text, model), + extract: (text) => extractBeneficialOwnership(text, model, args.context), persist: async (rows) => { for (const r of rows) { const observation_index = idx++; @@ -548,7 +548,7 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { "all $T confident related-party rows had source_span not present in section text", unverifiedPartialDetail: "$N of $T confident related-party rows had source_span not present in section text", - extract: (text) => extractRelatedParty(text, model), + extract: (text) => extractRelatedParty(text, model, args.context), persist: async (rows) => { let txIndex = 0; for (const r of rows) { @@ -619,6 +619,7 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { model_id, activeUnderwriterFamilyVersion, byName, + context: args.context, }); // --- SPAC sponsors (gated on deterministic classification) --- @@ -649,7 +650,7 @@ export async function processFormS1(args: ProcessFormS1Args): Promise { "all $T confident sponsor rows had source_span not present in section text", unverifiedPartialDetail: "$N of $T confident sponsor rows had source_span not present in section text", - extract: (text) => extractSpacSponsors(text, model), + extract: (text) => extractSpacSponsors(text, model, args.context), persist: async (rows) => { let wrote = 0; for (const r of rows) { diff --git a/src/sec/forms/registration-statements/s1/offeringSections.ts b/src/sec/forms/registration-statements/s1/offeringSections.ts index 7b11b700..658440e0 100644 --- a/src/sec/forms/registration-statements/s1/offeringSections.ts +++ b/src/sec/forms/registration-statements/s1/offeringSections.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type { ModelConfig } from "workglow"; +import type { IExecuteContext, ModelConfig } from "workglow"; import type { EntityObserver } from "../../../../resolver/EntityObserver"; import { UnderwriterFamilyResolver } from "../../../../resolver/UnderwriterFamilyResolver"; import { CanonicalUnderwriterFamilyRepo } from "../../../../storage/canonical/CanonicalUnderwriterFamilyRepo"; @@ -73,6 +73,8 @@ export interface OfferingSectionsArgs { readonly model_id: string | null; readonly activeUnderwriterFamilyVersion: string; readonly byName: ReadonlyMap; + /** Running task context, threaded to the generation calls for CLI progress. */ + readonly context?: IExecuteContext; } /** @@ -98,6 +100,7 @@ export async function runOfferingSections(args: OfferingSectionsArgs): Promise { - const terms = await extractOfferingTerms(text, model); + const terms = await extractOfferingTerms(text, model, context); return terms === null ? [] : [terms]; }, persist: async (rows) => { @@ -233,7 +236,7 @@ export async function runOfferingSections(args: OfferingSectionsArgs): Promise { - const promote = await extractSponsorPromote(text, model); + const promote = await extractSponsorPromote(text, model, context); return promote === null ? [] : [promote]; }, persist: async (rows) => { @@ -271,7 +274,7 @@ export async function runOfferingSections(args: OfferingSectionsArgs): Promise extractUnderwriters(text, model), + extract: (text) => extractUnderwriters(text, model, context), persist: async (rows) => { let wrote = 0; for (const r of rows) { @@ -333,7 +336,7 @@ export async function runOfferingSections(args: OfferingSectionsArgs): Promise extractUseOfProceeds(text, model), + extract: (text) => extractUseOfProceeds(text, model, context), persist: async (rows) => { const now = new Date().toISOString(); let lineIndex = 0; diff --git a/src/sec/forms/registration-statements/s1/sectionExtractors.test.ts b/src/sec/forms/registration-statements/s1/sectionExtractors.test.ts index f1c05511..0212868e 100644 --- a/src/sec/forms/registration-statements/s1/sectionExtractors.test.ts +++ b/src/sec/forms/registration-statements/s1/sectionExtractors.test.ts @@ -296,3 +296,48 @@ it("extractUseOfProceeds returns parsed line items", async () => { unregister(); } }); + +it("forwards generation phase progress to a threaded execute context", async () => { + // Regression guard: StructuredGenerationTask.execute() keeps only the finish + // event and drops the phase events, so runStructured drives executeStream and + // forwards phases itself. Without that, a threaded context sees no progress and + // the CLI task row stays silent through every section. + const { unregister } = registerFakeStructuredProvider([ + { + people: [ + { + full_name: "Jane Doe", + titles: ["Chief Executive Officer"], + relationship: null, + age: null, + bio: null, + confidence: 0.9, + source_span: "Jane Doe", + }, + ], + }, + ]); + const messages: Array = []; + const context = { + signal: new AbortController().signal, + updateProgress: async (_p: number | undefined, m?: string) => { + messages.push(m); + }, + own: (v: T): T => v, + registry: { has: () => false, get: () => { throw new Error("x"); } }, + resourceScope: { register: () => {}, dispose: async () => {} }, + } as any; + try { + const rows = await extractManagement( + "MANAGEMENT\n\nJane Doe has served as our Chief Executive Officer.", + fakeS1Model(), + context + ); + expect(rows).toHaveLength(1); + // The generation task's phase labels reached the threaded context's row. + expect(messages).toContain("Preparing"); + expect(messages).toContain("Generating"); + } finally { + unregister(); + } +}); diff --git a/src/sec/forms/registration-statements/s1/sectionExtractors.ts b/src/sec/forms/registration-statements/s1/sectionExtractors.ts index 10a6de2f..c67ee923 100644 --- a/src/sec/forms/registration-statements/s1/sectionExtractors.ts +++ b/src/sec/forms/registration-statements/s1/sectionExtractors.ts @@ -356,15 +356,18 @@ export function requireNonEmptyGrammarArrays(schema: object): object { async function runStructured( model: ModelConfig, prompt: string, - outputSchema: object + outputSchema: object, + callerContext?: IExecuteContext ): Promise> { - const context = makeExecuteContext(); + // The running task's context, when the form pipeline threads one down, so the + // generation task's `Preparing`/`Generating` phase events (and any download) + // render on that task's row in the CLI UI. Absent (eval sweeps, unit tests), a + // throwaway stub keeps the one-shot call self-contained. + const context = callerContext ?? makeExecuteContext(); // Correctness safety-net: local providers (GGUF especially) must have their // weights on disk before generation — cloud models no-op here. Memoized, so the // per-section sweep pays the download once; a form/eval run that prefetched with - // a real context (for visible progress) already satisfied this. This call uses - // the bare structured-generation context, so it downloads silently if it ever - // has to — the progress-bearing prefetch lives at the CLI-task boundary. + // a real context (for visible progress) already satisfied this. await ensureModelDownloaded(model, context); const grammarConstrained = (model as { provider?: string }).provider === "LOCAL_LLAMACPP"; const input = { @@ -375,7 +378,20 @@ async function runStructured( maxRetries: 1, }; const task = new StructuredGenerationTask({ defaults: input } as any); - const result = await task.execute(input as any, context); + // Drive the task's own stream rather than `execute()`: `execute()` keeps only + // the `finish` event and drops the `phase` events, so its per-section + // `Preparing`/`Generating` progress never reaches the caller's row. Consuming + // `executeStream` (identical result — the same retry/validation loop, ending in + // the same finish) and forwarding phases to `context.updateProgress` makes the + // active section visible in the CLI task UI (a no-op against the stub context). + let result: { object?: unknown } | undefined; + for await (const event of task.executeStream(input as any, context)) { + if (event.type === "phase") { + void context.updateProgress(event.progress, event.message); + } else if (event.type === "finish") { + result = (event as { data?: { object?: unknown } }).data; + } + } return (result?.object as Record | undefined) ?? {}; } @@ -420,7 +436,8 @@ async function runGuardedExtraction( model: ModelConfig, instructions: string, sectionText: string, - outputSchema: object + outputSchema: object, + context?: IExecuteContext ): Promise> { const local = isLocalProvider(model); const { wrapped, nonce } = wrapUntrusted(sectionText); @@ -429,7 +446,8 @@ async function runGuardedExtraction( const obj = await runStructured( model, prompt, - local ? stripNonceSeen(outputSchema) : outputSchema + local ? stripNonceSeen(outputSchema) : outputSchema, + context ); if (!local) verifyNonce(obj, nonce); return obj; @@ -437,7 +455,8 @@ async function runGuardedExtraction( export async function extractManagement( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "Extract every director and executive officer named in the S-1 MANAGEMENT section " + @@ -475,7 +494,7 @@ export async function extractManagement( "For example 'member of our board of directors' -> ['Director'] and 'Chairman of our " + "board of directors' -> ['Chairman of the Board of Directors']. " + "Return JSON matching the schema."; - const obj = await runGuardedExtraction(model, instructions, sectionText, ManagementOutputSchema); + const obj = await runGuardedExtraction(model, instructions, sectionText, ManagementOutputSchema, context); const people = (obj.people as ManagementPersonRow[] | undefined) ?? []; // Post-model canonicalization: split compound titles and canonicalize each // role, so the stored roles are consistent regardless of which model produced @@ -504,7 +523,8 @@ export function isOwnershipGroupSubtotal(name: string | null | undefined): boole export async function extractBeneficialOwnership( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "Extract every beneficial owner from the S-1 Principal and Selling Stockholders " + @@ -524,7 +544,8 @@ export async function extractBeneficialOwnership( model, instructions, sectionText, - BeneficialOwnershipOutputSchema + BeneficialOwnershipOutputSchema, + context ); const owners = (obj.owners as BeneficialOwnerRow[] | undefined) ?? []; // Enforce the subtotal exclusion rather than trusting the prompt: a leaked row @@ -534,7 +555,8 @@ export async function extractBeneficialOwnership( export async function extractRelatedParty( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "Extract related parties and their transactions from the S-1 Certain Relationships " + @@ -546,14 +568,16 @@ export async function extractRelatedParty( model, instructions, sectionText, - RelatedPartyOutputSchema + RelatedPartyOutputSchema, + context ); return (obj.parties as RelatedPartyRow[] | undefined) ?? []; } export async function extractOfferingTerms( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "Extract the offering terms from the S-1/F-1 'The Offering' and 'Underwriting' text " + @@ -569,7 +593,8 @@ export async function extractOfferingTerms( model, instructions, sectionText, - OfferingTermsOutputSchema + OfferingTermsOutputSchema, + context ); if (obj.confidence == null || obj.source_span == null) return null; return obj as unknown as OfferingTermsRow; @@ -584,7 +609,8 @@ export async function extractOfferingTerms( */ export async function extractSponsorPromote( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "The text between the tags below is from a SPAC (blank-check) prospectus. Extract the " + @@ -605,7 +631,8 @@ export async function extractSponsorPromote( model, instructions, sectionText, - SponsorPromoteOutputSchema + SponsorPromoteOutputSchema, + context ); if (obj.confidence == null || obj.source_span == null) return null; return { @@ -623,7 +650,8 @@ export async function extractSponsorPromote( export async function extractUnderwriters( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "Extract every underwriter named in the S-1/F-1 Underwriting (or Plan of " + @@ -634,13 +662,14 @@ export async function extractUnderwriters( "'underwriter'; null if unclear), shares_allocated (the number of shares " + "underwritten, or null), over_allotment_shares (or null), a confidence in [0,1], " + "and the verbatim source_span. Return JSON matching the schema."; - const obj = await runGuardedExtraction(model, instructions, sectionText, UnderwriterOutputSchema); + const obj = await runGuardedExtraction(model, instructions, sectionText, UnderwriterOutputSchema, context); return (obj.underwriters as UnderwriterRowOut[] | undefined) ?? []; } export async function extractSpacSponsors( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "The text between the tags below is from a SPAC (blank-check) registration " + @@ -648,7 +677,7 @@ export async function extractSpacSponsors( "legal entity, e.g. 'Acme Sponsor 2, LLC'), common_name (the sponsor brand/family " + "without the legal suffix or series number, e.g. 'Acme Sponsor'), a confidence in " + "[0,1], and the verbatim source_span. Return JSON matching the schema."; - const obj = await runGuardedExtraction(model, instructions, sectionText, SpacSponsorOutputSchema); + const obj = await runGuardedExtraction(model, instructions, sectionText, SpacSponsorOutputSchema, context); return (obj.sponsors as SpacSponsorRow[] | undefined) ?? []; } @@ -661,7 +690,8 @@ export async function extractSpacSponsors( */ export async function extractSpacProfile( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "The text between the tags below is from a SPAC (blank-check) registration " + @@ -677,7 +707,7 @@ export async function extractSpacProfile( "management/sponsor team's background and experience (or null). Give url_spac: the " + "SPAC's website URL if stated (or null). Give a confidence in [0,1] and the verbatim " + "source_span you drew the focus/description from. Return JSON matching the schema."; - const obj = await runGuardedExtraction(model, instructions, sectionText, SpacProfileOutputSchema); + const obj = await runGuardedExtraction(model, instructions, sectionText, SpacProfileOutputSchema, context); if (obj.confidence == null || obj.source_span == null) return null; return { focus: Array.isArray(obj.focus) ? (obj.focus as string[]) : [], @@ -701,7 +731,8 @@ export async function extractSpacProfile( */ export async function extractSpacClassification( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "The text between the tags below is prose from a company's SEC registration " + @@ -720,7 +751,8 @@ export async function extractSpacClassification( model, instructions, sectionText, - SpacClassificationOutputSchema + SpacClassificationOutputSchema, + context ); if (obj.is_spac !== true) return null; const kind = obj.entity_kind; @@ -743,7 +775,8 @@ export async function extractSpacClassification( export async function extractMergerDeal( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "The text between the tags below is from a SPAC merger proxy (DEFM14A/PREM14A). " + @@ -754,14 +787,15 @@ export async function extractMergerDeal( "describing the consideration — e.g. cash, stock, exchange ratio — or null), a " + "confidence in [0,1], and the verbatim source_span you drew the target from. " + "Return JSON matching the schema."; - const obj = await runGuardedExtraction(model, instructions, sectionText, MergerDealOutputSchema); + const obj = await runGuardedExtraction(model, instructions, sectionText, MergerDealOutputSchema, context); if (obj.confidence == null || obj.source_span == null) return null; return obj as unknown as MergerDealRow; } export async function extractUseOfProceeds( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "Extract the use-of-proceeds line items from the S-1/F-1 Use of Proceeds section " + @@ -772,7 +806,8 @@ export async function extractUseOfProceeds( model, instructions, sectionText, - UseOfProceedsOutputSchema + UseOfProceedsOutputSchema, + context ); return (obj.line_items as UseOfProceedsLineRow[] | undefined) ?? []; } @@ -784,7 +819,8 @@ export async function extractUseOfProceeds( */ export async function extractRedemption( sectionText: string, - model: ModelConfig + model: ModelConfig, + context?: IExecuteContext ): Promise { const instructions = "From the SEC 8-K text below, extract the REALIZED redemption of public " + @@ -792,7 +828,7 @@ export async function extractRedemption( "only figures explicitly stated — do NOT multiply shares by price to " + "synthesize an amount. If the text does not report realized redemptions, " + "return confidence 0 and null fields."; - const obj = await runGuardedExtraction(model, instructions, sectionText, RedemptionOutputSchema); + const obj = await runGuardedExtraction(model, instructions, sectionText, RedemptionOutputSchema, context); if (obj.confidence == null || obj.source_span == null) return null; // A "no realized redemption" response carries neither figure — not a redemption. if (obj.redemption_shares == null && obj.redemption_amount == null) return null; @@ -806,7 +842,11 @@ export async function extractRedemption( * (e.g. it announces a definitive agreement instead). Mirrors * {@link extractRedemption}. */ -export async function extractLoi(sectionText: string, model: ModelConfig): Promise { +export async function extractLoi( + sectionText: string, + model: ModelConfig, + context?: IExecuteContext +): Promise { const instructions = "From the SEC 8-K text below, determine whether it reports that the company " + "ENTERED INTO a NON-BINDING letter of intent (LOI), agreement in principle, or " + @@ -819,7 +859,7 @@ export async function extractLoi(sectionText: string, model: ModelConfig): Promi "confidence in [0,1], and the verbatim source_span you drew the determination " + "from. If the text reports no LOI, return is_loi false with confidence for that " + "determination and a null source_span."; - const obj = await runGuardedExtraction(model, instructions, sectionText, LoiOutputSchema); + const obj = await runGuardedExtraction(model, instructions, sectionText, LoiOutputSchema, context); if (obj.is_loi !== true) return null; // As in extractSpacClassification: a positive with no confidence/span is not the // auto-resolved "no LOI" negative, so surface it rather than dropping it. From d5abde96ba747bdc568e8cae409811c551a91039 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 06:22:15 +0000 Subject: [PATCH 4/4] refactor(models): drive model tasks through run(); address PR review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the task run() lifecycle instead of hand-driving execute()/executeStream for the three model tasks: - runStructured runs StructuredGenerationTask.run() and forwards its Preparing/Generating phases to the caller's context.updateProgress via the runConfig.updateProgress hook (caching disabled to match execute()'s never-cache semantics). run() also fails loudly — a generation that can't finish throws rather than returning {}, so the prior executeStream finish-guard concern no longer applies. - ensureModelDownloaded runs ModelDownloadTask.run(); unloadLocalModel runs ModelDownloadRemoveTask.run() (dropping its throwaway stub context). Address the PR review: - Memoize ensureModelDownloaded on a robust key (model_id ?? model ?? provider_config.model_url ?? model_path), mirroring resolveModelId, so a record without model_id still downloads once instead of every call. - isRemoteGgufUri matches hf: case-insensitively. - ggufCacheFileName folds the whole source path into the cache filename so distinct remote GGUF sources (same repo name / different org, or two repos each holding model.gguf) don't collide on disk. Adds tests for the model-identified-by-`model` memo path, uppercase HF:, and same-repo-name/different-org cache-target uniqueness. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011HoQKRtpU7mGpK8Dah9ySm --- src/config/ensureModelDownloaded.test.ts | 27 ++++++++++++++ src/config/ensureModelDownloaded.ts | 35 +++++++++++++++++-- src/config/registerModels.test.ts | 25 ++++++++++--- src/config/registerModels.ts | 19 ++++++---- src/eval/unloadModel.ts | 20 ++--------- .../s1/sectionExtractors.ts | 26 +++++++------- 6 files changed, 107 insertions(+), 45 deletions(-) diff --git a/src/config/ensureModelDownloaded.test.ts b/src/config/ensureModelDownloaded.test.ts index 9286e734..29fd04ce 100644 --- a/src/config/ensureModelDownloaded.test.ts +++ b/src/config/ensureModelDownloaded.test.ts @@ -152,5 +152,32 @@ describe("ensureModelDownloaded", () => { await ensureModelDownloaded(model, context); expect(runFnCalls).toBe(1); }); + + it("memoizes a model identified by `model` (no model_id)", async () => { + let runFnCalls = 0; + getAiProviderRegistry().registerRunFn("HF_TRANSFORMERS_ONNX", { + serves: ["model.download"], + runFn: async (input: any, _m: any, _s: any, emit: any) => { + runFnCalls += 1; + emit({ type: "finish", data: { model: input.model } }); + }, + } as any); + const context = { + signal: new AbortController().signal, + updateProgress: async () => {}, + own: (v: T): T => v, + registry: { has: () => false, get: () => { throw new Error("x"); } }, + resourceScope: { register: () => {}, dispose: async () => {} }, + } as unknown as IExecuteContext; + // No model_id — identity comes from `model` (mirrors resolveModelId's fallback). + const model = { + model: "onnx-community/no-id", + provider: "HF_TRANSFORMERS_ONNX", + provider_config: { model_path: "onnx-community/no-id" }, + } as unknown as ModelConfig; + await ensureModelDownloaded(model, context); + await ensureModelDownloaded(model, context); + expect(runFnCalls).toBe(1); + }); }); }); diff --git a/src/config/ensureModelDownloaded.ts b/src/config/ensureModelDownloaded.ts index 05d5e0c1..cf85bf95 100644 --- a/src/config/ensureModelDownloaded.ts +++ b/src/config/ensureModelDownloaded.ts @@ -30,6 +30,30 @@ export function resetEnsuredModelsForTesting(): void { ensured.clear(); } +/** + * Stable per-model memo key. Mirrors `resolveModelId` (`model_id ?? model`) — not + * every ModelConfig carries `model_id` (some identify via `model`) — and falls + * back to the provider_config download/load target so a record with neither still + * memoizes on something unique (`model_url` for a remote GGUF, `model_path` for a + * local one / HFT repo id). Empty only for a truly anonymous record, which then + * downloads every call rather than being wrongly deduped against another. + */ +function modelKey(model: ModelConfig): string { + const ref = model as { + model_id?: unknown; + model?: unknown; + provider_config?: { model_url?: unknown; model_path?: unknown }; + }; + const candidates = [ + ref.model_id, + ref.model, + ref.provider_config?.model_url, + ref.provider_config?.model_path, + ]; + const found = candidates.find((c): c is string => typeof c === "string" && c.length > 0); + return found ?? ""; +} + /** * Ensure a model's weights are present locally before it is used for generation. * @@ -60,7 +84,7 @@ export async function ensureModelDownloaded( const provider = (model as { provider?: string }).provider; if (!provider || !DOWNLOADABLE_PROVIDERS.has(provider)) return; - const modelId = (model as { model_id?: string }).model_id ?? ""; + const modelId = modelKey(model); if (modelId && ensured.has(modelId)) return; if (provider === LLAMACPP_PROVIDER) { @@ -74,7 +98,14 @@ export async function ensureModelDownloaded( const input = { model }; const task = new ModelDownloadTask({ defaults: input } as any); - await task.execute(input as any, context); + // Drive the download through its `run()` lifecycle. `run` routes the download + // run-fn's `phase` events to `config.updateProgress`, which we forward to the + // caller's `context.updateProgress` so a multi-GB fetch renders a live + // percentage in the CLI task UI; `signal` aborts it on Ctrl-C. + await task.run(input as any, { + updateProgress: (_t, progress, message) => context.updateProgress(progress, message), + signal: context.signal, + }); if (modelId) ensured.add(modelId); } diff --git a/src/config/registerModels.test.ts b/src/config/registerModels.test.ts index fb190f7e..a6db9d5b 100644 --- a/src/config/registerModels.test.ts +++ b/src/config/registerModels.test.ts @@ -144,22 +144,39 @@ describe("registerSecModels", () => { expect(config.model_url).toBeUndefined(); }); - it("turns an hf: URI into a download source + cache target", () => { + it("turns an hf: URI into a download source + cache target (full path, collision-safe)", () => { const config = llamaCppModelRecord( "gguf:hf:bartowski/SmolLM2-135M-Instruct-GGUF:Q4_K_M" ).provider_config; expect(config.model_url).toBe("hf:bartowski/SmolLM2-135M-Instruct-GGUF:Q4_K_M"); expect(config.models_dir).toBe("/models/gguf"); - expect(config.model_path).toBe("/models/gguf/SmolLM2-135M-Instruct-GGUF-Q4_K_M.gguf"); + expect(config.model_path).toBe( + "/models/gguf/bartowski-SmolLM2-135M-Instruct-GGUF-Q4_K_M.gguf" + ); }); - it("turns an https URL into a download source + cache target", () => { + it("classifies an uppercase HF: URI as remote (case-insensitive)", () => { + const config = llamaCppModelRecord("gguf:HF:org/repo:Q4").provider_config; + expect(config.model_url).toBe("HF:org/repo:Q4"); + expect(config.models_dir).toBe("/models/gguf"); + expect(config.model_path).toBe("/models/gguf/org-repo-Q4.gguf"); + }); + + it("derives distinct cache targets for same-repo-name different-org URIs", () => { + const a = llamaCppModelRecord("gguf:hf:org1/repo:Q4").provider_config.model_path; + const b = llamaCppModelRecord("gguf:hf:org2/repo:Q4").provider_config.model_path; + expect(a).not.toBe(b); + expect(a).toBe("/models/gguf/org1-repo-Q4.gguf"); + expect(b).toBe("/models/gguf/org2-repo-Q4.gguf"); + }); + + it("turns an https URL into a download source + cache target (full path, collision-safe)", () => { const config = llamaCppModelRecord( "gguf:https://host.example/a/b/model.gguf" ).provider_config; expect(config.model_url).toBe("https://host.example/a/b/model.gguf"); expect(config.models_dir).toBe("/models/gguf"); - expect(config.model_path).toBe("/models/gguf/model.gguf"); + expect(config.model_path).toBe("/models/gguf/host.example-a-b-model.gguf"); }); }); diff --git a/src/config/registerModels.ts b/src/config/registerModels.ts index fefdf101..92376984 100644 --- a/src/config/registerModels.ts +++ b/src/config/registerModels.ts @@ -300,7 +300,7 @@ const GGUF_GPU_LAYERS_ALL = 999; /** A `gguf:` remainder that names a remote source the download harness can fetch. */ function isRemoteGgufUri(rawPath: string): boolean { - return rawPath.startsWith("hf:") || /^https?:\/\//i.test(rawPath); + return /^hf:/i.test(rawPath) || /^https?:\/\//i.test(rawPath); } /** @@ -308,10 +308,13 @@ function isRemoteGgufUri(rawPath: string): boolean { * is only a fallback for the required `model_path` field: once the download runs, * the provider keys on `model_url` and resolves the real on-disk path itself, so * the exact name here does not affect which file generation loads — it just gives - * the cache target a stable, human-legible name. + * the cache target a stable, human-legible name. The whole path (not just the last + * segment) is folded into the name so distinct sources don't collide on disk + * (`hf:org1/repo:Q4` vs `hf:org2/repo:Q4`; two repos each holding `model.gguf`). * - * `hf:org/repo:Q4_K_M` → `repo-Q4_K_M.gguf`; `hf:org/repo/file.gguf` → `file.gguf`; - * `https://host/a/b/model.gguf` → `model.gguf`. + * `hf:org/repo:Q4_K_M` → `org-repo-Q4_K_M.gguf`; + * `hf:org/repo/file.gguf` → `org-repo-file.gguf`; + * `https://host/a/b/model.gguf` → `host-a-b-model.gguf`. */ function ggufCacheFileName(uri: string): string { let rest = uri.replace(/^https?:\/\//i, "").replace(/^hf:/i, ""); @@ -322,9 +325,11 @@ function ggufCacheFileName(uri: string): string { quant = quantMatch[1]; rest = rest.slice(0, rest.length - quant.length - 1); } - const last = rest.split(/[/?#]/).filter(Boolean).pop() ?? "model"; - const base = last.toLowerCase().endsWith(".gguf") ? last.slice(0, -".gguf".length) : last; - const name = [base, quant].filter(Boolean).join("-") || "model"; + const stripped = rest.replace(/\.gguf$/i, ""); + // Fold every path/query separator (and any other unsafe char) into `-` so the + // full source path is preserved as one filesystem-safe slug. + const slug = stripped.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, ""); + const name = [slug || "model", quant].filter(Boolean).join("-"); return `${name}.gguf`; } diff --git a/src/eval/unloadModel.ts b/src/eval/unloadModel.ts index d51acf4b..3394bdee 100644 --- a/src/eval/unloadModel.ts +++ b/src/eval/unloadModel.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type { IExecuteContext, ModelConfig } from "workglow"; +import type { ModelConfig } from "workglow"; import { ModelDownloadRemoveTask } from "workglow"; /** @@ -17,22 +17,6 @@ import { ModelDownloadRemoveTask } from "workglow"; */ const UNLOADABLE_PROVIDERS = new Set(["LOCAL_LLAMACPP"]); -/** Minimal execution context for driving a task's `execute` outside a task-graph run. */ -function makeStubContext(): IExecuteContext { - return { - signal: new AbortController().signal, - updateProgress: async () => {}, - own: (value: T): T => value, - registry: { - has: () => false, - get: () => { - throw new Error("not registered"); - }, - } as any, - resourceScope: { register: () => {}, dispose: async () => {} } as any, - } as IExecuteContext; -} - /** * Release a worker-backed local model's in-memory weights and context so a * multi-model sweep doesn't accumulate VRAM/RAM across candidates — each gets a @@ -48,7 +32,7 @@ export async function unloadLocalModel(model: ModelConfig): Promise { try { const input = { model }; const task = new ModelDownloadRemoveTask({ defaults: input } as any); - await task.execute(input as any, makeStubContext()); + await task.run(input as any); } catch { // Leave it resident if unload isn't supported / fails; auto-evict covers correctness. } diff --git a/src/sec/forms/registration-statements/s1/sectionExtractors.ts b/src/sec/forms/registration-statements/s1/sectionExtractors.ts index c67ee923..36beae38 100644 --- a/src/sec/forms/registration-statements/s1/sectionExtractors.ts +++ b/src/sec/forms/registration-statements/s1/sectionExtractors.ts @@ -378,20 +378,18 @@ async function runStructured( maxRetries: 1, }; const task = new StructuredGenerationTask({ defaults: input } as any); - // Drive the task's own stream rather than `execute()`: `execute()` keeps only - // the `finish` event and drops the `phase` events, so its per-section - // `Preparing`/`Generating` progress never reaches the caller's row. Consuming - // `executeStream` (identical result — the same retry/validation loop, ending in - // the same finish) and forwarding phases to `context.updateProgress` makes the - // active section visible in the CLI task UI (a no-op against the stub context). - let result: { object?: unknown } | undefined; - for await (const event of task.executeStream(input as any, context)) { - if (event.type === "phase") { - void context.updateProgress(event.progress, event.message); - } else if (event.type === "finish") { - result = (event as { data?: { object?: unknown } }).data; - } - } + // Drive the task through its `run()` lifecycle (not a bare `execute()` with a + // throwaway context): `run` routes the task's `Preparing`/`Generating` phase + // events to `config.updateProgress`, which we forward to the caller's + // `context.updateProgress` so the active section shows on that task's CLI row + // (a no-op against the stub context). `signal` propagates Ctrl-C. Caching is + // off — a fresh per-call nonce already makes cloud prompts unique, and matching + // `execute()`'s never-cache semantics keeps replays side-effect-identical. + const result = (await task.run(input as any, { + updateProgress: (_t, progress, message) => context.updateProgress(progress, message), + signal: context.signal, + cacheable: false, + })) as { object?: unknown } | undefined; return (result?.object as Record | undefined) ?? {}; }