diff --git a/electron/main.ts b/electron/main.ts index 5c1388407..785527065 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -23,7 +23,7 @@ import { import { mainT, setMainLocale } from "./i18n"; import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers"; import { installMainProcessErrorGuards } from "./main-process-errors"; -import { registerSttIpc } from "./stt"; +import { registerSttIpc, shutdownStt } from "./stt"; import { createCountdownOverlayWindow, createEditorWindow, @@ -512,6 +512,28 @@ app.on("activate", () => { } }); +let sttShutdownPromise: Promise | null = null; +let sttShutdownFinished = false; + +// Electron does not wait for an async event listener. Hold the first quit long +// enough to terminate the long-lived Whisper helper, then re-enter app.quit() +// with a guard so the second before-quit event can proceed normally. Without +// this, a normal Cmd+Q orphaned the helper under launchd with the model and GPU +// resources still resident after every OpenScreen window had gone away. +app.on("before-quit", (event) => { + if (sttShutdownFinished) return; + event.preventDefault(); + if (sttShutdownPromise) return; + sttShutdownPromise = shutdownStt() + .catch((error) => { + console.error("[stt] Failed to stop whisper helper during app quit:", error); + }) + .finally(() => { + sttShutdownFinished = true; + app.quit(); + }); +}); + app.on("will-quit", () => { unregisterAllGlobalShortcuts(); }); diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cpp index 339f759b1..9f6abab6e 100644 --- a/electron/native/whisper-stt/src/main.cpp +++ b/electron/native/whisper-stt/src/main.cpp @@ -224,6 +224,7 @@ int main(int argc, char** argv) { std::string model_path; std::string host = "127.0.0.1"; bool host_from_flag = false; + bool force_cpu = false; int port = 0; int threads = std::max(1u, std::thread::hardware_concurrency()); @@ -233,6 +234,7 @@ int main(int argc, char** argv) { else if (a == "--host" && i + 1 < argc) { host = argv[++i]; host_from_flag = true; } else if (a == "--port" && i + 1 < argc) port = std::atoi(argv[++i]); else if (a == "--threads" && i + 1 < argc) threads = std::atoi(argv[++i]); + else if (a == "--cpu") force_cpu = true; } // ponytail: prefer env var (matches the prior native STT model env var // shape; the Node wrapper passes both ways). @@ -268,20 +270,28 @@ int main(int argc, char** argv) { // ---- Init whisper context with DTW alignment (POC §4.1) ---- whisper_context_params cparams = whisper_context_default_params(); - cparams.use_gpu = true; // GPU offload via whatever backend was linked + cparams.use_gpu = !force_cpu; // Parent retries with --cpu if a GPU backend aborts. cparams.flash_attn = false; // CRITICAL: DTW is silently disabled by v1.9.1 // if flash_attn is true; the guardrail in the // /inference handler still runs, but skipping // the request is wasted work. cparams.dtw_token_timestamps = true; - cparams.dtw_aheads_preset = WHISPER_AHEADS_SMALL; + cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3_TURBO; whisper_context* ctx = whisper_init_from_file_with_params(model_path.c_str(), cparams); + if (!ctx && cparams.use_gpu) { + // Metal/Vulkan allocation can fail transiently when the editor or another + // creative app is already using most GPU memory. Captions should degrade to + // a slower CPU run instead of leaving the HTTP readiness probe to time out. + log("GPU model initialization failed; retrying with CPU inference"); + cparams.use_gpu = false; + ctx = whisper_init_from_file_with_params(model_path.c_str(), cparams); + } if (!ctx) { log("whisper_init_from_file_with_params failed for " + model_path); return 3; } - const std::string active_backend = detect_active_backend(); - log("model loaded: multilingual small; backend=" + active_backend); + const std::string active_backend = cparams.use_gpu ? detect_active_backend() : "whispercpp-cpu"; + log("model loaded; backend=" + active_backend); // ---- HTTP server ---- httplib::Server svr; diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index ad0eaf976..f06da2cd8 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { planChunks } from "./chunking"; -import { _resetSttManagerForTests, SttManager } from "./index"; +import { _resetSttManagerForTests, getSttManager, SttManager, shutdownStt } from "./index"; import type { SttStatusEvent, SttTranscribeResponse } from "./transcriptionContract"; // We swap the long-lived modules for fakes so the manager's `init()` and @@ -18,6 +18,7 @@ const fakeWhisperServer = { }, transcribe: vi.fn(), stop: vi.fn(), + shutdown: vi.fn(), }; vi.mock("./whisperServer", () => { @@ -26,6 +27,7 @@ vi.mock("./whisperServer", () => { status = fakeWhisperServer.status; transcribe = fakeWhisperServer.transcribe; stop = fakeWhisperServer.stop; + shutdown = fakeWhisperServer.shutdown; } return { WhisperServerManager: FakeWhisperServerManager }; }); @@ -52,6 +54,7 @@ describe("SttManager", () => { fakeWhisperServer.start.mockClear(); fakeWhisperServer.transcribe.mockClear(); fakeWhisperServer.stop.mockClear(); + fakeWhisperServer.shutdown.mockClear(); fakeWhisperServer.start.mockResolvedValue({ port: 9000, backend: "whispercpp-cpu" }); fakeWhisperServer.transcribe.mockResolvedValue({ segments: [{ text: "hello", startSec: 0, endSec: 0.5 }], @@ -198,7 +201,71 @@ describe("SttManager", () => { const mgr = new SttManager(); await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); await mgr.shutdown(); - expect(fakeWhisperServer.stop).toHaveBeenCalledOnce(); + expect(fakeWhisperServer.shutdown).toHaveBeenCalledOnce(); + }); + + it("shutdownStt() stops and releases the singleton exactly once", async () => { + const mgr = getSttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + + await shutdownStt(); + await shutdownStt(); + + expect(fakeWhisperServer.shutdown).toHaveBeenCalledOnce(); + expect(() => getSttManager()).toThrowError(/cancel/i); + }); + + it("does not respawn the helper when shutdown interrupts a failed chunk", async () => { + const mgr = getSttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + fakeWhisperServer.transcribe.mockImplementationOnce(async () => { + await shutdownStt(); + throw new Error("connection closed during quit"); + }); + + const error = await mgr + .transcribe({ samples: new Float32Array(16000), language: "en" }) + .catch((value: unknown) => value); + + expect((error as Error).name).toBe("AbortError"); + expect(fakeWhisperServer.start).toHaveBeenCalledOnce(); + expect(fakeWhisperServer.shutdown).toHaveBeenCalledOnce(); + }); + + it("cancels when shutdown starts as setup rejects", async () => { + const mgr = new SttManager(); + fakeWhisperServer.start.mockImplementationOnce(async () => { + await mgr.shutdown(); + throw new Error("connection closed during startup"); + }); + + const error = await mgr + .init({ modelsBaseDir: "/tmp/fake-stt-models" }) + .catch((value: unknown) => value); + + expect((error as Error).name).toBe("AbortError"); + expect(fakeWhisperServer.shutdown).toHaveBeenCalledOnce(); + }); + + it("cancels when shutdown starts as the final chunk resolves", async () => { + const mgr = getSttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + fakeWhisperServer.transcribe.mockImplementationOnce(async () => { + await shutdownStt(); + return { + segments: [{ text: "late", startSec: 0, endSec: 0.5 }], + wordSegments: [{ word: "late", startSec: 0, endSec: 0.5 }], + detectedLanguage: "en", + backend: "whispercpp-cpu" as const, + }; + }); + + const error = await mgr + .transcribe({ samples: new Float32Array(16_000), language: "en" }) + .catch((value: unknown) => value); + + expect((error as Error).name).toBe("AbortError"); + expect(fakeWhisperServer.transcribe).toHaveBeenCalledOnce(); }); it("retries setup after a failed one instead of caching the rejection", async () => { diff --git a/electron/stt/index.ts b/electron/stt/index.ts index aaa597184..a1ed3cb67 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -20,7 +20,7 @@ import { WhisperServerManager } from "./whisperServer"; * (`chunking.ts`) and runs each through whisper-stt-server's HTTP * `/inference`, which returns both phrase- and word-level segments in one * pass (see whisperServer.ts). Word timestamps come from whisper.cpp's - * native DTW token timestamps (`t_dtw`, SMALL aheads preset, + * native DTW token timestamps (`t_dtw`, LARGE_V3_TURBO aheads preset, * `flash_attn = false`), see * technical-documentation/architecture/transcription-and-captions.md § Decision rationale. * 3. `shutdown()` tears down on app quit. @@ -69,6 +69,7 @@ export interface SttManagerInitOptions { export class SttManager { private readonly server = new WhisperServerManager(); + private shuttingDown = false; private modelsBaseDir: string | null = null; private readonly statusSinks = new Set<(event: SttStatusEvent) => void>(); private initPromise: Promise | null = null; @@ -119,10 +120,11 @@ export class SttManager { * means the second caller just awaits the same completion. */ init(options: SttManagerInitOptions = {}): Promise { + if (this.shuttingDown) return Promise.reject(cancelledError()); if (options.statusSink) this.addStatusSink(options.statusSink); if (options.modelsBaseDir) this.modelsBaseDir = options.modelsBaseDir; if (!this.initPromise) { - // A REJECTED init must not be cached. `prepare()` downloads a 253 MB + // A REJECTED init must not be cached. `prepare()` downloads a large // model on first run, and caching its rejection meant one dropped // connection poisoned the whole app session: every later transcription // — including the retry the UI offers, and every remaining asset in the @@ -158,10 +160,20 @@ export class SttManager { }); }, }); + if (this.shuttingDown) throw cancelledError(); const paths = modelPaths(modelsDir); this.modelPath = paths.whisper; - await this.server.start({ modelPath: paths.whisper }); + try { + await this.server.start({ modelPath: paths.whisper }); + } catch (error) { + if (this.shuttingDown) throw cancelledError(); + throw error; + } + if (this.shuttingDown) { + await this.server.shutdown(); + throw cancelledError(); + } this.emit({ phase: "transcribe" }); } @@ -186,14 +198,17 @@ export class SttManager { ): Promise>> { let lastError: unknown; for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) { + if (this.shuttingDown) throw cancelledError(); try { return await this.server.transcribe({ samples, language }); } catch (error) { lastError = error; + if (this.shuttingDown) throw cancelledError(); if (attempt === CHUNK_ATTEMPTS) break; if (this.modelPath) { await this.server.start({ modelPath: this.modelPath }).catch(() => undefined); } + if (this.shuttingDown) throw cancelledError(); await new Promise((resolve) => setTimeout(resolve, 500 * attempt)); } } @@ -244,6 +259,7 @@ export class SttManager { req.samples.subarray(chunk.startSample, chunk.endSample), language, ).catch((error) => { + if (error instanceof Error && error.name === "AbortError") throw error; // Say where it died. Without this the user gets "Transcription // failed" for a 30-minute recording with no hint that 18 of those // minutes were fine and the helper fell over at one specific spot. @@ -259,6 +275,7 @@ export class SttManager { { cause: error }, ); }); + if (this.shuttingDown || this.cancelEpoch !== epoch) throw cancelledError(); // Chunk-relative timestamps → absolute, the only thing every consumer // (captions, transcript editor, trims) reads. for (const segment of result.segments) { @@ -298,21 +315,40 @@ export class SttManager { /** Best-effort shutdown; safe to call from `before-quit` hooks. */ async shutdown(): Promise { - await this.server.stop(); + if (this.shuttingDown) return; + this.shuttingDown = true; + this.cancelEpoch++; + await this.server.shutdown(); } } let singleton: SttManager | null = null; +let sttShuttingDown = false; /** Lazy singleton for the IPC layer; processes one transcription at a time. */ export function getSttManager(): SttManager { + if (sttShuttingDown) throw cancelledError(); if (!singleton) singleton = new SttManager(); return singleton; } +/** + * Stop and release the lazy singleton without creating one just to quit. + * + * Electron's GUI lifecycle awaits this from a guarded `before-quit` handler; + * clearing the slot first also makes repeated quit events idempotent. + */ +export async function shutdownStt(): Promise { + sttShuttingDown = true; + const manager = singleton; + await manager?.shutdown(); + singleton = null; +} + /** Reset the singleton — for tests. */ export function _resetSttManagerForTests(): void { singleton = null; + sttShuttingDown = false; } /** diff --git a/electron/stt/modelManager.test.ts b/electron/stt/modelManager.test.ts index e5f186fbd..e3f90ebf5 100644 --- a/electron/stt/modelManager.test.ts +++ b/electron/stt/modelManager.test.ts @@ -19,7 +19,7 @@ describe("modelManager", () => { expect(STT_MODELS.whisper.cacheDir).toBe("whisper-ggml"); expect(STT_MODELS.whisper.repoId).toBe("ggerganov/whisper.cpp"); expect(STT_MODELS.whisper.files.length).toBe(1); - expect(STT_MODELS.whisper.files[0].name).toBe("ggml-small-q8_0.bin"); + expect(STT_MODELS.whisper.files[0].name).toBe("ggml-large-v3-turbo-q5_0.bin"); expect(STT_MODELS.whisper.files[0].expectedSha256).not.toBeNull(); for (const f of STT_MODELS.whisper.files) { expect(f.approximateBytes).toBeGreaterThan(0); @@ -32,7 +32,7 @@ describe("modelManager", () => { it("modelPaths places the GGML file under the cache directory", () => { const paths = modelPaths(dir); - expect(paths.whisper).toBe(path.join(dir, "whisper-ggml", "ggml-small-q8_0.bin")); + expect(paths.whisper).toBe(path.join(dir, "whisper-ggml", "ggml-large-v3-turbo-q5_0.bin")); }); it("areModelsPresent returns false when the model file is missing", async () => { @@ -92,7 +92,7 @@ describe("modelManager", () => { expect(fetches).toBe(1); expect(await readFile(paths.whisper)).toEqual(replacement); // The stale copy is displaced by the atomic rename, not quarantined - // beside it: a `.bad` sibling would strand 264 MB nothing ever reaps. + // beside it: a `.bad` sibling would strand hundreds of MB nothing ever reaps. expect(existsSync(`${paths.whisper}.bad`)).toBe(false); expect(existsSync(`${paths.whisper}.partial`)).toBe(false); } finally { @@ -180,7 +180,7 @@ describe("modelManager", () => { const s = await stat(paths.whisper); expect(s.size).toBeGreaterThan(0); expect(progressCalls.length).toBeGreaterThanOrEqual(1); - expect(progressCalls[0].file).toBe("ggml-small-q8_0.bin"); + expect(progressCalls[0].file).toBe("ggml-large-v3-turbo-q5_0.bin"); } finally { STT_MODELS.whisper.files[0].expectedSha256 = originalSha; } diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.ts index cb13fd31f..8a2d9e4c8 100644 --- a/electron/stt/modelManager.ts +++ b/electron/stt/modelManager.ts @@ -13,9 +13,10 @@ import { pipeline } from "node:stream/promises"; * from the `ggml-org` GitHub org the engine itself now lives under; * `ggml-org/whisper.cpp` on HuggingFace is a different, access-gated repo * and returns 401 on every file including README.md — confirmed by curl). - * whisper.cpp bakes precision into the file, so - * there is no runtime `--int8` flag; OpenScreen ships the q8_0 quantized - * `small` multilingual model by default. + * whisper.cpp bakes precision into the file, so there is no runtime `--int8` + * flag. OpenScreen uses the q5_0 quantized large-v3-turbo multilingual model: + * substantially stronger recognition than `small`, while the turbo decoder + * remains practical for interactive captions on Apple Silicon. * * The file is verified by SHA-256 and written atomically (via .partial rename) * to prevent partial downloads from being treated as complete. @@ -27,7 +28,7 @@ import { pipeline } from "node:stream/promises"; export type SttModelId = "whisper"; export interface SttModelFile { - /** Relative path within the model directory (e.g. "ggml-small-q8_0.bin"). */ + /** Relative path within the model directory (e.g. "ggml-large-v3-turbo-q5_0.bin"). */ name: string; /** HuggingFace resolve URL for this file. */ url: string; @@ -54,13 +55,13 @@ const MODEL_BASE = "https://huggingface.co"; // long-standing public model-file repo that never moved when the engine's // GitHub org was renamed. const MODEL_REPO = "ggerganov/whisper.cpp"; -const MODEL_FILE = "ggml-small-q8_0.bin"; +const MODEL_FILE = "ggml-large-v3-turbo-q5_0.bin"; // Pinned to a commit rather than `main` so `expectedSha256` is an invariant and // not a bet: `main` is a mutable branch pointer, and a re-upload under it would // now invalidate every cache in the field at once instead of merely breaking new // installs. This revision was checked against HuggingFace's paths-info API — its // LFS oid for MODEL_FILE is exactly the digest below. -const MODEL_REVISION = "5359861c739e955e79d9a303bcbc70fb988958b1"; +const MODEL_REVISION = "98aa99a0a9db05ae2342309f5096248665f7cba3"; export const STT_MODELS: Record = { whisper: { @@ -70,8 +71,8 @@ export const STT_MODELS: Record = { { name: MODEL_FILE, url: `${MODEL_BASE}/${MODEL_REPO}/resolve/${MODEL_REVISION}/${MODEL_FILE}`, - expectedSha256: "49C8FB02B65E6049D5FA6C04F81F53B867B5EC9540406812C643F177317F779F", - approximateBytes: 264_000_000, + expectedSha256: "394221709CD5AD1F40C46E6031CA61BCE88931E6E088C188294C6D5A55FFA7E2", + approximateBytes: 574_041_195, }, ], }, @@ -179,7 +180,7 @@ async function ensureFile( // would buy nothing — the rename at the end of this function is already // atomic, so there is no window to close — while costing the user their // only model if the replacement never lands (offline, HF 5xx, ENOSPC) - // and stranding 264 MB that nothing ever cleans up. + // and stranding hundreds of MB that nothing ever cleans up. } } diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.ts index 2ee9bdfdc..9f52d5348 100644 --- a/electron/stt/whisperServer.test.ts +++ b/electron/stt/whisperServer.test.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from "node:events"; import { mkdtemp, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -88,6 +89,43 @@ describe("WhisperServerManager", () => { expect(mgr.status.running).toBe(false); }); + it("does not allow a helper to spawn after permanent shutdown", async () => { + const mgr = new WhisperServerManager(); + await mgr.shutdown(); + + await expect(mgr.start({ modelPath: "/missing/model.bin" })).rejects.toThrow(/shutting down/); + const { spawn } = await import("node:child_process"); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("bounds a readiness probe that accepts a connection but never responds", async () => { + vi.useFakeTimers(); + vi.stubGlobal( + "fetch", + vi.fn((_url: string, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { + once: true, + }); + }); + }), + ); + try { + const pollUntilReady = ( + WhisperServerManager as unknown as { + pollUntilReady: (baseUrl: string, timeoutMs: number) => Promise; + } + ).pollUntilReady; + const readiness = pollUntilReady("http://127.0.0.1:9999", 2_500); + const assertion = expect(readiness).rejects.toThrow(/did not respond within 2500ms/); + await vi.advanceTimersByTimeAsync(3_000); + await assertion; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + it("extracts phrase and word segments from a verbose_json response", async () => { const fakeJson = { task: "transcribe", @@ -274,6 +312,106 @@ describe("WhisperServerManager", () => { } }); + it("retries with CPU when the GPU helper exits during model startup", async () => { + const fs = await import("node:fs/promises"); + const { spawn } = await import("node:child_process"); + const dir = await mkdtemp(path.join(tmpdir(), "whisper-cpu-fallback-")); + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + try { + Object.defineProperty(process, "platform", { value: "linux", configurable: true }); + const modelPath = path.join(dir, "ggml-large-v3-turbo-q5_0.bin"); + const fakeBinaryPath = path.join(dir, "whisper-stt-server"); + await fs.writeFile(modelPath, "dummy-ggml"); + await fs.writeFile(fakeBinaryPath, "x", { mode: 0o755 }); + + const child = () => { + const proc = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + pid: 1234, + kill: vi.fn(), + }); + return proc; + }; + const gpuChild = child(); + const cpuChild = child(); + vi.mocked(spawn) + .mockImplementationOnce(() => { + queueMicrotask(() => { + gpuChild.stderr.emit( + "data", + Buffer.from("ggml_metal_buffer_init: initialized\nfailed to allocate GPU buffer"), + ); + gpuChild.emit("exit", 1); + }); + return gpuChild as never; + }) + .mockReturnValueOnce(cpuChild as never); + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementationOnce(() => new Promise(() => undefined)) + .mockResolvedValueOnce(new Response("ok", { status: 200 })), + ); + + const mgr = new WhisperServerManager(); + const result = await mgr.start({ + modelPath, + binaryPath: fakeBinaryPath, + backend: "whispercpp-metal", + }); + expect(result.backend).toBe("whispercpp-cpu"); + expect(spawn).toHaveBeenCalledTimes(2); + expect(vi.mocked(spawn).mock.calls[0]?.[1]).not.toContain("--cpu"); + expect(vi.mocked(spawn).mock.calls[1]?.[1]).toContain("--cpu"); + } finally { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + vi.unstubAllGlobals(); + await rm(dir, { recursive: true, force: true }); + } + }); + + it("serializes overlapping start calls onto one helper process", async () => { + const fs = await import("node:fs/promises"); + const { spawn } = await import("node:child_process"); + const dir = await mkdtemp(path.join(tmpdir(), "whisper-single-start-")); + try { + const modelPath = path.join(dir, "ggml-large-v3-turbo-q5_0.bin"); + const fakeBinaryPath = path.join( + dir, + process.platform === "win32" ? "whisper-stt-server.exe" : "whisper-stt-server", + ); + await fs.writeFile(modelPath, "dummy-ggml"); + await fs.writeFile(fakeBinaryPath, "x", { mode: 0o755 }); + const child = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + pid: 1234, + kill: vi.fn(), + }); + vi.mocked(spawn).mockReturnValue(child as never); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("ok", { status: 200 })), + ); + const mgr = new WhisperServerManager(); + const options = { + modelPath, + binaryPath: fakeBinaryPath, + backend: "whispercpp-cpu" as const, + }; + + const [first, second] = await Promise.all([mgr.start(options), mgr.start(options)]); + + expect(first).toEqual(second); + expect(spawn).toHaveBeenCalledOnce(); + } finally { + vi.unstubAllGlobals(); + await rm(dir, { recursive: true, force: true }); + } + }); + it("refuses to start when the model file is missing", async () => { const fs = await import("node:fs/promises"); const dir = await mkdtemp(path.join(tmpdir(), "whisper-no-model-")); diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index 087073a2e..4635134ed 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -42,7 +42,7 @@ const REQUEST_TIMEOUT_MS = 280_000; * and the renderer doesn't move. * * Word timestamps come from whisper.cpp's native DTW token timestamps - * (`t_dtw`, SMALL aheads preset, `flash_attn = false` so DTW is actually + * (`t_dtw`, LARGE_V3_TURBO aheads preset, `flash_attn = false` so DTW is actually * computed). The helper returns them already absolute, so no segment-offset * arithmetic is required. * @@ -52,7 +52,7 @@ const REQUEST_TIMEOUT_MS = 280_000; */ export interface WhisperServerStartOptions { - /** Absolute path to the GGML model file (e.g. ggml-small-q8_0.bin). */ + /** Absolute path to the GGML model file (e.g. ggml-large-v3-turbo-q5_0.bin). */ modelPath: string; /** Externally-resolved binary path (skips gpuDetector on startup); null = auto. */ binaryPath?: string | null; @@ -97,11 +97,13 @@ interface WhisperJsonResponse { export class WhisperServerManager { private process: WhisperChild | null = null; + private shuttingDown = false; private port: number | null = null; private backend: SttBackend | null = null; private lastError: string | null = null; private startedAtMs: number | null = null; private inFlight: Promise = Promise.resolve(); + private starting: Promise<{ port: number; backend: SttBackend }> | null = null; /** Buffered stderr from the helper; surfaced on shutdown + poll failures. */ private stderrTail = ""; @@ -127,15 +129,28 @@ export class WhisperServerManager { } /** Check the server's HTTP root for a 200; resolves once responsive. */ - private static async pollUntilReady(baseUrl: string, timeoutMs = 30_000): Promise { + private static async pollUntilReady( + baseUrl: string, + timeoutMs = 60_000, + shouldContinue: () => boolean = () => true, + ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { + if (!shouldContinue()) throw new Error("whisper-stt-server exited before readiness"); + const controller = new AbortController(); + const probeTimeout = setTimeout( + () => controller.abort(), + Math.min(2_000, Math.max(1, deadline - Date.now())), + ); try { - const res = await fetch(baseUrl, { method: "GET" }); + const res = await fetch(baseUrl, { method: "GET", signal: controller.signal }); if (res.ok) return; } catch { // not up yet + } finally { + clearTimeout(probeTimeout); } + if (!shouldContinue()) throw new Error("whisper-stt-server exited before readiness"); await new Promise((resolve) => setTimeout(resolve, 250)); } throw new Error(`whisper-stt-server at ${baseUrl} did not respond within ${timeoutMs}ms`); @@ -163,6 +178,19 @@ export class WhisperServerManager { * the cold-start cost twice. */ async start(options: WhisperServerStartOptions): Promise<{ port: number; backend: SttBackend }> { + if (this.starting) return this.starting; + this.starting = this.startImpl(options).finally(() => { + this.starting = null; + }); + return this.starting; + } + + private async startImpl( + options: WhisperServerStartOptions, + ): Promise<{ port: number; backend: SttBackend }> { + if (this.shuttingDown) { + throw new Error("whisper-stt-server manager is shutting down"); + } if (this.process && this.port) { return { port: this.port, backend: this.backend ?? options.backend ?? "whispercpp-cpu" }; } @@ -170,7 +198,8 @@ export class WhisperServerManager { const resolved = options.binaryPath ? { path: options.binaryPath, backend: options.backend ?? "whispercpp-cpu" } : await resolveBinaryPath(); - if (!resolved.path) { + const binaryPath = resolved.path; + if (!binaryPath) { const message = "whisper-stt-server binary not found; build it via scripts/build-whisper-stt.sh"; this.recordError(message); @@ -178,12 +207,12 @@ export class WhisperServerManager { } try { if (process.platform !== "win32") { - await access(resolved.path, fsConstants.X_OK); - } else if (!existsSync(resolved.path)) { + await access(binaryPath, fsConstants.X_OK); + } else if (!existsSync(binaryPath)) { throw new Error("not found"); } } catch { - const message = `whisper-stt-server binary at ${resolved.path} is not executable`; + const message = `whisper-stt-server binary at ${binaryPath} is not executable`; this.recordError(message); throw new Error(message); } @@ -191,10 +220,12 @@ export class WhisperServerManager { throw new Error(`Whisper GGML model not found at ${options.modelPath}`); } - const port = await WhisperServerManager.pickFreePort(); - const child = spawn( - resolved.path, - [ + const launch = async (forceCpu: boolean): Promise<{ port: number; backend: SttBackend }> => { + const port = await WhisperServerManager.pickFreePort(); + if (this.shuttingDown) { + throw new Error("whisper-stt-server manager is shutting down"); + } + const args = [ "--model", options.modelPath, "--port", @@ -203,50 +234,91 @@ export class WhisperServerManager { "127.0.0.1", "--threads", String(Math.max(1, os.cpus().length)), - ], - { stdio: ["ignore", "pipe", "pipe"] }, - ); - - this.process = child; - this.port = port; - this.backend = resolved.backend; - this.startedAtMs = Date.now(); - this.stderrTail = ""; - this.lastError = null; - - child.stdout?.on("data", (chunk: Buffer) => { - process.stdout.write(`[whisper-stt-server] ${chunk.toString()}`); - }); + ]; + if (forceCpu) args.push("--cpu"); + const child = spawn(binaryPath, args, { stdio: ["ignore", "pipe", "pipe"] }); + const activeBackend: SttBackend = forceCpu ? "whispercpp-cpu" : resolved.backend; - child.stderr.on("data", (chunk: Buffer) => { - const text = chunk.toString(); - process.stderr.write(`[whisper-stt-server] ${text}`); - this.stderrTail = (this.stderrTail + text).slice(-this.stderrTailMax); - }); - child.once("exit", (code) => { - if (this.process === child) { - const reason = - code === null - ? "exited without code" - : `exited with code ${code}; stderr=${this.stderrTail.slice(-512)}`; - this.recordError(reason); - this.process = null; - this.port = null; - this.startedAtMs = null; + this.process = child; + this.port = port; + this.backend = activeBackend; + this.startedAtMs = Date.now(); + this.stderrTail = ""; + this.lastError = null; + + child.stdout?.on("data", (chunk: Buffer) => { + process.stdout.write(`[whisper-stt-server] ${chunk.toString()}`); + }); + + child.stderr.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + process.stderr.write(`[whisper-stt-server] ${text}`); + this.stderrTail = (this.stderrTail + text).slice(-this.stderrTailMax); + }); + child.once("exit", (code) => { + if (this.process === child) { + const reason = + code === null + ? "exited without code" + : `exited with code ${code}; stderr=${this.stderrTail.slice(-512)}`; + this.recordError(reason); + this.process = null; + this.port = null; + this.startedAtMs = null; + } + }); + child.once("error", (err) => { + this.recordError(`spawn error: ${err.message}`); + }); + + const exitedBeforeReady = new Promise((_, reject) => { + child.once("exit", (code) => { + reject( + new Error( + `whisper-stt-server exited during startup (${code ?? "no code"}); ` + + `stderr=${this.stderrTail.slice(-512)}`, + ), + ); + }); + child.once("error", reject); + }); + try { + await Promise.race([ + WhisperServerManager.pollUntilReady( + `http://127.0.0.1:${port}`, + 60_000, + () => this.process === child, + ), + exitedBeforeReady, + ]); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + await this.stop(); + this.recordError(message); + throw new Error(message); } - }); - child.once("error", (err) => { - this.recordError(`spawn error: ${err.message}`); - }); + return { port, backend: activeBackend }; + }; - const baseUrl = `http://127.0.0.1:${port}`; try { - await WhisperServerManager.pollUntilReady(baseUrl); + return await launch(false); } catch (err) { - await this.stop(); - throw err instanceof Error ? err : new Error(String(err)); + const startupLog = `${err instanceof Error ? err.message : String(err)} ${this.stderrTail}`; + const startupLines = startupLog.split(/\r?\n/); + const mentionsGpuBackend = startupLines.some((line) => + /(?:ggml_(?:metal|vulkan|cuda)|gpu)/i.test(line), + ); + const mentionsStartupFailure = startupLines.some((line) => + /(?:fail|error|allocat)/i.test(line), + ); + const gpuStartupFailed = + resolved.backend !== "whispercpp-cpu" && mentionsGpuBackend && mentionsStartupFailure; + if (!gpuStartupFailed) throw err; + process.stderr.write( + "[whisper-stt-server] GPU startup failed; retrying with CPU inference\n", + ); + return launch(true); } - return { port, backend: resolved.backend }; } /** Send SIGTERM and wait for the helper to exit. Resolves even if it was already down. */ @@ -274,6 +346,12 @@ export class WhisperServerManager { } } + /** Permanently prevent respawn, then stop the currently owned helper. */ + async shutdown(): Promise { + this.shuttingDown = true; + await this.stop(); + } + private baseUrl(): string { if (!this.port) throw new Error("whisper-stt-server not started"); return `http://127.0.0.1:${this.port}`; diff --git a/scripts/build-whisper-stt.sh b/scripts/build-whisper-stt.sh index 3dd1f304e..ade88df57 100644 --- a/scripts/build-whisper-stt.sh +++ b/scripts/build-whisper-stt.sh @@ -217,7 +217,7 @@ build_variant() { # @rpath/libggml.0.dylib and dies in dyld before main(). The `lib` prefix and # the `.so.` version suffixes are what the globs below add. # - # -a preserves the symlink farm (libggml.dylib -> libggml.0.dylib -> + # -P preserves the symlink farm (libggml.dylib -> libggml.0.dylib -> # libggml.0.15.1.dylib); plain `cp` dereferences each one into a full copy of # the same payload, which tripled the staged size for no benefit. local found_libs=0 @@ -237,7 +237,13 @@ libggml*.dylib|libwhisper*.dylib|libparakeet*.dylib|\ libggml*.so|libggml*.so.*|libwhisper*.so|libwhisper*.so.*|\ libparakeet*.so|libparakeet*.so.*|\ *.metal) - cp -a "${f}" "${OUT_DIR}/" + # The output directory may contain dereferenced regular files from a + # downloaded release artifact. Remove the exact destination first so + # cp can recreate this build's symlink farm without following an old + # regular-file/symlink mix. -P preserves links without -a's macOS + # chflags pass, which can itself report ELOOP while replacing a link. + rm -f "${OUT_DIR}/${f##*/}" + cp -P "${f}" "${OUT_DIR}/" found_libs=1 ;; esac diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index 47538bc65..1641e2dee 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -98,7 +98,7 @@ subtitle is the reason rather than "With AI". ### One phase, including the first-run model download -On a fresh install the GGML model (~253 MB) is not on disk. It is fetched by +On a fresh install the GGML model (~574 MB) is not on disk. It is fetched by `SttManager.prepare()` **inside** the `stt:transcribe` IPC call — i.e. inside a run this store has already marked `running` — so the user sees one single busy phase that simply takes longer the first time. That is deliberate: no separate @@ -107,7 +107,7 @@ clickable in the meantime (`phase: "model"` is emitted by the main process but deliberately not forwarded to the renderer by `transcribeAsset`). Nothing in the renderer imposes a timeout that a slow download could trip: the preload does a bare `ipcRenderer.invoke`, `fetchWithRetry` has no per-request deadline, and -whisper-server's 30 s readiness budget only starts once the download resolved. +whisper-server's 60 s readiness budget only starts once the download resolved. Three edges make that promise hold, and each is load-bearing: @@ -145,11 +145,11 @@ per-platform binary name only, and the real backend is corrected from the helper's `/inference` JSON. Why whisper.cpp, in one paragraph: a single C++ dependency with native DTW -token timestamps for word-level timing, a portable runtime device selection -that covers Metal, Vulkan and CPU in one binary, and self-contained long-form -chunking (`whisper_full()` over recordings longer than 30 s) without manual -windowing on our side. Validation data — backend-by-backend WER and real-time -factors — lives in +token timestamps for word-level timing and portable runtime device selection +that covers Metal, Vulkan and CPU in one binary. OpenScreen supplies bounded, +sequential chunks to `whisper_full()` and restores their absolute timestamps, +so long recordings retain progress and retry boundaries. Validation data — +backend-by-backend WER and real-time factors — lives in [`tools/stt-eval/whispercpp-dtw-poc/REPORT.md`](../../tools/stt-eval/whispercpp-dtw-poc/REPORT.md). ### Per-platform backend @@ -187,7 +187,7 @@ it verbatim in the response. array. 2. **DTW timestamp** — every non-special token carries `t_dtw` in centiseconds from whisper.cpp's native DTW - (`dtw_token_timestamps=true`, `dtw_aheads_preset=WHISPER_AHEADS_SMALL`, + (`dtw_token_timestamps=true`, `dtw_aheads_preset=WHISPER_AHEADS_LARGE_V3_TURBO`, `flash_attn=false`, which together are the prerequisites for DTW to actually run). `t_dtw == -1` is the DTW-inactive guardrail: the helper fails the request rather than emit zero-quality timestamps. @@ -224,15 +224,16 @@ so no second pass is needed. ### Long-form recordings -`whisper_full()` handles recordings longer than 30 s internally; OpenScreen -implements no chunking of its own. The validation set exercises 130 s at WER -0.076 with full per-word coverage (see the validation report linked above). +OpenScreen splits long recordings at nearby low-energy boundaries, transcribes +the bounded chunks sequentially, then restores their absolute timestamps. This +keeps progress observable and makes a failed chunk retryable without running a +whole long recording again. ### Model -The single shipped artifact is `ggml-small-q8_0.bin` from -`ggerganov/whisper.cpp` on HuggingFace: Whisper `small`, multilingual (~99 -languages), q8_0 quantised, ~264 MB. Precision is baked into the GGML file — +The single shipped artifact is `ggml-large-v3-turbo-q5_0.bin` from +`ggerganov/whisper.cpp` on HuggingFace: Whisper `large-v3-turbo`, multilingual +(~99 languages), q5_0 quantised, ~574 MB. Precision is baked into the GGML file — there is no runtime `--int8` flag. `electron/stt/modelManager.ts` downloads the file once into the user-data cache and writes it through an atomic `.partial` rename, so a half-downloaded file can never be picked up as a @@ -284,7 +285,7 @@ bash scripts/build-whisper-stt.sh # issues inside whisper.cpp's vulkan-shaders-gen sub-project. # Run the helper directly for manual testing -set OPENSCREEN_WHISPER_MODEL=%APPDATA%\Electron\stt-models\whisper-ggml\ggml-small-q8_0.bin +set OPENSCREEN_WHISPER_MODEL=%APPDATA%\Electron\stt-models\whisper-ggml\ggml-large-v3-turbo-q5_0.bin electron\native\bin\win32-x64\whisper-stt-server.exe --port 20199 --threads 8 # Test