Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions bindings/python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ with model.session() as session, session.stream() as stream:
stream.feed(chunk)
text = stream.text() # .committed (stable) + .tentative
stream.finalize()
result = stream.snapshot() # language, segments, words, tokens, timings
```

Long transcriptions can be cancelled from another thread with
Expand Down
5 changes: 5 additions & 0 deletions bindings/python/src/transcribe_cpp/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1413,6 +1413,11 @@ def text(self) -> StreamText:
tentative=_decode(txt.tentative_text),
)

def snapshot(self) -> Result:
"""Full structured snapshot of the current hypothesis (owned copies)."""
_ = self._h # validate that this stream has not been reset
return self._session._materialize()

@property
def state(self) -> str:
"""``"idle"`` / ``"active"`` / ``"finished"`` / ``"failed"``."""
Expand Down
8 changes: 8 additions & 0 deletions bindings/python/tests/test_streaming.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,19 @@ def test_streaming_real(streaming_model_path, audio_pcm):
stream.feed(audio_pcm[i : i + 16000])
update = stream.finalize()
committed = stream.text().committed
snapshot = stream.snapshot()
revision, state = stream.revision, stream.state
last_status = stream.last_status
assert update.is_final, update
assert state == "finished", state
assert revision >= 1
assert last_status is None, last_status
assert "country" in committed.lower(), committed
assert snapshot.text
assert isinstance(snapshot.language, str)
assert snapshot.segments
assert isinstance(snapshot.words, tuple)
assert isinstance(snapshot.tokens, tuple)


def test_streaming_with_language_hint(prompted_streaming_model_path, audio_pcm):
Expand Down Expand Up @@ -122,6 +128,8 @@ def test_stream_use_after_reset_rejected(streaming_model_path, audio_pcm):
stream.feed(audio_pcm[:16000])
with pytest.raises(t.TranscribeError, match="reset"):
stream.text()
with pytest.raises(t.TranscribeError, match="reset"):
stream.snapshot()


def test_stream_reset_idempotent_and_session_reusable(
Expand Down
12 changes: 12 additions & 0 deletions bindings/rust/transcribe-cpp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,18 @@ println!("{}", result.text);
# Ok::<(), transcribe_cpp::Error>(())
```

Streaming exposes both UI-stable text and a fully materialized structured
snapshot:

```rust
let mut stream = session.stream(&RunOptions::default(), &Default::default())?;
stream.feed(&chunk)?;
println!("{}", stream.text().committed);
stream.finalize()?;
let transcript = stream.snapshot(); // language, segments, words, tokens, timings
# Ok::<(), transcribe_cpp::Error>(())
```

Runnable examples:

```sh
Expand Down
11 changes: 11 additions & 0 deletions bindings/rust/transcribe-cpp/tests/streaming.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,17 @@ fn streams_jfk_committed_text() {
"stream text: {:?}",
text.full
);

let snapshot = stream.snapshot();
assert!(!snapshot.text.is_empty(), "structured snapshot is empty");
if let Some(language) = &snapshot.language {
assert!(!language.is_empty(), "detected language must not be empty");
}
assert!(
!snapshot.segments.is_empty(),
"structured segments are empty"
);
let _ = (&snapshot.words, &snapshot.tokens);
}

#[test]
Expand Down
1 change: 1 addition & 0 deletions bindings/swift/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ for chunk in chunks { // 16 kHz mono float32 frames
if update.committedChanged { print(stream.text.committed) }
}
try stream.finalize()
let transcript = stream.snapshot // language, segments, words, tokens, timings
```

Runnable examples live in
Expand Down
2 changes: 1 addition & 1 deletion bindings/swift/Sources/TranscribeCpp/Session.swift
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ public final class Session {
}
}

private func readTranscript() -> Transcript {
func readTranscript() -> Transcript {
var segments: [Segment] = []
for i in 0..<Int(transcribe_n_segments(ptr)) {
var s = transcribe_segment(); transcribe_segment_init(&s)
Expand Down
3 changes: 3 additions & 0 deletions bindings/swift/Sources/TranscribeCpp/Streaming.swift
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ public final class Stream {
_ = transcribe_stream_get_text(session.ptr, &t)
return StreamText(t)
}

/// Full structured snapshot of the current hypothesis (owned copies).
public var snapshot: Transcript { session.readTranscript() }
}

extension Session {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@ final class StreamingTests: XCTestCase {
let stream = try session.stream()
try Fixtures.drive(stream, pcm: pcm)
XCTAssertTrue(stream.text.full.lowercased().contains("country"), stream.text.full)
let snapshot = stream.snapshot
XCTAssertFalse(snapshot.text.isEmpty)
if let language = snapshot.language { XCTAssertFalse(language.isEmpty) }
XCTAssertFalse(snapshot.segments.isEmpty)
_ = snapshot.words
_ = snapshot.tokens
}

func testOnFinalizePolicyCommitsAtFinalize() throws {
Expand Down
8 changes: 5 additions & 3 deletions bindings/typescript/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const model = await TranscribeModel.load("whisper-tiny-Q5_K_M.gguf");
const result = await model.transcribe(pcm, { timestamps: "segment" });

console.log(result.text);
console.log(result.language); // detected or requested
console.log(result.language); // model-detected, or "" when unavailable/a hint was supplied
for (const seg of result.segments) {
console.log(`[${seg.t0Ms}–${seg.t1Ms}ms] ${seg.text}`);
}
Expand All @@ -47,6 +47,7 @@ for (const chunk of pcmChunks) {
render(committed, tentative);
}
await stream.finalize();
const snapshot = stream.snapshot; // text, language, segments, words, tokens, timings
stream.reset();
```

Expand Down Expand Up @@ -146,8 +147,9 @@ the teardown — is refused with `Busy`, by design.
Because the compute is genuinely on another thread, **do not touch a session
while a call against it is in flight** — it is single-threaded in the C library:

- Reading a stream's `text`/`state`/`revision`/`lastStatus`, or a session's
`limits`/`wasAborted`, during an un-awaited `feed`/`finalize`/`run` **throws**.
- Reading a stream's `text`/`snapshot`/`state`/`revision`/`lastStatus`, or a
session's `limits`/`wasAborted`, during an un-awaited
`feed`/`finalize`/`run`/`runBatch` **throws**.
- `reset()` and `dispose()` are safe to call any time: the native teardown is
deferred behind any in-flight call, so it never frees a session mid-compute.
- Disposing a `Session` or `TranscribeModel` while a stream is still active
Expand Down
49 changes: 26 additions & 23 deletions bindings/typescript/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import type {
Timings,
TimestampKind,
Token,
Transcript,
TranscribeOptions,
TranscriptionResult,
Word,
Expand Down Expand Up @@ -429,10 +430,7 @@ function batchAccessors(n: Native, h: any, i: number): Accessors {
};
}

function materialize(
n: Native,
acc: Accessors,
): Omit<TranscriptionResult, "aborted" | "truncated"> {
function materialize(n: Native, acc: Accessors): Transcript {
const F = n.F;

const segments: Segment[] = [];
Expand Down Expand Up @@ -683,6 +681,7 @@ const STREAM_TEARDOWN = new WeakMap<
interface SessionControl {
enterCompute(kind: string): void;
leaveCompute(kind: string): void;
currentCompute(): string | null;
isCurrentStream(stream: Stream): boolean;
replaceCurrentStream(stream: Stream): void;
clearCurrentStream(stream: Stream): void;
Expand Down Expand Up @@ -722,6 +721,7 @@ export class Session {
leaveCompute: (kind) => {
if (this.#inFlight === kind) this.#inFlight = null;
},
currentCompute: () => this.#inFlight,
isCurrentStream: (stream) => this.#activeStream === stream,
replaceCurrentStream: (stream) => {
if (this.#activeStream && this.#activeStream !== stream) {
Expand Down Expand Up @@ -1039,7 +1039,6 @@ export class Stream {
#keepalive: unknown[] | null;
#active = true;
#stale = false; // true once the session has begun a newer native stream
#inFlight = false; // true while a feed/finalize native call runs on a worker
#holdsLease = true; // born holding the model's compute lease (claimed at begin)

/** @internal */
Expand Down Expand Up @@ -1108,9 +1107,8 @@ export class Stream {
// The native feed runs on a libuv worker. While it is in flight the
// session must not be touched from the main thread — the C session API
// is single-threaded (transcribe.h), and stream_get_text hands back
// pointers the feed may free/realloc. Flag it so the read getters fail
// fast instead of racing into a use-after-free.
this.#inFlight = true;
// pointers the feed may free/realloc. Flag the owning session so every
// result getter fails fast instead of racing into a use-after-free.
this.#sessionControl.enterCompute("feed()/finalize()");
try {
const status = await callAsync<number>(
Expand All @@ -1128,7 +1126,6 @@ export class Stream {
}
check(n, status, "transcribe_stream_feed");
} finally {
this.#inFlight = false;
this.#sessionControl.leaveCompute("feed()/finalize()");
}
return toStreamUpdate(u);
Expand All @@ -1144,7 +1141,6 @@ export class Stream {
return this.#lock.run(async () => {
const u: any = {};
n.F.streamUpdateInit(u);
this.#inFlight = true; // see feed(): worker-thread compute, no concurrent reads
this.#sessionControl.enterCompute("feed()/finalize()");
try {
check(
Expand All @@ -1153,7 +1149,6 @@ export class Stream {
"transcribe_stream_finalize",
);
} finally {
this.#inFlight = false;
this.#sessionControl.leaveCompute("feed()/finalize()");
// Finalize ends the active stream (FINISHED on success, FAILED on
// error), so the model is free again — release the lease either way.
Expand All @@ -1164,16 +1159,15 @@ export class Stream {
}

/**
* Reads borrow session-owned snapshot memory, so they are forbidden while a
* feed()/finalize() is computing on a worker thread (concurrent use of a
* single session is undefined per transcribe.h). The natural pattern —
* `await stream.feed(chunk)` then read — is unaffected; this only rejects a
* read issued against an un-awaited feed.
* Reads borrow session-owned snapshot memory, so they are forbidden while
* any worker call is computing on this session (concurrent use is undefined
* per transcribe.h). The natural await-then-read pattern is unaffected.
*/
#assertNotFeeding(what: string): void {
if (this.#inFlight) {
#assertNotComputing(what: string): void {
const compute = this.#sessionControl.currentCompute();
if (compute) {
throw new TranscribeError(
`cannot read stream ${what} while a feed()/finalize() is in flight; await it first`,
`cannot read stream ${what} while ${compute} is in flight; await it first`,
);
}
}
Expand All @@ -1182,7 +1176,7 @@ export class Stream {
get text(): StreamText {
const h = this.#session.handle; // throws if the session was disposed
this.#assertCurrent("read stream text");
this.#assertNotFeeding("text");
this.#assertNotComputing("text");
const n = this.#n;
const t: any = {};
n.F.streamTextInit(t);
Expand All @@ -1194,18 +1188,27 @@ export class Stream {
};
}

/** Full structured snapshot of the current hypothesis (owned copies). */
get snapshot(): Transcript {
const h = this.#session.handle; // throws if the session was disposed
this.#assertCurrent("read stream snapshot");
if (!this.#active) throw new TranscribeError("stream has been reset");
this.#assertNotComputing("snapshot");
return materialize(this.#n, singleAccessors(this.#n, h));
}

get state(): StreamState {
const h = this.#session.handle; // throws if the session was disposed
this.#assertCurrent("read stream state");
if (!this.#active) return "idle"; // reset() returns to idle; native reset may still be queued
this.#assertNotFeeding("state");
this.#assertNotComputing("state");
return STREAM_STATES[this.#n.F.streamGetState(h)] ?? "idle";
}

get revision(): number {
const h = this.#session.handle; // throws if the session was disposed
this.#assertCurrent("read stream revision");
this.#assertNotFeeding("revision");
this.#assertNotComputing("revision");
return this.#n.F.streamRevision(h);
}

Expand All @@ -1217,7 +1220,7 @@ export class Stream {
get lastStatus(): TranscribeError | null {
const h = this.#session.handle; // throws if the session was disposed
this.#assertCurrent("read stream lastStatus");
this.#assertNotFeeding("lastStatus");
this.#assertNotComputing("lastStatus");
const n = this.#n;
const status = n.F.streamLastStatus(h);
if (status === g.TRANSCRIBE_OK) return null;
Expand Down
7 changes: 6 additions & 1 deletion bindings/typescript/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,19 +83,24 @@ export interface SessionLimits {
maxKvBytes: number;
}

export interface TranscriptionResult {
export interface Transcript {
text: string;
/** The model's decoded output before family post-processing (diarization
* markers, timestamp/special tokens, tag filtering, whitespace trims).
* Equal to `text` modulo whitespace for families that emit clean text. */
rawText: string;
/** Model-detected language, or an empty string when none applies. */
language: string;
timestampKind: TimestampKind;
segments: Segment[];
speakerSegments: SpeakerSegment[];
words: Word[];
tokens: Token[];
timings: Timings;
}

/** An offline transcript plus terminal run status flags. */
export interface TranscriptionResult extends Transcript {
aborted: boolean;
truncated: boolean;
}
Expand Down
21 changes: 20 additions & 1 deletion bindings/typescript/test/streaming.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,22 @@ modelTest("streaming commits text and finalizes", STREAMING_MODEL, async () => {
assert.equal(stream.lastStatus, null, "a healthy finished stream has no failure status");
const t = stream.text;
assert.ok((t.committed + t.full).trim().length > 0, "expected non-empty streamed text");
const snapshot = stream.snapshot;
assert.ok(snapshot.text.trim().length > 0, "expected non-empty structured snapshot");
assert.ok(snapshot.segments.length > 0, "expected structured segment rows");
assert.ok(Array.isArray(snapshot.words));
assert.ok(Array.isArray(snapshot.tokens));
assert.equal(typeof snapshot.language, "string");
stream.reset();
assert.equal(stream.state, "idle");
assert.throws(() => stream.snapshot, /reset/i, "reading .snapshot after reset must throw");
s.dispose();
} finally {
m.dispose();
}
});

modelTest("stream reads reject while a feed is in flight", STREAMING_MODEL, async () => {
modelTest("stream reads reject while the session is computing", STREAMING_MODEL, async () => {
const m = await TranscribeModel.load(STREAMING_MODEL);
try {
const s = m.createSession();
Expand All @@ -34,13 +41,25 @@ modelTest("stream reads reject while a feed is in flight", STREAMING_MODEL, asyn
const pending = stream.feed(chunk); // do NOT await — leave it in flight
await Promise.resolve(); // let the feed's native call reach the worker
assert.throws(() => stream.text, /in flight/i, "reading .text mid-feed must throw");
assert.throws(() => stream.snapshot, /in flight/i, "reading .snapshot mid-feed must throw");
assert.throws(() => stream.state, /in flight/i);
assert.throws(() => stream.revision, /in flight/i);
assert.throws(() => s.limits, /in flight/i);
assert.throws(() => s.wasAborted, /in flight/i);

await pending; // once awaited, reads are fine again
assert.doesNotThrow(() => stream.text);
assert.doesNotThrow(() => stream.snapshot);

// A finished stream wrapper still reads the same session-owned result
// storage. Guard it during a later offline call on that session too, not
// only during the wrapper's own feed/finalize calls.
await stream.finalize();
const runPending = s.run(jfk());
await Promise.resolve(); // let run() reach its worker
assert.throws(() => stream.snapshot, /run\(\).*in flight/i);
assert.throws(() => stream.text, /run\(\).*in flight/i);
await runPending;

stream.reset();
s.dispose();
Expand Down
Loading
Loading