diff --git a/README.md b/README.md index 620597ac..b5815a2f 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,9 @@ Official bindings wrap the C API for other languages: | Swift / ObjC | [bindings/swift](bindings/swift) | See [`docs/bindings.md`](docs/bindings.md) for how the bindings are generated -and kept in sync with the header. +and kept in sync with the header. Upgrading from 0.1? Read the +[0.2 migration guide](docs/migrating-to-0.2.md), including the new exact-device +selection API and the changed meaning of CLI `--device 0`. ## Tests diff --git a/bindings/python/README.md b/bindings/python/README.md index 3e92202f..7b5fbbe1 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -6,6 +6,10 @@ a C/C++ speech-to-text library built on ggml. > **Status: in development.** Until wheels are published, use a locally built > `libtranscribe` through repo auto-discovery or `TRANSCRIBE_LIBRARY`. +Upgrading from 0.1? See the +[0.2 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.2.md), +including the replacement of `gpu_device=` with exact device objects. + ```python import transcribe_cpp @@ -44,9 +48,17 @@ Long transcriptions can be cancelled from another thread with ## Backends -`Model(backend=...)` picks the compute device (`"auto"` uses the best -available). `transcribe_cpp.backends()` lists registered backends and -`backend_available(kind)` checks one kind. +`Model(backend=...)` applies a backend policy (`"auto"` uses the best +available). `transcribe_cpp.backends()` returns process-local device objects; +pass one as `Model(device=device)` for exact selection with no fallback. Persist +a device's `device_id`, not its runtime handle or index. `backend_available(kind)` +checks whether a backend policy can currently be satisfied. + +```python +device = next(d for d in transcribe_cpp.backends() if d.device_type == "cpu") +with transcribe_cpp.Model("model.gguf", device=device) as model: + print(model.device) +``` | Variable | Effect | |---|---| diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index 0cf68086..7f38330b 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -24,7 +24,7 @@ import os import threading import weakref -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Literal, Optional, Sequence, Union from . import _abi, _generated @@ -266,9 +266,13 @@ def native_provider() -> str | None: } -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class BackendDevice: - """One registered compute device (owned copies of the C strings).""" + """One registered compute device (owned copies of the C strings). + + Equality compares opaque native identity; display index and live memory + snapshots do not affect whether two values name the same device. + """ name: str description: str @@ -283,18 +287,26 @@ class BackendDevice: # (via backends() or Model.device) to refresh; backend-defined and not # comparable across device kinds. memory_free: int - # Registry index of this device — the value to pass as ``Model(..., - # gpu_device=index)`` to select it (0 means auto: discrete GPUs are - # probed before integrated). - # None when the device came from Model.device, since the underlying - # transcribe_model_get_device() does not expose an index; correlate such a - # device back to backends() by device_id / name instead. The index is - # order-dependent and not stable across driver updates or hosts. + # Registry index for display. Exact model selection uses the BackendDevice + # itself; indices are process-local and not stable across driver updates. index: Optional[int] = None + # Opaque process-local native device handle. Applications persist device_id, + # never this value. + _handle: Optional[int] = field(default=None, repr=False, compare=False) + + def __eq__(self, other: object) -> bool: + if self is other: + return True + if not isinstance(other, BackendDevice): + return NotImplemented + return self._handle is not None and self._handle == other._handle + + def __hash__(self) -> int: + return hash(self._handle) if self._handle is not None else object.__hash__(self) -def _backend_device_from_raw(dev, index: Optional[int] = None) -> BackendDevice: - """Build a BackendDevice from a library-filled transcribe_backend_device.""" +def _backend_device_from_raw(dev, handle: Optional[int], index: Optional[int] = None) -> BackendDevice: + """Build a BackendDevice from a library-filled transcribe_device_info.""" return BackendDevice( name=_decode(dev.name), description=_decode(dev.description), @@ -304,6 +316,7 @@ def _backend_device_from_raw(dev, index: Optional[int] = None) -> BackendDevice: memory_total=int(dev.memory_total), memory_free=int(dev.memory_free), index=index, + _handle=handle, ) @@ -315,12 +328,15 @@ def backends() -> list[BackendDevice]: Each device's ``memory_free`` is live as of the call; call again to poll a device's available memory over time.""" devices = [] - for i in range(_lib.transcribe_backend_device_count()): - dev = _generated.transcribe_backend_device() - _lib.transcribe_backend_device_init(_byref(dev)) - _check(_lib.transcribe_get_backend_device(i, _byref(dev)), + for i in range(_lib.transcribe_device_count()): + handle = _lib.transcribe_device_get(i) + if not handle: + continue + dev = _generated.transcribe_device_info() + _lib.transcribe_device_info_init(_byref(dev)) + _check(_lib.transcribe_device_get_info(handle, _byref(dev)), f"reading backend device {i}") - devices.append(_backend_device_from_raw(dev, index=i)) + devices.append(_backend_device_from_raw(dev, int(handle), index=i)) return devices @@ -857,7 +873,7 @@ class Model: """ def __init__(self, path: str | os.PathLike, *, - backend: Backend = "auto", gpu_device: int = 0): + backend: Backend = "auto", device: BackendDevice | None = None): # Live sessions, tracked weakly: close() must free them before the # model, because transcribe_model_free is only valid once every # derived session is gone (use-after-free otherwise). Created before @@ -870,7 +886,10 @@ def __init__(self, path: str | os.PathLike, *, params = _ModelLoadParams() _lib.transcribe_model_load_params_init(_byref(params)) params.backend = _enum(_BACKENDS, backend, backend_source) - params.gpu_device = gpu_device + if device is not None: + if not isinstance(device, BackendDevice) or device._handle is None: + raise TypeError("device must be a BackendDevice returned by backends()") + params.device = device._handle handle = ctypes.c_void_p() status = _lib.transcribe_model_load_file( @@ -905,11 +924,14 @@ def device(self) -> BackendDevice: live snapshot, so read this again to poll how much device memory is left after the model loaded. Raises if the model has no resolved compute device.""" - dev = _generated.transcribe_backend_device() - _lib.transcribe_backend_device_init(_byref(dev)) - _check(_lib.transcribe_model_get_device(self._h, _byref(dev)), - "model_get_device") - return _backend_device_from_raw(dev) + handle = _lib.transcribe_model_device(self._h) + if not handle: + raise BackendError("model has no resolved compute device") + dev = _generated.transcribe_device_info() + _lib.transcribe_device_info_init(_byref(dev)) + _check(_lib.transcribe_device_get_info(handle, _byref(dev)), + "device_get_info") + return _backend_device_from_raw(dev, int(handle)) @property def capabilities(self) -> Capabilities: @@ -1431,7 +1453,7 @@ def transcribe( pcm: PCMLike, *, backend: Backend = "auto", - gpu_device: int = 0, + device: BackendDevice | None = None, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0, @@ -1449,7 +1471,7 @@ def transcribe( *model* may be a path (loaded and freed within this call) or an existing Model (reused and left open). Loading a model is not free, so to transcribe many clips keep a Model and call ``model.session().run(...)`` yourself; this - helper is for the one-shot case. ``backend`` / ``gpu_device`` apply only when + helper is for the one-shot case. ``backend`` / ``device`` apply only when *model* is a path — they are ignored when an already-loaded Model is passed. ``family`` / ``spec_k_drafts`` pass through to :meth:`Session.run`. """ @@ -1463,6 +1485,6 @@ def transcribe( with model.session(**session_opts) as session: return session.run(pcm, **run_opts) - with Model(model, backend=backend, gpu_device=gpu_device) as owned: + with Model(model, backend=backend, device=device) as owned: with owned.session(**session_opts) as session: return session.run(pcm, **run_opts) diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index e6d42931..ac9763c2 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "7896d8d4c2a46147" +PUBLIC_HEADER_HASH = "7df72bf9e667b8c2" # === enum constants === TRANSCRIBE_OK = 0 @@ -48,7 +48,7 @@ TRANSCRIBE_ABI_STREAM_TEXT = 10 TRANSCRIBE_ABI_SESSION_LIMITS = 11 TRANSCRIBE_ABI_EXT = 12 -TRANSCRIBE_ABI_BACKEND_DEVICE = 13 +TRANSCRIBE_ABI_DEVICE_INFO = 13 TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14 TRANSCRIBE_LOG_LEVEL_NONE = 0 TRANSCRIBE_LOG_LEVEL_INFO = 1 @@ -120,7 +120,7 @@ # === structs === class transcribe_ext(_c.Structure): pass -class transcribe_backend_device(_c.Structure): +class transcribe_device_info(_c.Structure): pass class transcribe_model_load_params(_c.Structure): pass @@ -164,8 +164,8 @@ class transcribe_whisper_chunk_trace(_c.Structure): pass transcribe_ext._fields_ = [("size", _c.c_uint64), ("kind", _c.c_uint32)] -transcribe_backend_device._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)] -transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("gpu_device", _c.c_int)] +transcribe_device_info._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)] +transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("device", _c.c_void_p)] transcribe_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int), ("kv_type", _c.c_int), ("n_ctx", _c.c_int32)] transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("diarize", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32)] transcribe_capabilities._fields_ = [("struct_size", _c.c_uint64), ("native_sample_rate", _c.c_int32), ("n_languages", _c.c_int), ("languages", _c.POINTER(_c.c_char_p)), ("max_timestamp_kind", _c.c_int), ("supports_language_detect", _c.c_bool), ("supports_translate", _c.c_bool), ("supports_streaming", _c.c_bool), ("supports_spec_decode", _c.c_bool), ("max_audio_ms", _c.c_int64), ("n_translate_target_languages", _c.c_int), ("translate_target_languages", _c.POINTER(_c.c_char_p))] @@ -190,7 +190,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): # transcribe_abi_struct id per struct (for the native size/align check). ABI_STRUCT_IDS = { 'transcribe_ext': 12, - 'transcribe_backend_device': 13, + 'transcribe_device_info': 13, 'transcribe_model_load_params': 0, 'transcribe_session_params': 1, 'transcribe_run_params': 2, @@ -209,8 +209,8 @@ class transcribe_whisper_chunk_trace(_c.Structure): # C-compiler layout captured at generation (for offset self-check). STRUCT_LAYOUT = { 'transcribe_ext': {'size': 16, 'align': 8, 'offsets': {'size': 0, 'kind': 8}}, - 'transcribe_backend_device': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}}, - 'transcribe_model_load_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'gpu_device': 12}}, + 'transcribe_device_info': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}}, + 'transcribe_model_load_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'device': 16}}, 'transcribe_session_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16}}, 'transcribe_run_params': {'size': 72, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'diarize': 24, 'language': 32, 'target_language': 40, 'keep_special_tags': 48, 'family': 56, 'spec_k_drafts': 64}}, 'transcribe_capabilities': {'size': 56, 'align': 8, 'offsets': {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48}}, @@ -241,10 +241,6 @@ def configure(lib): lib.transcribe_abi_struct_size.argtypes = [_c.c_int] lib.transcribe_backend_available.restype = _c.c_bool lib.transcribe_backend_available.argtypes = [_c.c_int] - lib.transcribe_backend_device_count.restype = _c.c_int - lib.transcribe_backend_device_count.argtypes = [] - lib.transcribe_backend_device_init.restype = None - lib.transcribe_backend_device_init.argtypes = [_c.POINTER(transcribe_backend_device)] lib.transcribe_batch_detected_language.restype = _c.c_char_p lib.transcribe_batch_detected_language.argtypes = [_c.c_void_p, _c.c_int] lib.transcribe_batch_full_text.restype = _c.c_char_p @@ -281,12 +277,18 @@ def configure(lib): lib.transcribe_close.argtypes = [_c.c_void_p] lib.transcribe_detected_language.restype = _c.c_char_p lib.transcribe_detected_language.argtypes = [_c.c_void_p] + lib.transcribe_device_count.restype = _c.c_int + lib.transcribe_device_count.argtypes = [] + lib.transcribe_device_get.restype = _c.c_void_p + lib.transcribe_device_get.argtypes = [_c.c_int] + lib.transcribe_device_get_info.restype = _c.c_int + lib.transcribe_device_get_info.argtypes = [_c.c_void_p, _c.POINTER(transcribe_device_info)] + lib.transcribe_device_info_init.restype = None + lib.transcribe_device_info_init.argtypes = [_c.POINTER(transcribe_device_info)] lib.transcribe_ext_check.restype = _c.c_int lib.transcribe_ext_check.argtypes = [_c.POINTER(transcribe_ext), _c.c_uint32, _c.c_uint64] lib.transcribe_full_text.restype = _c.c_char_p lib.transcribe_full_text.argtypes = [_c.c_void_p] - lib.transcribe_get_backend_device.restype = _c.c_int - lib.transcribe_get_backend_device.argtypes = [_c.c_int, _c.POINTER(transcribe_backend_device)] lib.transcribe_get_model.restype = _c.c_void_p lib.transcribe_get_model.argtypes = [_c.c_void_p] lib.transcribe_get_segment.restype = _c.c_int @@ -315,12 +317,12 @@ def configure(lib): lib.transcribe_model_arch_string.argtypes = [_c.c_void_p] lib.transcribe_model_backend.restype = _c.c_char_p lib.transcribe_model_backend.argtypes = [_c.c_void_p] + lib.transcribe_model_device.restype = _c.c_void_p + lib.transcribe_model_device.argtypes = [_c.c_void_p] lib.transcribe_model_free.restype = None lib.transcribe_model_free.argtypes = [_c.c_void_p] lib.transcribe_model_get_capabilities.restype = _c.c_int lib.transcribe_model_get_capabilities.argtypes = [_c.c_void_p, _c.POINTER(transcribe_capabilities)] - lib.transcribe_model_get_device.restype = _c.c_int - lib.transcribe_model_get_device.argtypes = [_c.c_void_p, _c.POINTER(transcribe_backend_device)] lib.transcribe_model_load_file.restype = _c.c_int lib.transcribe_model_load_file.argtypes = [_c.c_char_p, _c.POINTER(transcribe_model_load_params), _c.POINTER(_c.c_void_p)] lib.transcribe_model_load_params_init.restype = None diff --git a/bindings/python/tests/test_backends.py b/bindings/python/tests/test_backends.py index 4d1bb3fa..ab57eb82 100644 --- a/bindings/python/tests/test_backends.py +++ b/bindings/python/tests/test_backends.py @@ -9,6 +9,8 @@ from __future__ import annotations +from dataclasses import replace + import pytest import transcribe_cpp as t @@ -31,8 +33,8 @@ def test_backends_non_empty(): def test_device_index_and_fields(): - # Each device carries its registry index (the value Model(..., gpu_device=) - # selects with) and well-formed metadata. Pin the device-selection surface. + # Each device carries a process-local display index and an opaque selection + # handle, plus well-formed metadata. Pin the device-selection surface. devices = t.backends() for i, dev in enumerate(devices): assert dev.index == i, f"device {i} reported index {dev.index}" @@ -46,6 +48,13 @@ def test_device_index_and_fields(): assert isinstance(dev.kind, str) and dev.kind +def test_device_equality_uses_native_identity(): + device = t.backends()[0] + refreshed = replace(device, memory_free=device.memory_free + 1, index=None) + assert refreshed == device + assert hash(refreshed) == hash(device) + + def test_cpu_always_available(): # Every shipped configuration includes a CPU backend (compiled in or as # the baseline module); a process without one is mispackaged. @@ -93,6 +102,6 @@ def test_init_backends_rejects_bad_dirs(): def test_init_backends_idempotent(): lib = t._lib adir = str(_library.artifact_dir()).encode("utf-8") - n = lib.transcribe_backend_device_count() + n = lib.transcribe_device_count() assert lib.transcribe_init_backends(adir) == 0 - assert lib.transcribe_backend_device_count() == n # no re-registration + assert lib.transcribe_device_count() == n # no re-registration diff --git a/bindings/python/tests/test_device_select.py b/bindings/python/tests/test_device_select.py index 9ca351c8..1c9f0b18 100644 --- a/bindings/python/tests/test_device_select.py +++ b/bindings/python/tests/test_device_select.py @@ -1,55 +1,38 @@ -"""Model-gated device-selection tests. - -These take the ``model_path`` / ``transcribe_cpp`` fixtures, which ``skip`` -when the default whisper-tiny.en asset is absent (override with -``TRANSCRIBE_SMOKE_MODEL``). They pin the device-selection surface added -alongside the per-device ``index`` field: ``Model.device`` reports where the -model landed (its ``.index`` is ``None`` because it did not come from -enumeration), and an out-of-range / negative ``gpu_device`` is rejected with -``InvalidArgument``. -""" - -from __future__ import annotations +"""Exact opaque-device selection tests.""" import pytest -import transcribe_cpp as t +def _primary_devices(transcribe_cpp): + return [d for d in transcribe_cpp.backends() if d.device_type != "accel"] -def test_model_device_matches_enumeration(transcribe_cpp, model_path): - # The model lands on some registered device. Model.device does not come - # from enumeration, so its .index is None; correlate it back to backends() - # by name (and by device_id when that is reported). - with transcribe_cpp.Model(model_path) as model: - dev = model.device - assert isinstance(dev, transcribe_cpp.BackendDevice) - assert dev.index is None, "Model.device should not carry a registry index" - devices = transcribe_cpp.backends() - by_name = [d for d in devices if d.name == dev.name] - assert by_name, ( - f"model device {dev.name!r} not found among backends() " - f"{[d.name for d in devices]}" - ) - if dev.device_id is not None: - assert any(d.device_id == dev.device_id for d in by_name), ( - f"model device_id {dev.device_id!r} matched no enumerated device" - ) +def test_enumerated_device_can_be_passed_to_model(transcribe_cpp, model_path): + devices = _primary_devices(transcribe_cpp) + if not devices: + pytest.skip("no selectable devices") + device = devices[0] + try: + with transcribe_cpp.Model(model_path, device=device) as model: + assert model.device == device + except transcribe_cpp.BackendError: + # Registered devices may still fail driver initialization. Exact + # selection must report that failure rather than choosing another. + pass -def test_negative_gpu_device_rejected(transcribe_cpp, model_path): - with pytest.raises(transcribe_cpp.InvalidArgument): - transcribe_cpp.Model(model_path, gpu_device=-1) - -def test_out_of_range_gpu_device_rejected(transcribe_cpp, model_path): - bad = len(transcribe_cpp.backends()) + 1000 - with pytest.raises(transcribe_cpp.InvalidArgument): - transcribe_cpp.Model(model_path, gpu_device=bad) +def test_device_argument_rejects_non_device(transcribe_cpp, model_path): + with pytest.raises(TypeError): + transcribe_cpp.Model(model_path, device=0) -def test_cpu_backend_with_gpu_index_rejected(transcribe_cpp, model_path): - # Hardware-independent: a CPU backend has no GPU to select, so a non-zero - # gpu_device is invalid regardless of what hardware is present. +def test_backend_must_match_explicit_device(transcribe_cpp, model_path): + gpu = next( + (d for d in transcribe_cpp.backends() if d.device_type in ("gpu", "igpu")), + None, + ) + if gpu is None: + pytest.skip("no GPU device") with pytest.raises(transcribe_cpp.InvalidArgument): - transcribe_cpp.Model(model_path, backend="cpu", gpu_device=1) + transcribe_cpp.Model(model_path, backend="cpu", device=gpu) diff --git a/bindings/rust/sys/README.md b/bindings/rust/sys/README.md index b992545b..ac88571f 100644 --- a/bindings/rust/sys/README.md +++ b/bindings/rust/sys/README.md @@ -4,10 +4,13 @@ Raw native FFI bindings for [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp), a C/C++ speech-to-text library built on ggml. -> **Status: in development (0.0.1).** This crate exposes the unsafe, generated +> **Status: in development (0.2.0).** This crate exposes the unsafe, generated > FFI surface. Most users want the safe wrapper, > [`transcribe-cpp`](https://crates.io/crates/transcribe-cpp). +Raw-FFI consumers upgrading from 0.1 should follow the +[0.2 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.2.md). + ## What it does `build.rs` compiles the vendored C++ tree from source via CMake (the crate diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index eebcf3b0..363cd6e3 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 7896d8d4c2a46147 +// Pinned to include/transcribe.abihash = 7df72bf9e667b8c2 /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "7896d8d4c2a46147"; +pub const PUBLIC_HEADER_HASH: &str = "7df72bf9e667b8c2"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -63,7 +63,7 @@ impl transcribe_abi_struct { pub const TRANSCRIBE_ABI_STREAM_TEXT: transcribe_abi_struct = transcribe_abi_struct(10); pub const TRANSCRIBE_ABI_SESSION_LIMITS: transcribe_abi_struct = transcribe_abi_struct(11); pub const TRANSCRIBE_ABI_EXT: transcribe_abi_struct = transcribe_abi_struct(12); - pub const TRANSCRIBE_ABI_BACKEND_DEVICE: transcribe_abi_struct = transcribe_abi_struct(13); + pub const TRANSCRIBE_ABI_DEVICE_INFO: transcribe_abi_struct = transcribe_abi_struct(13); pub const TRANSCRIBE_ABI_SPEAKER_SEGMENT: transcribe_abi_struct = transcribe_abi_struct(14); } #[repr(transparent)] @@ -213,8 +213,17 @@ unsafe extern "C" { unsafe extern "C" { pub fn transcribe_init_backends_default() -> transcribe_status; } +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct transcribe_device { + _unused: [u8; 0], +} +pub type transcribe_device_t = *mut transcribe_device; unsafe extern "C" { - pub fn transcribe_backend_device_count() -> ::std::os::raw::c_int; + pub fn transcribe_device_count() -> ::std::os::raw::c_int; +} +unsafe extern "C" { + pub fn transcribe_device_get(index: ::std::os::raw::c_int) -> transcribe_device_t; } impl transcribe_device_type { pub const TRANSCRIBE_DEVICE_TYPE_CPU: transcribe_device_type = transcribe_device_type(0); @@ -227,7 +236,7 @@ impl transcribe_device_type { pub struct transcribe_device_type(pub ::std::os::raw::c_uint); #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct transcribe_backend_device { +pub struct transcribe_device_info { pub struct_size: u64, pub name: *const ::std::os::raw::c_char, pub description: *const ::std::os::raw::c_char, @@ -239,64 +248,60 @@ pub struct transcribe_backend_device { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of transcribe_backend_device"] - [::std::mem::size_of::() - 64usize]; - ["Alignment of transcribe_backend_device"] - [::std::mem::align_of::() - 8usize]; - ["Offset of field: transcribe_backend_device::struct_size"] - [::std::mem::offset_of!(transcribe_backend_device, struct_size) - 0usize]; - ["Offset of field: transcribe_backend_device::name"] - [::std::mem::offset_of!(transcribe_backend_device, name) - 8usize]; - ["Offset of field: transcribe_backend_device::description"] - [::std::mem::offset_of!(transcribe_backend_device, description) - 16usize]; - ["Offset of field: transcribe_backend_device::kind"] - [::std::mem::offset_of!(transcribe_backend_device, kind) - 24usize]; - ["Offset of field: transcribe_backend_device::device_id"] - [::std::mem::offset_of!(transcribe_backend_device, device_id) - 32usize]; - ["Offset of field: transcribe_backend_device::memory_total"] - [::std::mem::offset_of!(transcribe_backend_device, memory_total) - 40usize]; - ["Offset of field: transcribe_backend_device::memory_free"] - [::std::mem::offset_of!(transcribe_backend_device, memory_free) - 48usize]; - ["Offset of field: transcribe_backend_device::device_type"] - [::std::mem::offset_of!(transcribe_backend_device, device_type) - 56usize]; + ["Size of transcribe_device_info"][::std::mem::size_of::() - 64usize]; + ["Alignment of transcribe_device_info"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_device_info::struct_size"] + [::std::mem::offset_of!(transcribe_device_info, struct_size) - 0usize]; + ["Offset of field: transcribe_device_info::name"] + [::std::mem::offset_of!(transcribe_device_info, name) - 8usize]; + ["Offset of field: transcribe_device_info::description"] + [::std::mem::offset_of!(transcribe_device_info, description) - 16usize]; + ["Offset of field: transcribe_device_info::kind"] + [::std::mem::offset_of!(transcribe_device_info, kind) - 24usize]; + ["Offset of field: transcribe_device_info::device_id"] + [::std::mem::offset_of!(transcribe_device_info, device_id) - 32usize]; + ["Offset of field: transcribe_device_info::memory_total"] + [::std::mem::offset_of!(transcribe_device_info, memory_total) - 40usize]; + ["Offset of field: transcribe_device_info::memory_free"] + [::std::mem::offset_of!(transcribe_device_info, memory_free) - 48usize]; + ["Offset of field: transcribe_device_info::device_type"] + [::std::mem::offset_of!(transcribe_device_info, device_type) - 56usize]; }; unsafe extern "C" { - pub fn transcribe_backend_device_init(p: *mut transcribe_backend_device); + pub fn transcribe_device_info_init(p: *mut transcribe_device_info); } unsafe extern "C" { - pub fn transcribe_get_backend_device( - index: ::std::os::raw::c_int, - out: *mut transcribe_backend_device, + pub fn transcribe_device_get_info( + device: transcribe_device_t, + out: *mut transcribe_device_info, ) -> transcribe_status; } unsafe extern "C" { pub fn transcribe_backend_available(kind: transcribe_backend_request) -> bool; } unsafe extern "C" { - pub fn transcribe_model_get_device( - model: *const transcribe_model, - out: *mut transcribe_backend_device, - ) -> transcribe_status; + pub fn transcribe_model_device(model: *const transcribe_model) -> transcribe_device_t; } #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct transcribe_model_load_params { pub struct_size: u64, pub backend: transcribe_backend_request, - pub gpu_device: ::std::os::raw::c_int, + pub device: transcribe_device_t, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of transcribe_model_load_params"] - [::std::mem::size_of::() - 16usize]; + [::std::mem::size_of::() - 24usize]; ["Alignment of transcribe_model_load_params"] [::std::mem::align_of::() - 8usize]; ["Offset of field: transcribe_model_load_params::struct_size"] [::std::mem::offset_of!(transcribe_model_load_params, struct_size) - 0usize]; ["Offset of field: transcribe_model_load_params::backend"] [::std::mem::offset_of!(transcribe_model_load_params, backend) - 8usize]; - ["Offset of field: transcribe_model_load_params::gpu_device"] - [::std::mem::offset_of!(transcribe_model_load_params, gpu_device) - 12usize]; + ["Offset of field: transcribe_model_load_params::device"] + [::std::mem::offset_of!(transcribe_model_load_params, device) - 16usize]; }; unsafe extern "C" { pub fn transcribe_model_load_params_init(params: *mut transcribe_model_load_params); diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index 81e2c85e..2720d429 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -4,9 +4,14 @@ Safe, idiomatic Rust bindings for [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp), a C/C++ speech-to-text library built on ggml. -> **Status: in development (0.0.1).** Core model, session, run, stream, +> **Status: in development (0.2.0).** Core model, session, run, stream, > cancellation, backend, and family-extension APIs are implemented and tested. +Upgrading from 0.1? See the +[0.2 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.2.md), +including the replacement of `ModelOptions::gpu_device` with exact `Device` +handles. + ## Install ```sh @@ -60,6 +65,17 @@ available through `shared` and `dynamic-backends`; see the `transcribe-cpp-sys` README if you need runtime-loaded backend modules or custom `TRANSCRIBE_CMAKE_ARGS`. +## Exact device selection + +`devices()` returns process-local `Device` handles. Leave +`ModelOptions::device` as `None` for the backend's automatic policy, or pass +`Some(device)` to select that exact primary device with no fallback. Persist +`device_id` and resolve a fresh handle after backend initialization; registry +indices and handles are not stable across processes. In dynamic-backend builds, +finish `init_backends()` or `init_backends_default()` before any thread +enumerates devices, queries backend availability, or loads a model; native +registry mutation is a startup-only operation and must not race those calls. + ## Packaging a distributable (`shared` / `dynamic-backends`) With the **default static** build there is nothing to do — the native code is diff --git a/bindings/rust/transcribe-cpp/examples/backend-select.rs b/bindings/rust/transcribe-cpp/examples/backend-select.rs index cc7d303b..4f5539dd 100644 --- a/bindings/rust/transcribe-cpp/examples/backend-select.rs +++ b/bindings/rust/transcribe-cpp/examples/backend-select.rs @@ -1,11 +1,10 @@ -//! backend-select — device discovery, explicit `backend=`, graceful failure. +//! backend-select — device discovery, exact selection, backend-policy failure. //! //! cargo run --example backend-select -- [model.gguf] //! -//! Device discovery needs no model and always runs. With a model it shows an -//! explicit, satisfiable request (`Backend::Auto`) and, to demonstrate the -//! degradation contract, an explicit request for a backend this build lacks — -//! which must error cleanly from the request path, not crash. +//! Device discovery needs no model and always runs. With a model it selects the +//! enumerated CPU device exactly, then demonstrates that an unavailable backend +//! policy fails cleanly from the request path rather than crashing. #[path = "common/mod.rs"] mod common; @@ -21,8 +20,9 @@ fn main() -> Result<(), Box> { // happen once, before the first model load. init_backends_default()?; + let registered = devices(); println!("registered compute devices:"); - for d in devices() { + for d in ®istered { println!(" {} [{}] — {}", d.name, d.kind, d.description); } println!("\nbackend availability:"); @@ -54,18 +54,26 @@ fn main() -> Result<(), Box> { return Ok(()); }; - // Explicit, satisfiable request: Auto resolves to the best available device. + // Exact selection uses a process-local Device handle, not its display + // index. CPU is always registered and provides a deterministic example. + let cpu = registered + .iter() + .find(|device| device.kind == "cpu") + .expect("the CPU device must be registered") + .clone(); let model = Model::load_with( &model_path, &ModelOptions { - backend: Backend::Auto, + device: Some(cpu.clone()), ..Default::default() }, )?; println!( - "\nloaded with Backend::Auto -> bound backend: {}", + "\nselected exact device {} -> bound backend: {}", + model.device()?.name, model.backend() ); + assert_eq!(model.device()?, cpu); drop(model); // Graceful failure: requesting an unavailable backend must error cleanly. diff --git a/bindings/rust/transcribe-cpp/src/backend.rs b/bindings/rust/transcribe-cpp/src/backend.rs index e75296f9..97986403 100644 --- a/bindings/rust/transcribe-cpp/src/backend.rs +++ b/bindings/rust/transcribe-cpp/src/backend.rs @@ -52,7 +52,7 @@ impl DeviceType { } /// One registered compute device. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct Device { /// ggml device name, e.g. "Metal". pub name: String, @@ -73,21 +73,35 @@ pub struct Device { /// [`crate::Model::device`]) to refresh it; the value is backend-defined /// and not comparable across device kinds. pub memory_free: u64, - /// Registry index of this device — the value to pass as - /// [`ModelOptions::gpu_device`](crate::ModelOptions) to select it (0 - /// means auto: discrete GPUs are probed before integrated). `None` when - /// this `Device` came from [`crate::Model::device`], since - /// `transcribe_model_get_device` does not expose an index; correlate such - /// a device back to [`devices`] by `device_id` / `name` instead. - /// Order-dependent and not stable across driver updates or hosts. + /// Registry index when this device came from [`devices`]. This is useful + /// for display only; pass the [`Device`] itself to [`ModelOptions`] for + /// exact selection. Registry indices are not stable across processes. pub index: Option, + pub(crate) handle: sys::transcribe_device_t, } +// Device handles are immutable process-lifetime registry entries. Backend +// registration must finish before enumeration, so sharing a handle is safe. +unsafe impl Send for Device {} +unsafe impl Sync for Device {} + +impl PartialEq for Device { + fn eq(&self, other: &Self) -> bool { + self.handle == other.handle + } +} + +impl Eq for Device {} + impl Device { /// Build a [`Device`] from the raw FFI struct filled by the library. /// `index` is the registry index when the device came from enumeration, - /// or `None` when it came from `transcribe_model_get_device`. - pub(crate) fn from_raw(raw: &sys::transcribe_backend_device, index: Option) -> Device { + /// or `None` when it came from a loaded model. + pub(crate) fn from_raw( + raw: &sys::transcribe_device_info, + handle: sys::transcribe_device_t, + index: Option, + ) -> Device { Device { name: owned_str(raw.name), description: owned_str(raw.description), @@ -97,6 +111,7 @@ impl Device { memory_total: raw.memory_total, memory_free: raw.memory_free, index, + handle, } } } @@ -108,6 +123,11 @@ impl Device { /// registered compute device (a dynamic build pointed at a directory with no /// usable modules). This call is idempotent per directory and NOT retryable in /// the same process. +/// +/// This mutates the native process-global device registry. Complete every +/// backend-init call before other threads enumerate devices, query backend +/// availability, or load models; the native registry does not support racing +/// registration against those operations. pub fn init_backends(dir: impl AsRef) -> Result<()> { let dir = dir.as_ref(); // Pass the path bytes through faithfully (Unix) / reject non-UTF-8 (Windows), @@ -128,27 +148,39 @@ pub fn init_backends(dir: impl AsRef) -> Result<()> { /// /// If your app uses a different layout, call [`init_backends`] with that /// resolved module directory instead. Like [`init_backends`], this is -/// idempotent and must run once before the first model load. +/// idempotent and must run once before the first model load. It must also +/// complete before other threads enumerate devices or query backend +/// availability; see [`init_backends`] for the registry-ordering contract. pub fn init_backends_default() -> Result<()> { let status = unsafe { sys::transcribe_init_backends_default() }; check(status, "init_backends_default") } /// The number of compute devices currently registered. +/// +/// Do not race this query with [`init_backends`] or [`init_backends_default`]. pub fn device_count() -> usize { - let n = unsafe { sys::transcribe_backend_device_count() }; + let n = unsafe { sys::transcribe_device_count() }; n.max(0) as usize } /// Every registered compute device. +/// +/// Do not race enumeration with [`init_backends`] or +/// [`init_backends_default`]. Finish backend registration before sharing +/// devices across threads. pub fn devices() -> Vec { let mut out = Vec::with_capacity(device_count()); for i in 0..device_count() as i32 { - let mut raw: sys::transcribe_backend_device = unsafe { std::mem::zeroed() }; - unsafe { sys::transcribe_backend_device_init(&mut raw) }; - let status = unsafe { sys::transcribe_get_backend_device(i, &mut raw) }; + let handle = unsafe { sys::transcribe_device_get(i) }; + if handle.is_null() { + continue; + } + let mut raw: sys::transcribe_device_info = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_device_info_init(&mut raw) }; + let status = unsafe { sys::transcribe_device_get_info(handle, &mut raw) }; if status == sys::transcribe_status::TRANSCRIBE_OK { - out.push(Device::from_raw(&raw, Some(i as usize))); + out.push(Device::from_raw(&raw, handle, Some(i as usize))); } } out @@ -156,7 +188,8 @@ pub fn devices() -> Vec { /// Whether a backend request can be satisfied by some registered device. This /// is the probe to turn `Backend::Vulkan` on a machine without Vulkan into a -/// clear error instead of a failed model load. +/// clear error instead of a failed model load. Do not race this query with +/// [`init_backends`] or [`init_backends_default`]. pub fn backend_available(backend: Backend) -> bool { unsafe { sys::transcribe_backend_available(backend.to_raw()) } } diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index 10c14cd9..0e9a4e32 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -30,18 +30,19 @@ use crate::version; /// Options for loading a model. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ModelOptions { - /// Which backend to request. Default [`Backend::Auto`]. + /// Which backend to request. Default [`Backend::Auto`]. An explicit device + /// must match a non-Auto backend request. pub backend: Backend, - /// GPU device registry index. 0 means auto: the first device that - /// initializes, probing discrete GPUs before integrated. - pub gpu_device: i32, + /// Exact process-local compute device. `None` applies the backend's + /// automatic policy; `Some` selects that device or fails without fallback. + pub device: Option, } impl Default for ModelOptions { fn default() -> Self { ModelOptions { backend: Backend::Auto, - gpu_device: 0, + device: None, } } } @@ -121,7 +122,10 @@ impl Model { let mut params: sys::transcribe_model_load_params = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_model_load_params_init(&mut params) }; params.backend = options.backend.to_raw(); - params.gpu_device = options.gpu_device; + params.device = options + .device + .as_ref() + .map_or(std::ptr::null_mut(), |device| device.handle); let mut out: *mut sys::transcribe_model = std::ptr::null_mut(); let status = unsafe { sys::transcribe_model_load_file(c_path.as_ptr(), ¶ms, &mut out) }; @@ -219,11 +223,17 @@ impl Model { /// much memory is left on the device after the model loaded. Errors with /// [`Error::Backend`](crate::Error) if the model has no resolved device. pub fn device(&self) -> Result { - let mut raw: sys::transcribe_backend_device = unsafe { std::mem::zeroed() }; - unsafe { sys::transcribe_backend_device_init(&mut raw) }; - let status = unsafe { sys::transcribe_model_get_device(self.inner.ptr, &mut raw) }; - check(status, "model_get_device")?; - Ok(Device::from_raw(&raw, None)) + let handle = unsafe { sys::transcribe_model_device(self.inner.ptr) }; + if handle.is_null() { + return Err(crate::Error::Backend( + "model has no resolved device".to_string(), + )); + } + let mut raw: sys::transcribe_device_info = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_device_info_init(&mut raw) }; + let status = unsafe { sys::transcribe_device_get_info(handle, &mut raw) }; + check(status, "device_get_info")?; + Ok(Device::from_raw(&raw, handle, None)) } /// Tokenize plain UTF-8 text into the model's vocabulary (no BOS/EOS, no diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index c390af7c..02089439 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -288,7 +288,7 @@ pub enum AbiStruct { StreamText, SessionLimits, Ext, - BackendDevice, + DeviceInfo, SpeakerSegment, } @@ -309,7 +309,7 @@ impl AbiStruct { AbiStruct::StreamText => A::TRANSCRIBE_ABI_STREAM_TEXT, AbiStruct::SessionLimits => A::TRANSCRIBE_ABI_SESSION_LIMITS, AbiStruct::Ext => A::TRANSCRIBE_ABI_EXT, - AbiStruct::BackendDevice => A::TRANSCRIBE_ABI_BACKEND_DEVICE, + AbiStruct::DeviceInfo => A::TRANSCRIBE_ABI_DEVICE_INFO, AbiStruct::SpeakerSegment => A::TRANSCRIBE_ABI_SPEAKER_SEGMENT, } } diff --git a/bindings/rust/transcribe-cpp/tests/device_select.rs b/bindings/rust/transcribe-cpp/tests/device_select.rs index 8460acae..ca4447e4 100644 --- a/bindings/rust/transcribe-cpp/tests/device_select.rs +++ b/bindings/rust/transcribe-cpp/tests/device_select.rs @@ -1,92 +1,60 @@ -//! Model-gated device-selection tests. Exercise the `ModelOptions.gpu_device` -//! registry selector and the `Model::device()` correlation back to `devices()`. -//! Skip cleanly (via the `common::` fixtures) when the canary GGUF is absent. +//! Model-gated exact-device selection tests. mod common; use transcribe_cpp::{devices, Backend, Error, Model, ModelOptions}; #[test] -fn loaded_model_reports_an_enumerated_device() { - // A model loaded with default options resolves a device. That Device has no - // registry index (transcribe_model_get_device does not expose one), and it - // correlates back to a device from devices() by name (+ device_id when set). - let Some((model_path, _)) = common::smoke_fixtures("loaded_model_reports_an_enumerated_device") +fn every_primary_device_can_be_selected_exactly() { + let Some((model_path, _)) = + common::smoke_fixtures("every_primary_device_can_be_selected_exactly") else { return; }; - let model = Model::load(&model_path).unwrap(); - let dev = model.device().expect("model.device()"); - - // A model-resolved device never carries a registry index. - assert_eq!(dev.index, None, "{dev:?}"); - // It must match one of the enumerated devices by name, and by device_id - // when the model device reports one. - let all = devices(); - let matched = all - .iter() - .any(|d| d.name == dev.name && (dev.device_id.is_none() || d.device_id == dev.device_id)); - assert!( - matched, - "model device {dev:?} not found among enumerated devices {all:?}" - ); + for device in devices() + .into_iter() + .filter(|d| d.device_type != transcribe_cpp::DeviceType::Accel) + { + let result = Model::load_with( + &model_path, + &ModelOptions { + backend: Backend::Auto, + device: Some(device.clone()), + }, + ); + match result { + Ok(model) => assert_eq!(model.device().unwrap(), device), + Err(Error::Backend(_)) => { + // A registered device may still fail initialization on the + // current driver. Exact selection must fail rather than move. + } + Err(error) => panic!("unexpected exact-device error: {error}"), + } + } } #[test] -fn negative_gpu_device_is_invalid_argument() { - // gpu_device must be >= 0; -1 is rejected before any device lookup. - let Some((model_path, _)) = common::smoke_fixtures("negative_gpu_device_is_invalid_argument") - else { +fn explicit_backend_must_match_device() { + let Some((model_path, _)) = common::smoke_fixtures("explicit_backend_must_match_device") else { return; }; - let err = Model::load_with( - &model_path, - &ModelOptions { - gpu_device: -1, - ..Default::default() - }, - ) - .unwrap_err(); - assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); -} - -#[test] -fn out_of_range_gpu_device_is_invalid_argument() { - // A registry index well past the device count is out of range. - let Some((model_path, _)) = - common::smoke_fixtures("out_of_range_gpu_device_is_invalid_argument") - else { + let Some(gpu) = devices().into_iter().find(|d| { + matches!( + d.device_type, + transcribe_cpp::DeviceType::Gpu | transcribe_cpp::DeviceType::Igpu + ) + }) else { return; }; - let out_of_range = devices().len() as i32 + 1000; - let err = Model::load_with( - &model_path, - &ModelOptions { - gpu_device: out_of_range, - ..Default::default() - }, - ) - .unwrap_err(); - assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); -} -#[test] -fn gpu_device_with_cpu_backend_is_invalid_argument() { - // Selecting a device index under a strict-CPU backend request is a category - // error (there is no GPU to select) — hardware-independent. - let Some((model_path, _)) = - common::smoke_fixtures("gpu_device_with_cpu_backend_is_invalid_argument") - else { - return; - }; - let err = Model::load_with( + let error = Model::load_with( &model_path, &ModelOptions { backend: Backend::Cpu, - gpu_device: 1, + device: Some(gpu), }, ) .unwrap_err(); - assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); + assert!(matches!(error, Error::InvalidArgument(_))); } diff --git a/bindings/swift/README.md b/bindings/swift/README.md index 8c861bfd..0b32a47f 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -5,9 +5,13 @@ a C/C++ speech-to-text library built on ggml. Native code ships as a prebuilt `.xcframework` SwiftPM `binaryTarget`, with Metal embedded on supported Apple slices. -> Status: in development (0.0.1). Core model, session, run, stream, +> Status: in development (0.2.0). Core model, session, run, stream, > cancellation, backend, and family-extension APIs are implemented and tested. +Upgrading from 0.1? See the +[0.2 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.2.md), +including the replacement of `gpuDevice` with exact `Device` values. + ## Install Apple platforms only: **macOS 13+** and **iOS 16+**. @@ -19,7 +23,7 @@ custom artifact path through `TRANSCRIBE_XCFRAMEWORK_PATH`. The standalone SwiftPM mirror is planned but not published yet: ```swift -.package(url: "https://github.com/handy-computer/transcribe-cpp-swift.git", from: "0.0.1") +.package(url: "https://github.com/handy-computer/transcribe-cpp-swift.git", from: "0.2.0") ``` Until that mirror repo and tag exist, use the release xcframework directly when @@ -28,7 +32,7 @@ you only need the raw C module: ```swift .binaryTarget( name: "CTranscribe", - url: "https://github.com/handy-computer/transcribe.cpp/releases/download/v0.0.1/TranscribeCpp.xcframework.zip", + url: "https://github.com/handy-computer/transcribe.cpp/releases/download/v0.2.0/TranscribeCpp.xcframework.zip", checksum: "" ) ``` @@ -81,8 +85,10 @@ Backends are compiled into the xcframework per Apple slice: | iOS device arm64 | Metal + CPU | | iOS simulator | CPU only | -Request a backend with `ModelOptions(backend:)`; probe availability with -`Transcribe.backendAvailable(_:)` or inspect `Transcribe.devices()`. +Request a backend policy with `ModelOptions(backend:)`; probe availability with +`Transcribe.backendAvailable(_:)`. For exact selection, pass an entry from +`Transcribe.devices()` to `ModelOptions(device:)`; exact selection never falls +back to another primary device. ## Concurrency and lifetime diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index aa501f65..5ea6cbc1 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "7896d8d4c2a46147" + public static let pinnedHeaderHash = "7df72bf9e667b8c2" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Backend.swift b/bindings/swift/Sources/TranscribeCpp/Backend.swift index 3298887f..9aeef03f 100644 --- a/bindings/swift/Sources/TranscribeCpp/Backend.swift +++ b/bindings/swift/Sources/TranscribeCpp/Backend.swift @@ -51,7 +51,7 @@ public enum DeviceType: Sendable, Equatable { } /// A registered compute device. -public struct Device: Sendable, Equatable { +public struct Device: @unchecked Sendable, Equatable { /// ggml device name, e.g. "Metal". public let name: String /// Human-readable description, e.g. "Apple M4 Max". @@ -69,17 +69,14 @@ public struct Device: Sendable, Equatable { /// unreported. Re-query (`TranscribeCpp.devices()` or `Model.device`) to /// refresh; backend-defined and not comparable across device kinds. public let memoryFree: UInt64 - /// Registry index of this device — the value to pass as - /// `ModelOptions(gpuDevice:)` to select it (0 selects the auto / first - /// device). `nil` when this `Device` came from `Model.device`, since - /// `transcribe_model_get_device` does not expose an index; correlate such a - /// device back to `devices()` by `deviceId` / `name` instead. - /// Order-dependent and not stable across driver updates or hosts. + /// Process-local registry index for display. Pass this `Device` through + /// `ModelOptions(device:)` for exact selection; persist `deviceId` instead. public let index: Int? + let handle: transcribe_device_t /// Build from the raw C struct the library filled. `index` is the registry /// index when the device came from enumeration, else nil. - init(_ raw: transcribe_backend_device, index: Int? = nil) { + init(_ raw: transcribe_device_info, handle: transcribe_device_t, index: Int? = nil) { name = raw.name.map { String(cString: $0) } ?? "" description = raw.description.map { String(cString: $0) } ?? "" kind = raw.kind.map { String(cString: $0) } ?? "" @@ -88,6 +85,11 @@ public struct Device: Sendable, Equatable { memoryTotal = raw.memory_total memoryFree = raw.memory_free self.index = index + self.handle = handle + } + + public static func == (lhs: Device, rhs: Device) -> Bool { + lhs.handle == rhs.handle } } @@ -107,7 +109,7 @@ public enum AbiStruct: Sendable { case streamText case sessionLimits case ext - case backendDevice + case deviceInfo var cValue: transcribe_abi_struct { switch self { @@ -124,7 +126,7 @@ public enum AbiStruct: Sendable { case .streamText: return TRANSCRIBE_ABI_STREAM_TEXT case .sessionLimits: return TRANSCRIBE_ABI_SESSION_LIMITS case .ext: return TRANSCRIBE_ABI_EXT - case .backendDevice: return TRANSCRIBE_ABI_BACKEND_DEVICE + case .deviceInfo: return TRANSCRIBE_ABI_DEVICE_INFO } } } diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index 506a00c5..6045b17b 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -27,7 +27,7 @@ public final class Model: @unchecked Sendable { var params = transcribe_model_load_params() transcribe_model_load_params_init(¶ms) params.backend = options.backend.cValue - params.gpu_device = options.gpuDevice + params.device = options.device?.handle var out: OpaquePointer? let status = transcribe_model_load_file(path, ¶ms, &out) try TranscribeError.check(status, context: "loading \(path)") @@ -79,11 +79,14 @@ public final class Model: @unchecked Sendable { /// has no resolved compute device. public var device: Device { get throws { - var raw = transcribe_backend_device() - transcribe_backend_device_init(&raw) + guard let handle = transcribe_model_device(ptr) else { + throw TranscribeError.backend("model has no resolved compute device") + } + var raw = transcribe_device_info() + transcribe_device_info_init(&raw) try TranscribeError.check( - transcribe_model_get_device(ptr, &raw), context: "model_get_device") - return Device(raw) + transcribe_device_get_info(handle, &raw), context: "device_get_info") + return Device(raw, handle: handle) } } diff --git a/bindings/swift/Sources/TranscribeCpp/Options.swift b/bindings/swift/Sources/TranscribeCpp/Options.swift index 36a957d6..10ff81f3 100644 --- a/bindings/swift/Sources/TranscribeCpp/Options.swift +++ b/bindings/swift/Sources/TranscribeCpp/Options.swift @@ -99,11 +99,11 @@ public enum Feature: Sendable { public struct ModelOptions: Sendable { public var backend: Backend - /// GPU device registry index. 0 means auto / first matching device. - public var gpuDevice: Int32 - public init(backend: Backend = .auto, gpuDevice: Int32 = 0) { + /// Exact process-local device. `nil` applies the backend's automatic policy. + public var device: Device? + public init(backend: Backend = .auto, device: Device? = nil) { self.backend = backend - self.gpuDevice = gpuDevice + self.device = device } } diff --git a/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift b/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift index f190aa28..3b97be10 100644 --- a/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift +++ b/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift @@ -66,14 +66,15 @@ public enum Transcribe { /// The compute devices the native library has registered. public static func devices() -> [Device] { - let count = transcribe_backend_device_count() + let count = transcribe_device_count() var devices: [Device] = [] devices.reserveCapacity(Int(count)) for index in 0..`); there is nothing to compile and no environment variables to set. +Upgrading from 0.1? See the +[0.2 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.2.md), +including the replacement of `gpuDevice` with exact device objects. + ## Quickstart ```ts @@ -101,10 +105,15 @@ the model lease). Disposal is idempotent and order-independent. ```ts import { getAvailableBackends, backendAvailable } from "transcribe-cpp"; -getAvailableBackends(); // [{ kind: "metal", name: "MTL0", description: "…" }, …] +const devices = getAvailableBackends(); backendAvailable("rocm"); // boolean — never throws -const model = await TranscribeModel.load("model.gguf", { backend: "rocm" }); +// Policy selection: first matching ROCm device. +const automatic = await TranscribeModel.load("model.gguf", { backend: "rocm" }); +// Exact selection: use this process-local CPU device or fail without fallback. +const cpu = devices.find((device) => device.deviceType === "cpu"); +if (!cpu) throw new Error("CPU device is not registered"); +const exact = await TranscribeModel.load("model.gguf", { device: cpu }); ``` `backend` defaults to `"auto"` (best accelerator, else CPU). A missing Vulkan diff --git a/bindings/typescript/examples/backend-select.mjs b/bindings/typescript/examples/backend-select.mjs index f266fdc7..e7d4b975 100644 --- a/bindings/typescript/examples/backend-select.mjs +++ b/bindings/typescript/examples/backend-select.mjs @@ -1,15 +1,15 @@ -// backend-select — discover devices, request an explicit backend, fall back. +// backend-select — discover devices, select exactly, show policy failure. // // node examples/backend-select.mjs [model.gguf] // -// Device discovery needs no model; the explicit-backend load is skipped when no -// model is given. +// Device discovery needs no model; model loads are skipped when none is given. import { getAvailableBackends, backendAvailable, TranscribeModel } from "../dist/index.js"; import { model, skip } from "./_support.mjs"; +const devices = getAvailableBackends(); console.log("discovered devices:"); -for (const d of getAvailableBackends()) { +for (const d of devices) { console.log(` ${d.kind.padEnd(7)} ${d.name} — ${d.description}`); } console.log("\nbackend availability:"); @@ -20,21 +20,26 @@ for (const b of ["cpu", "metal", "vulkan", "cuda", "rocm"]) { const path = model("TRANSCRIBE_SMOKE_MODEL"); if (!path) skip("\nno model — device discovery only (set TRANSCRIBE_SMOKE_MODEL to load)"); -// Prefer an accelerator, fall back to CPU on a clean failure. -const preferred = backendAvailable("metal") - ? "metal" - : backendAvailable("cuda") - ? "cuda" - : backendAvailable("rocm") - ? "rocm" - : "cpu"; -console.log(`\nrequesting backend: ${preferred}`); -let m; -try { - m = await TranscribeModel.load(path, { backend: preferred }); -} catch (e) { - console.log(` ${preferred} unavailable (${e.constructor.name}); retrying on cpu`); - m = await TranscribeModel.load(path, { backend: "cpu" }); +// Exact selection passes an object returned by getAvailableBackends(), not its +// display index. CPU provides a deterministic example on every platform. +const cpu = devices.find((device) => device.deviceType === "cpu"); +if (!cpu) skip("no selectable CPU device"); +const exact = await TranscribeModel.load(path, { device: cpu }); +console.log(`\nselected exact device ${exact.device.name}, bound to: ${exact.backend}`); +exact.dispose(); + +// An explicit backend policy that cannot be satisfied fails cleanly. +const unavailable = ["cuda", "rocm", "vulkan", "metal"].find( + (backend) => !backendAvailable(backend), +); +if (unavailable) { + try { + const unexpected = await TranscribeModel.load(path, { backend: unavailable }); + unexpected.dispose(); + console.log(`requesting ${unavailable} unexpectedly succeeded`); + } catch (error) { + console.log(`requesting ${unavailable} failed cleanly: ${error.message}`); + } +} else { + console.log("every optional backend is available on this build"); } -console.log(`loaded on backend: ${m.backend}`); -m.dispose(); diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index ab5d280d..fff2ca59 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "7896d8d4c2a46147"; +export const PUBLIC_HEADER_HASH = "7df72bf9e667b8c2"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -46,7 +46,7 @@ export const TRANSCRIBE_ABI_STREAM_UPDATE = 9; export const TRANSCRIBE_ABI_STREAM_TEXT = 10; export const TRANSCRIBE_ABI_SESSION_LIMITS = 11; export const TRANSCRIBE_ABI_EXT = 12; -export const TRANSCRIBE_ABI_BACKEND_DEVICE = 13; +export const TRANSCRIBE_ABI_DEVICE_INFO = 13; export const TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14; export const TRANSCRIBE_LOG_LEVEL_NONE = 0; export const TRANSCRIBE_LOG_LEVEL_INFO = 1; @@ -118,8 +118,8 @@ export const TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319; export interface StructLayout { size: number; align: number; offsets: Record; } export const STRUCT_LAYOUT: Record = { 'transcribe_ext': { size: 16, align: 8, offsets: {'size': 0, 'kind': 8} }, - 'transcribe_backend_device': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} }, - 'transcribe_model_load_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'gpu_device': 12} }, + 'transcribe_device_info': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} }, + 'transcribe_model_load_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'device': 16} }, 'transcribe_session_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16} }, 'transcribe_run_params': { size: 72, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'diarize': 24, 'language': 32, 'target_language': 40, 'keep_special_tags': 48, 'family': 56, 'spec_k_drafts': 64} }, 'transcribe_capabilities': { size: 56, align: 8, offsets: {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48} }, @@ -143,7 +143,7 @@ export const STRUCT_LAYOUT: Record = { export const ABI_STRUCT_IDS: Record = { 'transcribe_ext': 12, - 'transcribe_backend_device': 13, + 'transcribe_device_info': 13, 'transcribe_model_load_params': 0, 'transcribe_session_params': 1, 'transcribe_run_params': 2, @@ -163,8 +163,8 @@ export const ABI_STRUCT_IDS: Record = { export function defineTypes(koffi: any): Record { const T: Record = {}; T['transcribe_ext'] = koffi.struct({ size: 'uint64_t', kind: 'uint32_t' }); - T['transcribe_backend_device'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' }); - T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', gpu_device: 'int' }); + T['transcribe_device_info'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' }); + T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', device: 'void *' }); T['transcribe_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int', kv_type: 'int', n_ctx: 'int32_t' }); T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', diarize: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t' }); T['transcribe_capabilities'] = koffi.struct({ struct_size: 'uint64_t', native_sample_rate: 'int32_t', n_languages: 'int', languages: 'void *', max_timestamp_kind: 'int', supports_language_detect: 'bool', supports_translate: 'bool', supports_streaming: 'bool', supports_spec_decode: 'bool', max_audio_ms: 'int64_t', n_translate_target_languages: 'int', translate_target_languages: 'void *' }); @@ -192,8 +192,6 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_abi_struct_align': { ret: 'size_t', args: ['transcribe_abi_struct'] }, 'transcribe_abi_struct_size': { ret: 'size_t', args: ['transcribe_abi_struct'] }, 'transcribe_backend_available': { ret: '_Bool', args: ['transcribe_backend_request'] }, - 'transcribe_backend_device_count': { ret: 'int', args: [] }, - 'transcribe_backend_device_init': { ret: 'void', args: ['struct transcribe_backend_device *'] }, 'transcribe_batch_detected_language': { ret: 'const char *', args: ['const struct transcribe_session *', 'int'] }, 'transcribe_batch_full_text': { ret: 'const char *', args: ['const struct transcribe_session *', 'int'] }, 'transcribe_batch_get_segment': { ret: 'transcribe_status', args: ['const struct transcribe_session *', 'int', 'int', 'struct transcribe_segment *'] }, @@ -212,9 +210,12 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_capabilities_init': { ret: 'void', args: ['struct transcribe_capabilities *'] }, 'transcribe_close': { ret: 'void', args: ['struct transcribe_session *'] }, 'transcribe_detected_language': { ret: 'const char *', args: ['const struct transcribe_session *'] }, + 'transcribe_device_count': { ret: 'int', args: [] }, + 'transcribe_device_get': { ret: 'transcribe_device_t', args: ['int'] }, + 'transcribe_device_get_info': { ret: 'transcribe_status', args: ['transcribe_device_t', 'struct transcribe_device_info *'] }, + 'transcribe_device_info_init': { ret: 'void', args: ['struct transcribe_device_info *'] }, 'transcribe_ext_check': { ret: 'transcribe_status', args: ['const struct transcribe_ext *', 'uint32_t', 'uint64_t'] }, 'transcribe_full_text': { ret: 'const char *', args: ['const struct transcribe_session *'] }, - 'transcribe_get_backend_device': { ret: 'transcribe_status', args: ['int', 'struct transcribe_backend_device *'] }, 'transcribe_get_model': { ret: 'const struct transcribe_model *', args: ['const struct transcribe_session *'] }, 'transcribe_get_segment': { ret: 'transcribe_status', args: ['const struct transcribe_session *', 'int', 'struct transcribe_segment *'] }, 'transcribe_get_speaker_segment': { ret: 'transcribe_status', args: ['const struct transcribe_session *', 'int', 'struct transcribe_speaker_segment *'] }, @@ -229,9 +230,9 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_model_accepts_ext_kind': { ret: '_Bool', args: ['const struct transcribe_model *', 'transcribe_ext_slot', 'uint32_t'] }, 'transcribe_model_arch_string': { ret: 'const char *', args: ['const struct transcribe_model *'] }, 'transcribe_model_backend': { ret: 'const char *', args: ['const struct transcribe_model *'] }, + 'transcribe_model_device': { ret: 'transcribe_device_t', args: ['const struct transcribe_model *'] }, 'transcribe_model_free': { ret: 'void', args: ['struct transcribe_model *'] }, 'transcribe_model_get_capabilities': { ret: 'transcribe_status', args: ['const struct transcribe_model *', 'struct transcribe_capabilities *'] }, - 'transcribe_model_get_device': { ret: 'transcribe_status', args: ['const struct transcribe_model *', 'struct transcribe_backend_device *'] }, 'transcribe_model_load_file': { ret: 'transcribe_status', args: ['const char *', 'const struct transcribe_model_load_params *', 'struct transcribe_model **'] }, 'transcribe_model_load_params_init': { ret: 'void', args: ['struct transcribe_model_load_params *'] }, 'transcribe_model_meta_val_str': { ret: 'const char *', args: ['const struct transcribe_model *', 'const char *'] }, diff --git a/bindings/typescript/src/ffi.ts b/bindings/typescript/src/ffi.ts index e0d1ec11..938a436c 100644 --- a/bindings/typescript/src/ffi.ts +++ b/bindings/typescript/src/ffi.ts @@ -39,13 +39,14 @@ export function bindLibrary(libraryPath: string): Bound { // backends initBackends: lib.func("transcribe_init_backends", "int", ["str"]), initBackendsDefault: lib.func("transcribe_init_backends_default", "int", []), - backendDeviceCount: lib.func("transcribe_backend_device_count", "int", []), - backendDeviceInit: lib.func("transcribe_backend_device_init", "void", [ - outp(T.transcribe_backend_device), + deviceCount: lib.func("transcribe_device_count", "int", []), + deviceGet: lib.func("transcribe_device_get", "void *", ["int"]), + deviceInfoInit: lib.func("transcribe_device_info_init", "void", [ + outp(T.transcribe_device_info), ]), - getBackendDevice: lib.func("transcribe_get_backend_device", "int", [ - "int", - iop(T.transcribe_backend_device), + deviceGetInfo: lib.func("transcribe_device_get_info", "int", [ + "void *", + iop(T.transcribe_device_info), ]), backendAvailable: lib.func("transcribe_backend_available", "bool", ["int"]), @@ -65,10 +66,7 @@ export function bindLibrary(libraryPath: string): Bound { modelArch: lib.func("transcribe_model_arch_string", "str", ["void *"]), modelVariant: lib.func("transcribe_model_variant_string", "str", ["void *"]), modelBackend: lib.func("transcribe_model_backend", "str", ["void *"]), - modelGetDevice: lib.func("transcribe_model_get_device", "int", [ - "void *", - iop(T.transcribe_backend_device), - ]), + modelDevice: lib.func("transcribe_model_device", "void *", ["void *"]), modelSupports: lib.func("transcribe_model_supports", "bool", ["void *", "int"]), tokenize: lib.func("transcribe_tokenize", "int", ["void *", "str", "int32_t *", "size_t"]), capabilitiesInit: lib.func("transcribe_capabilities_init", "void", [ diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index 9b60cf00..234943ea 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -321,6 +321,8 @@ export function artifactDir(): string { return resolveLibrary().artifactDir; } +const DEVICE_HANDLES = new WeakMap(); + const DEVICE_TYPE_NAMES: Record = { [g.TRANSCRIBE_DEVICE_TYPE_CPU]: "cpu", [g.TRANSCRIBE_DEVICE_TYPE_GPU]: "gpu", @@ -328,11 +330,15 @@ const DEVICE_TYPE_NAMES: Record = { [g.TRANSCRIBE_DEVICE_TYPE_ACCEL]: "accel", }; -// Decode a koffi-filled transcribe_backend_device struct into a BackendInfo. +// Decode a koffi-filled transcribe_device_info struct into a BackendInfo. // memory_* are uint64 (bigint from koffi) but stay well under 2^53 for any // real device, so num() narrows them losslessly. -function deviceFromRaw(dev: any, index: number | null = null): BackendInfo { - return { +function deviceFromRaw( + dev: any, + handle: unknown, + index: number | null = null, +): BackendInfo { + const info: BackendInfo = { name: dev.name ?? "", description: dev.description ?? "", kind: dev.kind ?? "", @@ -342,17 +348,21 @@ function deviceFromRaw(dev: any, index: number | null = null): BackendInfo { memoryFree: num(dev.memory_free), index, }; + DEVICE_HANDLES.set(info, handle); + return info; } export function getAvailableBackends(): BackendInfo[] { const n = native(); - const count = n.F.backendDeviceCount(); + const count = n.F.deviceCount(); const out: BackendInfo[] = []; for (let i = 0; i < count; i++) { + const handle = n.F.deviceGet(i); + if (!handle) continue; const dev: any = {}; - n.F.backendDeviceInit(dev); - check(n, n.F.getBackendDevice(i, dev), `reading backend device ${i}`); - out.push(deviceFromRaw(dev, i)); + n.F.deviceInfoInit(dev); + check(n, n.F.deviceGetInfo(handle, dev), `reading backend device ${i}`); + out.push(deviceFromRaw(dev, handle, i)); } return out; } @@ -1271,8 +1281,21 @@ export class TranscribeModel { const n = native(); const p: any = {}; n.F.modelLoadParamsInit(p); + if ("gpuDevice" in opts) { + throw new TranscribeError( + "gpuDevice was removed in 0.2; pass a device from getAvailableBackends() instead", + ); + } if (opts.backend) p.backend = lookup(BACKENDS, opts.backend, "backend"); - if (opts.gpuDevice !== undefined) p.gpu_device = opts.gpuDevice; + if (opts.device !== undefined) { + const handle = DEVICE_HANDLES.get(opts.device); + if (!handle) { + throw new TranscribeError( + "device must be an entry returned by getAvailableBackends() or model.device", + ); + } + p.device = handle; + } const out: any[] = [null]; const st = await callAsync(n.F.modelLoadFile, path, p, out); @@ -1405,14 +1428,12 @@ export class TranscribeModel { * snapshot, so read this again to poll how much device memory is left * after the model loaded. */ get device(): BackendInfo { + const handle = this.#n.F.modelDevice(this.handle); + if (!handle) throw new TranscribeError("model has no resolved compute device"); const dev: any = {}; - this.#n.F.backendDeviceInit(dev); - check( - this.#n, - this.#n.F.modelGetDevice(this.handle, dev), - "reading model device", - ); - return deviceFromRaw(dev); + this.#n.F.deviceInfoInit(dev); + check(this.#n, this.#n.F.deviceGetInfo(handle, dev), "reading model device"); + return deviceFromRaw(dev, handle); } dispose(): void { diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 71a40261..24bcc86c 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -120,22 +120,18 @@ export interface BackendInfo { * unreported. Re-query (via {@link getAvailableBackends} or `model.device`) * to refresh; backend-defined and not comparable across device kinds. */ memoryFree: number; - /** Registry index of this device — the value to pass as - * {@link ModelOptions.gpuDevice} to select it (0 means auto: discrete - * GPUs are probed before integrated). `null` when this came from - * `model.device`, since `transcribe_model_get_device` does not expose an - * index; correlate such a device back to {@link getAvailableBackends} by - * `deviceId` / `name` instead. Order-dependent and not stable across - * driver updates or hosts. */ + /** Process-local registry index for display. Pass this object via + * {@link ModelOptions.device} for exact selection; persist `deviceId`, not + * the index. */ index: number | null; } export interface ModelOptions { /** "auto" (default), or an explicit backend. */ backend?: Backend; - /** GPU device registry index. 0 means auto: the first device that - * initializes, probing discrete GPUs before integrated. */ - gpuDevice?: number; + /** Exact device returned by {@link getAvailableBackends}. Omit for the + * backend's automatic policy. Exact selection never falls back. */ + device?: BackendInfo; } export interface SessionOptions { diff --git a/bindings/typescript/test/device-select.test.mjs b/bindings/typescript/test/device-select.test.mjs index 3b9da626..e7386cac 100644 --- a/bindings/typescript/test/device-select.test.mjs +++ b/bindings/typescript/test/device-select.test.mjs @@ -1,58 +1,44 @@ -// Model-gated tier: device selection on load. Skips cleanly when the canary -// GGUF is absent (modelTest), otherwise loads it and exercises model.device -// plus the gpuDevice/backend validation surface. +// Exact opaque-device selection tests. + import assert from "node:assert/strict"; -import { modelTest, MODEL } from "./common.mjs"; -import { TranscribeModel, getAvailableBackends, InvalidArgument } from "../dist/index.js"; +import test from "node:test"; +import { + backendAvailable, + getAvailableBackends, + InvalidArgument, + TranscribeModel, +} from "../dist/index.js"; +import { MODEL, modelTest } from "./common.mjs"; -modelTest("model.device reports an index-less device that matches a registry entry", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); +modelTest("an enumerated device can be selected exactly", MODEL, async () => { + const device = getAvailableBackends().find((d) => d.deviceType !== "accel"); + if (!device) return; try { - const dev = m.device; - assert.equal(typeof dev, "object"); - assert.notEqual(dev, null); - // model.device comes from transcribe_model_get_device, which does not expose - // a registry index — the binding reports it as null (see types.ts). - assert.equal(dev.index, null); - - // It must correspond to a device the registry enumerates: match by name, and - // by deviceId too when the backend reports a stable hardware id. - const backends = getAvailableBackends(); - const match = backends.find( - (b) => - b.name === dev.name && - (dev.deviceId === null || b.deviceId === dev.deviceId), - ); - assert.ok( - match, - `model.device (${JSON.stringify({ name: dev.name, deviceId: dev.deviceId })}) ` + - `should match a getAvailableBackends() entry`, - ); - } finally { - m.dispose(); + const model = await TranscribeModel.load(MODEL, { device }); + try { + assert.equal(model.device.name, device.name); + assert.equal(model.device.deviceId, device.deviceId); + } finally { + model.dispose(); + } + } catch (error) { + // A registered device can still fail driver initialization. The important + // contract is that native selection fails rather than moving elsewhere. + assert.match(String(error), /backend/i); } }); -modelTest("negative gpuDevice is rejected with InvalidArgument", MODEL, async () => { - await assert.rejects( - () => TranscribeModel.load(MODEL, { gpuDevice: -1 }), - (e) => e instanceof InvalidArgument, +modelTest("explicit backend must match exact device", MODEL, async () => { + const gpu = getAvailableBackends().find( + (d) => d.deviceType === "gpu" || d.deviceType === "igpu", ); -}); - -modelTest("out-of-range gpuDevice is rejected with InvalidArgument", MODEL, async () => { - const outOfRange = getAvailableBackends().length + 1000; + if (!gpu) return; await assert.rejects( - () => TranscribeModel.load(MODEL, { gpuDevice: outOfRange }), - (e) => e instanceof InvalidArgument, + () => TranscribeModel.load(MODEL, { backend: "cpu", device: gpu }), + InvalidArgument, ); }); -modelTest("selecting a GPU index under the cpu backend is rejected", MODEL, async () => { - // Hardware-independent: the cpu backend has no GPU device 1 to select, so the - // pairing must be rejected regardless of what GPUs the host actually has. - await assert.rejects( - () => TranscribeModel.load(MODEL, { backend: "cpu", gpuDevice: 1 }), - (e) => e instanceof InvalidArgument, - ); +test("automatic backend probing remains available", () => { + assert.equal(typeof backendAvailable("auto"), "boolean"); }); diff --git a/bindings/typescript/test/no-model.test.mjs b/bindings/typescript/test/no-model.test.mjs index a92b4acc..82eaccaf 100644 --- a/bindings/typescript/test/no-model.test.mjs +++ b/bindings/typescript/test/no-model.test.mjs @@ -89,6 +89,13 @@ test("invalid backend string is rejected, not silently coerced", () => { assert.throws(() => backendAvailable("nope")); }); +test("removed gpuDevice option is rejected instead of selecting auto", async () => { + await assert.rejects( + () => TranscribeModel.load("/no/such/model.gguf", { gpuDevice: 0 }), + /gpuDevice was removed in 0\.2/, + ); +}); + test("missing model file maps to ModelFileNotFound", async () => { await assert.rejects( () => TranscribeModel.load("/no/such/model.gguf"), diff --git a/docs/migrating-to-0.2.md b/docs/migrating-to-0.2.md new file mode 100644 index 00000000..59b14fc6 --- /dev/null +++ b/docs/migrating-to-0.2.md @@ -0,0 +1,158 @@ +# Migrating to transcribe.cpp 0.2 + +Version 0.2 is a deliberate pre-1.0 API and ABI break. Rebuild native +consumers against the 0.2 headers and upgrade each language package and native +provider together. The version and ABI-hash checks in the official bindings +reject mixed 0.1/0.2 installations. + +## Device selection + +The integer `gpu_device` selector has been replaced by an opaque, +process-local device handle. This removes the old collision where `0` meant +"automatic" and therefore could not select registry device 0. + +The new rules are: + +- Leave `device` unset (`NULL`, `None`, `nil`, or omitted) for the backend's + automatic policy. +- Pass a device returned by the current process's enumeration API to select + that exact primary device. Exact selection never falls back to another + primary. +- An explicit `backend` and exact `device` must match. `AUTO` accepts a CPU, + GPU, or integrated GPU. ACCEL devices such as BLAS/AMX cannot be primary; + select the CPU device with `CPU_ACCEL` to layer them onto exact CPU. +- Handles and registry indices are not persistent identifiers. Persist + `device_id` when the backend provides one, then enumerate and resolve a fresh + handle after backend initialization in each process. +- Treat dynamic backend registration as startup-only: every + `transcribe_init_backends*()` call must finish before any thread enumerates + devices, queries backend availability, or loads a model. The native registry + does not support racing registration against those operations. + +### C API replacements + +| 0.1 API | 0.2 API | +| --- | --- | +| `transcribe_backend_device_count()` | `transcribe_device_count()` | +| `transcribe_get_backend_device(index, &info)` | `transcribe_device_get(index)`, then `transcribe_device_get_info(device, &info)` | +| `struct transcribe_backend_device` | `struct transcribe_device_info` | +| `transcribe_backend_device_init()` | `transcribe_device_info_init()` | +| `transcribe_model_get_device(model, &info)` | `transcribe_model_device(model)`, then `transcribe_device_get_info(device, &info)` | +| `TRANSCRIBE_ABI_BACKEND_DEVICE` | `TRANSCRIBE_ABI_DEVICE_INFO` | +| `transcribe_model_load_params::gpu_device` | `transcribe_model_load_params::device` | + +For automatic selection, initialize the params and leave `device == NULL`: + +```c +struct transcribe_model_load_params params; +transcribe_model_load_params_init(¶ms); +params.backend = TRANSCRIBE_BACKEND_AUTO; +``` + +For exact selection, enumerate after registering dynamic backends and retain the +handle whose metadata matches the application's saved `device_id`. Never assign +an unchecked `transcribe_device_get()` result to model params: an out-of-range +index returns `NULL`, and `NULL` requests automatic selection. + +```c +#include + +transcribe_device_t selected = NULL; +for (int i = 0; i < transcribe_device_count(); ++i) { + transcribe_device_t device = transcribe_device_get(i); + struct transcribe_device_info info; + transcribe_device_info_init(&info); + if (transcribe_device_get_info(device, &info) != TRANSCRIBE_OK) { + continue; + } + if (info.device_type != TRANSCRIBE_DEVICE_TYPE_ACCEL && + info.device_id != NULL && strcmp(info.device_id, saved_device_id) == 0) { + selected = device; + break; + } +} + +if (selected != NULL) { + struct transcribe_model_load_params params; + transcribe_model_load_params_init(¶ms); + params.device = selected; + /* Load with params: exact selection is now guaranteed. */ +} else { + /* Report the unavailable device and do not load unless auto is intended. */ +} +``` + +`struct transcribe_model_load_params` changed layout (16 to 24 bytes on the +supported 64-bit ABIs), and the old device symbols were removed. A 0.1 binary +must not load a 0.2 library without being rebuilt. + +## Official bindings + +All official bindings now pass an enumerated device object rather than a +registry integer. + +### Python + +```python +device = next(d for d in transcribe_cpp.backends() if d.device_type == "cpu") +model = transcribe_cpp.Model("model.gguf", device=device) +``` + +Replace `Model(..., gpu_device=index)` and the one-shot helper's `gpu_device=` +with `device=`. Omit `device` for automatic selection. + +### Rust + +```rust +let device = transcribe_cpp::devices() + .into_iter() + .find(|device| device.kind == "cpu") + .unwrap(); +let model = transcribe_cpp::Model::load_with( + "model.gguf", + &transcribe_cpp::ModelOptions { + device: Some(device), + ..Default::default() + }, +)?; +``` + +Replace `ModelOptions::gpu_device` with `ModelOptions::device`. Use `None` for +automatic selection. + +### Swift + +```swift +let device = Transcribe.devices().first { $0.deviceType == .cpu }! +let model = try Model(path: "model.gguf", options: ModelOptions(device: device)) +``` + +Replace `ModelOptions(gpuDevice:)` with `ModelOptions(device:)`. Use `nil` for +automatic selection. + +### TypeScript / JavaScript + +```ts +const device = getAvailableBackends().find((device) => device.deviceType === "cpu"); +if (!device) throw new Error("CPU device is not registered"); +const model = await TranscribeModel.load("model.gguf", { device }); +``` + +Replace `gpuDevice` with `device`. Version 0.2 rejects the removed +`gpuDevice` property at runtime so JavaScript callers cannot silently fall back +to automatic selection. The object must come from `getAvailableBackends()` or +`model.device`; copying its visible fields does not copy its opaque native +identity. Omit `device` for automatic selection. + +## Command-line tools + +`transcribe-cli` and `transcribe-bench` now interpret every non-negative +`--device N` as an exact index from `transcribe-cli --list-devices`. + +In particular, **`--device 0` changed meaning**: + +- 0.1: automatic selection +- 0.2: exact registry device 0 + +To retain automatic selection, remove the `--device` option. Device indices are +process-local and should be resolved again rather than stored in configuration. diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 654980a8..f4cbbd05 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -12,6 +12,7 @@ #include "wav.h" #include +#include #include #include #include @@ -23,6 +24,20 @@ namespace { +bool parse_device_index(const char * text, int & out) { + if (text == nullptr || text[0] == '\0') { + return false; + } + const char * end = text + std::strlen(text); + int parsed = 0; + const auto result = std::from_chars(text, end, parsed); + if (result.ec != std::errc{} || result.ptr != end || parsed < 0) { + return false; + } + out = parsed; + return true; +} + // Minimal JSON string escape: covers the characters MUST be escaped by // the JSON spec (quote, backslash, control chars). Transcribed text is // short UTF-8 in practice; we don't need unicode escaping. @@ -205,13 +220,13 @@ struct cli_args { bool list_devices = false; // --list-devices: print devices and exit bool batch_jsonl = false; // --batch-jsonl: output JSONL std::string output_path; // -o/--output: write raw text here - int repeat = 1; - int n_threads = 0; // 0 = library default (all cores) - int n_ctx = 0; // 0 = model's true max; >0 lowers the cap - transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; - transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int gpu_device = 0; // --device N: 0 = auto, >0 = registry index - transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; + int repeat = 1; + int n_threads = 0; // 0 = library default (all cores) + int n_ctx = 0; // 0 = model's true max; >0 lowers the cap + transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; + transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; + int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device + transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; // Whisper-family knobs. Ignored for non-Whisper models. std::string initial_prompt; // --initial-prompt TEXT @@ -292,8 +307,8 @@ void print_usage(const char * argv0) { " --kv-type TYPE flash-attn KV type: auto, f32, f16 (default: auto)\n" " --backend TYPE compute backend: auto, cpu, cpu_accel, metal, vulkan, cuda, rocm\n" " (default: auto)\n" - " --device N GPU device index from --list-devices: 0 = auto\n" - " (first of kind), >0 selects that registry index\n" + " --device N exact device index from --list-devices, including 0\n" + " (default: automatic device selection)\n" " --timestamps TYPE timestamps: auto, none, segment, word, token (default: auto)\n" " --batch FILE batch mode: FILE has one wav path per line\n" " --batch-jsonl output one JSON line per file (for batch)\n" @@ -351,16 +366,16 @@ int list_devices_main() { "listing whatever registered\n", (int) st); } - const int n = transcribe_backend_device_count(); + const int n = transcribe_device_count(); if (n <= 0) { std::fprintf(stderr, "no compute devices registered\n"); return EXIT_FAILURE; } std::printf("%d compute device(s):\n", n); for (int i = 0; i < n; ++i) { - struct transcribe_backend_device d; - transcribe_backend_device_init(&d); - if (transcribe_get_backend_device(i, &d) != TRANSCRIBE_OK) { + struct transcribe_device_info d; + transcribe_device_info_init(&d); + if (transcribe_device_get_info(transcribe_device_get(i), &d) != TRANSCRIBE_OK) { continue; } const char * type_str = d.device_type == TRANSCRIBE_DEVICE_TYPE_CPU ? "cpu" : @@ -489,9 +504,8 @@ bool parse_args(int argc, char ** argv, cli_args & out) { if (!v) { return false; } - out.gpu_device = std::atoi(v); - if (out.gpu_device < 0) { - std::fprintf(stderr, "error: --device must be >= 0 (0 = auto)\n"); + if (!parse_device_index(v, out.device_index)) { + std::fprintf(stderr, "error: --device must be an integer index >= 0\n"); return false; } } else if (a == "--timestamps") { @@ -790,8 +804,12 @@ int main(int argc, char ** argv) { struct transcribe_model_load_params mp; transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.gpu_device = args.gpu_device; + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; + } struct transcribe_model * model = nullptr; const transcribe_status load_st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); if (load_st != TRANSCRIBE_OK) { @@ -1174,8 +1192,12 @@ int main(int argc, char ** argv) { if (!args.model_path.empty()) { struct transcribe_model_load_params mp; transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.gpu_device = args.gpu_device; + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; + } struct transcribe_model * model = nullptr; const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); std::printf("model: %s -> %s\n", args.model_path.c_str(), transcribe_status_string(st)); diff --git a/include/transcribe.abihash b/include/transcribe.abihash index bd8c2750..b0e23c5d 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -7896d8d4c2a46147 +7df72bf9e667b8c2 diff --git a/include/transcribe.h b/include/transcribe.h index e789bbd7..435a5ef5 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -362,7 +362,7 @@ typedef enum { TRANSCRIBE_ABI_STREAM_TEXT = 10, TRANSCRIBE_ABI_SESSION_LIMITS = 11, TRANSCRIBE_ABI_EXT = 12, - TRANSCRIBE_ABI_BACKEND_DEVICE = 13, + TRANSCRIBE_ABI_DEVICE_INFO = 13, TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14, } transcribe_abi_struct; @@ -793,13 +793,33 @@ TRANSCRIBE_API transcribe_status transcribe_init_backends(const char * artifact_ */ TRANSCRIBE_API transcribe_status transcribe_init_backends_default(void); +/* + * Opaque process-local compute-device handle. Handles are owned by the + * runtime, remain valid for the life of the process, and must not be freed. + * They may be compared for equality but are not persistent identifiers; use + * transcribe_device_get_info() and its device_id field for persistence. + */ +struct transcribe_device; +typedef struct transcribe_device * transcribe_device_t; + /* * Number of compute devices currently registered with the runtime * (compiled-in backends plus any modules loaded by * transcribe_init_backends). A device is something a model can be placed * on: the CPU, an Apple GPU via Metal, a Vulkan GPU, ... */ -TRANSCRIBE_API int transcribe_backend_device_count(void); +TRANSCRIBE_API int transcribe_device_count(void); + +/* + * Return the registered device at `index`, or NULL when index is out of + * range. The returned handle is runtime-owned and process-local. + * + * IMPORTANT: NULL is also the automatic-selection sentinel in + * transcribe_model_load_params::device. Always check this return value before + * assigning it to model-load params; assigning an unchecked out-of-range + * result would request automatic selection rather than exact selection. + */ +TRANSCRIBE_API transcribe_device_t transcribe_device_get(int index); /* * Device type: ggml's vendor-agnostic classification of a device, @@ -841,7 +861,7 @@ typedef enum { * this process's allocations; on a discrete GPU they are device-global; on * the CPU they are system RAM. 0 means the backend does not report it. */ -struct transcribe_backend_device { +struct transcribe_device_info { uint64_t struct_size; /* sizeof(*this); set by _init() */ const char * name; /* ggml device name, e.g. "Metal" */ const char * description; /* human-readable, e.g. "Apple M4 Max" */ @@ -852,18 +872,19 @@ struct transcribe_backend_device { transcribe_device_type device_type; /* CPU/GPU/IGPU/ACCEL axis */ }; -TRANSCRIBE_API void transcribe_backend_device_init(struct transcribe_backend_device * p); +TRANSCRIBE_API void transcribe_device_info_init(struct transcribe_device_info * p); /* - * Fill *out (initialized via transcribe_backend_device_init) with device - * `index` in [0, transcribe_backend_device_count()). + * Fill *out (initialized via transcribe_device_info_init) with information + * about `device`. memory_free is live as of this call; re-invoke to refresh it + * (e.g. to poll a device's available memory over time). * - * memory_free is live as of this call; re-invoke to refresh it (e.g. to - * poll a device's available memory over time). The device handles are - * stable for the life of the process, so the same index always names the - * same device. + * Returns TRANSCRIBE_ERR_INVALID_ARG if device or out is NULL or device is + * not from this runtime's registry. Returns TRANSCRIBE_ERR_BAD_STRUCT_SIZE if + * out fails the struct-size check. */ -TRANSCRIBE_API transcribe_status transcribe_get_backend_device(int index, struct transcribe_backend_device * out); +TRANSCRIBE_API transcribe_status transcribe_device_get_info(transcribe_device_t device, + struct transcribe_device_info * out); /* * Whether a backend request can be satisfied by some registered device: @@ -876,19 +897,12 @@ TRANSCRIBE_API transcribe_status transcribe_get_backend_device(int index, struct TRANSCRIBE_API bool transcribe_backend_available(transcribe_backend_request kind); /* - * Fill *out (initialized via transcribe_backend_device_init) with the - * compute device this loaded model is running on — the device that owns its - * weights and runs most of its graph. Same struct and same live-snapshot - * semantics as transcribe_get_backend_device: memory_free is current as of - * the call, so re-invoke to ask "how much memory is left on the device my - * model landed on" at any time after load. - * - * Returns TRANSCRIBE_ERR_INVALID_ARG if model or out is NULL (or out fails - * the struct-size check), or TRANSCRIBE_ERR_BACKEND if the model has no - * resolved compute device. + * Return the compute device this loaded model is running on — the device + * that owns its weights and runs most of its graph. Returns NULL if model is + * NULL or has no resolved compute device. Pass the returned handle to + * transcribe_device_get_info() for metadata and a live memory snapshot. */ -TRANSCRIBE_API transcribe_status transcribe_model_get_device(const struct transcribe_model * model, - struct transcribe_backend_device * out); +TRANSCRIBE_API transcribe_device_t transcribe_model_device(const struct transcribe_model * model); /* * Initialization of caller-owned params structs. @@ -920,37 +934,26 @@ TRANSCRIBE_API transcribe_status transcribe_model_get_device(const struct transc * backend: which backend to request. See transcribe_backend_request * for the semantics of each value. Default is AUTO. * - * gpu_device: Multi-GPU selector. 0 (the default) means "auto / the first - * device of the chosen kind": AUTO picks the first GPU that - * initializes, and explicit METAL/VULKAN/CUDA/ROCM requests pick the - * first matching device — in both cases probing every discrete - * GPU before any integrated GPU, in ggml's registry order - * within each tier. - * - * A value > 0 selects the GPU/IGPU device at that global ggml - * registry index — the same index space transcribe_get_backend_device() - * enumerates, so enumerate first to choose one. The selected - * device becomes the model's primary backend, validated against - * `backend`: it must be a GPU/IGPU, and for an explicit - * METAL/VULKAN/CUDA/ROCM request it must be that vendor. The index is - * order-dependent — ggml's registry order can shift across driver - * updates or hosts, so treat it as a runtime selection, not a - * stable identifier; correlate via the enumerated device's name / - * device_id when you need stability. - * - * gpu_device is rejected with TRANSCRIBE_ERR_INVALID_ARG when it - * is negative, out of range, names a non-GPU device, names a - * device whose vendor doesn't match an explicit GPU request, or - * is non-zero alongside a CPU / CPU_ACCEL request (there is no - * GPU to select). Note there is no way to explicitly select the - * device at registry index 0 — 0 is the auto sentinel. An - * integrated GPU sitting at index 0 is therefore reachable only - * via the probe order, when no discrete GPU initializes. + * device: NULL (the default) applies the backend's automatic policy. AUTO + * probes every discrete GPU before integrated GPUs and finally falls + * back to CPU; an explicit GPU backend picks the first matching + * device. A non-NULL handle selects that exact registered device, + * including the device returned at index 0. + * + * Exact selection never silently falls back to another primary + * device. With backend=AUTO, the selected device determines the + * backend. With an explicit backend, the device must match it. CPU + * and CPU_ACCEL accept an exact CPU device; ACCEL devices cannot be + * selected as a primary. Invalid, foreign, or mismatched handles are + * rejected with TRANSCRIBE_ERR_INVALID_ARG. + * + * Handles are process-local. Persist device_id (when available), then + * enumerate and resolve a fresh handle in each process. */ struct transcribe_model_load_params { uint64_t struct_size; transcribe_backend_request backend; - int gpu_device; + transcribe_device_t device; }; TRANSCRIBE_API void transcribe_model_load_params_init(struct transcribe_model_load_params * params); diff --git a/scripts/ci/link_smoke.c b/scripts/ci/link_smoke.c index 710117bd..c8b057f5 100644 --- a/scripts/ci/link_smoke.c +++ b/scripts/ci/link_smoke.c @@ -31,16 +31,16 @@ int main(int argc, char ** argv) { } } - int n = transcribe_backend_device_count(); + int n = transcribe_device_count(); printf("devices=%d\n", n); if (n < 1) { fprintf(stderr, "link-smoke: no registered compute devices\n"); return 1; } for (int i = 0; i < n; i++) { - struct transcribe_backend_device dev; - transcribe_backend_device_init(&dev); - if (transcribe_get_backend_device(i, &dev) != TRANSCRIBE_OK) { + struct transcribe_device_info dev; + transcribe_device_info_init(&dev); + if (transcribe_device_get_info(transcribe_device_get(i), &dev) != TRANSCRIBE_OK) { fprintf(stderr, "link-smoke: device %d query failed\n", i); return 1; } diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index 15809fda..da36675d 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -479,7 +479,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "canary", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "canary", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index c4afd55a..f61ecb0f 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -645,7 +645,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // Backend plan + alloc + stream tensor data. const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, "canary_qwen", + if (auto st = load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "canary_qwen", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index 368b3f86..45c10b76 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -561,7 +561,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "cohere", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "cohere", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index 278ee2ab..2f752f06 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -354,7 +354,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par } const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "funasr_nano", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/gigaam/model.cpp b/src/arch/gigaam/model.cpp index 4f13efec..b2abc4e0 100644 --- a/src/arch/gigaam/model.cpp +++ b/src/arch/gigaam/model.cpp @@ -135,7 +135,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par } const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "gigaam", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index c66354f6..f3e76fa4 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -397,7 +397,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // Backend plan. const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "granite", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "granite", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/granite_nar/model.cpp b/src/arch/granite_nar/model.cpp index 2a514583..c99dfe75 100644 --- a/src/arch/granite_nar/model.cpp +++ b/src/arch/granite_nar/model.cpp @@ -340,7 +340,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "granite_nar", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "granite_nar", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/medasr/model.cpp b/src/arch/medasr/model.cpp index ccf0e3c8..4f90668f 100644 --- a/src/arch/medasr/model.cpp +++ b/src/arch/medasr/model.cpp @@ -183,7 +183,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par } const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "medasr", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/moonshine/model.cpp b/src/arch/moonshine/model.cpp index e48177ec..bc4de95a 100644 --- a/src/arch/moonshine/model.cpp +++ b/src/arch/moonshine/model.cpp @@ -253,7 +253,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par } const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "moonshine", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/moonshine_streaming/model.cpp b/src/arch/moonshine_streaming/model.cpp index e3f47221..2fc1fe0e 100644 --- a/src/arch/moonshine_streaming/model.cpp +++ b/src/arch/moonshine_streaming/model.cpp @@ -266,7 +266,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par } const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "moonshine_streaming", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index f1330ca5..a8b7fb75 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -313,7 +313,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "moss", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "moss", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 242d9931..825adf90 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -525,7 +525,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "parakeet", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "parakeet", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 74a72c4f..e48da627 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -244,7 +244,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // Backend plan. const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "qwen3_asr", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "qwen3_asr", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/sensevoice/model.cpp b/src/arch/sensevoice/model.cpp index 33bbe639..2915a577 100644 --- a/src/arch/sensevoice/model.cpp +++ b/src/arch/sensevoice/model.cpp @@ -144,7 +144,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par } const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "sensevoice", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index ef989c44..a2be174b 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -602,7 +602,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "sortformer", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "sortformer", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index 0f55fa97..133af9c2 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -494,7 +494,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "voxtral", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "voxtral", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/voxtral_realtime/model.cpp b/src/arch/voxtral_realtime/model.cpp index 3b55b586..c7f3cb5b 100644 --- a/src/arch/voxtral_realtime/model.cpp +++ b/src/arch/voxtral_realtime/model.cpp @@ -258,7 +258,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par } const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->device : nullptr, "voxtral_realtime", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); diff --git a/src/arch/whisper/bin_load.cpp b/src/arch/whisper/bin_load.cpp index 6e8fb32a..a7ac6da1 100644 --- a/src/arch/whisper/bin_load.cpp +++ b/src/arch/whisper/bin_load.cpp @@ -590,7 +590,7 @@ transcribe_status load_from_bin(const char * path } // ---- Backend plan ---- - if (auto st = transcribe::load_common::init_backends(params->backend, params->gpu_device, "whisper", m->plan); + if (auto st = transcribe::load_common::init_backends(params->backend, params->device, "whisper", m->plan); st != TRANSCRIBE_OK) { return st; } diff --git a/src/arch/whisper/model.cpp b/src/arch/whisper/model.cpp index ce4d2272..0b362cda 100644 --- a/src/arch/whisper/model.cpp +++ b/src/arch/whisper/model.cpp @@ -473,7 +473,7 @@ transcribe_status whisper_load(Loader & loader, // Backend plan. const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, (params != nullptr) ? params->gpu_device : 0, "whisper", m->plan); + backend_req, (params != nullptr) ? params->device : nullptr, "whisper", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/transcribe-load-common.cpp b/src/transcribe-load-common.cpp index 157add2c..4b9bc99a 100644 --- a/src/transcribe-load-common.cpp +++ b/src/transcribe-load-common.cpp @@ -193,35 +193,41 @@ bool valid_backend_request(int raw) { return false; } -// Resolve a BackendPlan for an explicit device selection (gpu_device > 0). -// `dev_index` is a global ggml registry index. The selected device becomes -// the primary; `requested` constrains what kind it must be. Only GPU/IGPU -// devices are selectable this way — strict/accel CPU requests reject a -// non-zero gpu_device before reaching here. The assembled plan mirrors the -// specific-GPU path: primary GPU, then ACCEL, then CPU last as the fallback. -transcribe_status init_backends_explicit_index(transcribe_backend_request requested, - int dev_index, - const char * error_tag, - BackendPlan & out) { - const size_t n = ggml_backend_dev_count(); - if (dev_index < 0 || static_cast(dev_index) >= n) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: gpu_device %d out of range [0, %zu)", error_tag, dev_index, n); +// Resolve a BackendPlan for an exact process-local device handle. The handle +// must be one currently registered with ggml. GPU primaries retain ACCEL + CPU +// scheduler fallbacks; an exact CPU primary is strict CPU unless CPU_ACCEL was +// requested, in which case host-memory accelerators are layered ahead of it. +transcribe_status init_backends_explicit_device(transcribe_backend_request requested, + transcribe_device_t device, + const char * error_tag, + BackendPlan & out) { + ggml_backend_dev_t dev = reinterpret_cast(device); + bool registered = false; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + if (ggml_backend_dev_get(i) == dev) { + registered = true; + break; + } + } + if (!registered) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: device handle is not registered with this runtime", error_tag); return TRANSCRIBE_ERR_INVALID_ARG; } - ggml_backend_dev_t dev = ggml_backend_dev_get(static_cast(dev_index)); - const auto dev_type = ggml_backend_dev_type(dev); - if (dev_type != GGML_BACKEND_DEVICE_TYPE_GPU && dev_type != GGML_BACKEND_DEVICE_TYPE_IGPU) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: gpu_device %d (%s) is not a GPU device", error_tag, dev_index, + const auto dev_type = ggml_backend_dev_type(dev); + if (dev_type == GGML_BACKEND_DEVICE_TYPE_ACCEL) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: accelerator device %s cannot be a model primary", error_tag, ggml_backend_dev_name(dev)); return TRANSCRIBE_ERR_INVALID_ARG; } - const BackendKind got = classify_device(dev); - - // A specific vendor request pins the kind; AUTO accepts any GPU. - BackendKind wanted = BackendKind::Unknown; // Unknown == "any GPU" (AUTO) + const BackendKind got = classify_device(dev); + BackendKind wanted = BackendKind::Unknown; switch (requested) { + case TRANSCRIBE_BACKEND_CPU: + case TRANSCRIBE_BACKEND_CPU_ACCEL: + wanted = BackendKind::Cpu; + break; case TRANSCRIBE_BACKEND_METAL: wanted = BackendKind::Metal; break; @@ -237,28 +243,24 @@ transcribe_status init_backends_explicit_index(transcribe_backend_request reques case TRANSCRIBE_BACKEND_AUTO: break; default: - // CPU / CPU_ACCEL never reach here (caller rejects nonzero - // gpu_device for them); anything else is a programming error. return TRANSCRIBE_ERR_INVALID_ARG; } if (wanted != BackendKind::Unknown && got != wanted) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: gpu_device %d is a %s device but %s was requested", error_tag, - dev_index, kind_name(got), kind_name(wanted)); + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: selected %s device does not match requested %s backend", error_tag, + kind_name(got), kind_name(wanted)); return TRANSCRIBE_ERR_INVALID_ARG; } - ggml_backend_t gpu_be = dev_init_checked(dev, error_tag); - if (gpu_be == nullptr) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to initialize gpu_device %d (%s)", error_tag, dev_index, + ggml_backend_t primary = dev_init_checked(dev, error_tag); + if (primary == nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to initialize selected device %s", error_tag, ggml_backend_dev_name(dev)); return TRANSCRIBE_ERR_BACKEND; } - log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: using %s backend (gpu_device %d): %s", error_tag, kind_name(got), dev_index, + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: using selected %s device: %s", error_tag, kind_name(got), ggml_backend_dev_name(dev)); - // An explicit device index is always honored: warn only (same gate as - // try_init_kind). - if (got == BackendKind::Metal && metal_backend_lacks_simdgroup_mm(gpu_be, dev)) { + if (got == BackendKind::Metal && metal_backend_lacks_simdgroup_mm(primary, dev)) { const char * dname = ggml_backend_dev_name(dev); log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: Metal device \"%s\" has no simdgroup matrix multiply " @@ -266,12 +268,18 @@ transcribe_status init_backends_explicit_index(transcribe_backend_request reques error_tag, dname != nullptr ? dname : "?"); } - out.primary = gpu_be; + out.primary = primary; out.primary_kind = got; - out.scheduler_list.push_back(gpu_be); + if (got == BackendKind::Cpu) { + if (requested == TRANSCRIBE_BACKEND_CPU_ACCEL) { + append_accel_backends(out.scheduler_list, error_tag); + } + out.scheduler_list.push_back(primary); + return TRANSCRIBE_OK; + } + out.scheduler_list.push_back(primary); append_accel_backends(out.scheduler_list, error_tag); - ggml_backend_t cpu_be = init_cpu_backend(error_tag); if (cpu_be == nullptr) { return TRANSCRIBE_ERR_BACKEND; @@ -283,7 +291,7 @@ transcribe_status init_backends_explicit_index(transcribe_backend_request reques } // namespace transcribe_status init_backends(transcribe_backend_request requested, - int gpu_device, + transcribe_device_t device, const char * error_tag, BackendPlan & out) { // Read the request as raw bytes before any enum-typed load: a C caller @@ -297,26 +305,15 @@ transcribe_status init_backends(transcribe_backend_request requested, out.requested = static_cast( valid_backend_request(requested_raw) ? requested_raw : TRANSCRIBE_BACKEND_AUTO); - // Explicit device selection. 0 is "auto / first of kind" and falls - // through to the per-request logic below; a negative index is always - // invalid; a positive index pins a specific GPU/IGPU device. - if (gpu_device < 0) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: gpu_device must be >= 0 (got %d)", error_tag, gpu_device); - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (gpu_device > 0) { + // A non-NULL handle pins one exact registered device. Validate the raw + // backend value before passing its enum representation onward. + if (device != nullptr) { if (!valid_backend_request(requested_raw)) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: invalid transcribe_backend_request value %d", error_tag, requested_raw); return TRANSCRIBE_ERR_INVALID_ARG; } - // gpu_device names a GPU; a CPU-only request has nothing to select. - if (requested_raw == TRANSCRIBE_BACKEND_CPU || requested_raw == TRANSCRIBE_BACKEND_CPU_ACCEL) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: gpu_device %d is invalid for a CPU backend request", error_tag, - gpu_device); - return TRANSCRIBE_ERR_INVALID_ARG; - } - return init_backends_explicit_index(out.requested, gpu_device, error_tag, out); + return init_backends_explicit_device(out.requested, device, error_tag, out); } // Explicit switch over the enum so an unknown / garbage value diff --git a/src/transcribe-load-common.h b/src/transcribe-load-common.h index ddd71111..4d3eb7aa 100644 --- a/src/transcribe-load-common.h +++ b/src/transcribe-load-common.h @@ -53,16 +53,11 @@ bool metal_backend_lacks_simdgroup_mm(ggml_backend_t be, ggml_backend_dev_t dev) // Library-internal classification of what each value // actually picks up is documented in // transcribe-backend.h. -// gpu_device: multi-GPU selector. 0 (the default) means "auto / the -// first device of the chosen kind" — the existing -// first-of-kind behavior. A value > 0 selects the GPU/IGPU -// device at that global ggml registry index (the same index -// space transcribe_get_backend_device() enumerates) as the -// primary, validated against `requested`: the device must be -// a GPU/IGPU, and for a specific METAL/VULKAN/CUDA/ROCM request it -// must be that vendor. gpu_device is not valid for a -// CPU / CPU_ACCEL request (there is no GPU to pick) nor -// negative; both return TRANSCRIBE_ERR_INVALID_ARG. +// device: NULL applies the requested backend's automatic policy. A +// non-NULL process-local handle selects that exact registered +// device. AUTO accepts a GPU, IGPU, or CPU; explicit backend +// requests require a matching device. ACCEL devices cannot be +// primary. Exact selection never falls back to another primary. // error_tag: log prefix, e.g. "parakeet". // out: populated on success. out.primary is the backend // that owns the weight buffer; out.scheduler_list @@ -72,17 +67,15 @@ bool metal_backend_lacks_simdgroup_mm(ggml_backend_t be, ggml_backend_dev_t dev) // // Returns: // TRANSCRIBE_OK on success. -// TRANSCRIBE_ERR_INVALID_ARG if gpu_device is negative, out of range, -// names a non-GPU device, names a device whose -// vendor doesn't match a specific GPU request, or -// is non-zero for a CPU request. +// TRANSCRIBE_ERR_INVALID_ARG if device is foreign, is an ACCEL device, or +// does not match a specific backend request. // TRANSCRIBE_ERR_BACKEND if the caller asked for a specific // backend (METAL / VULKAN / CUDA / ROCM) that could not // be initialized, or if the CPU backend // itself fails to initialize (there is no // fallback past CPU). transcribe_status init_backends(transcribe_backend_request requested, - int gpu_device, + transcribe_device_t device, const char * error_tag, BackendPlan & out); diff --git a/src/transcribe-model.h b/src/transcribe-model.h index ec1ebe35..ce522cd9 100644 --- a/src/transcribe-model.h +++ b/src/transcribe-model.h @@ -62,7 +62,7 @@ struct transcribe_model { // The resolved primary compute backend this model runs on (the handle // that owns the weight buffer). Set by per-family load() right where it // sets `backend`, from BackendPlan::primary. Used by the public - // transcribe_model_get_device() accessor to recover the device — and its + // transcribe_model_device() accessor to recover the device — and its // live memory — without exposing the per-family BackendPlan. nullptr // until a family binds it. ggml_backend_t primary_backend = nullptr; diff --git a/src/transcribe.cpp b/src/transcribe.cpp index d3aba582..c370fabc 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -226,8 +226,8 @@ extern "C" size_t transcribe_abi_struct_size(transcribe_abi_struct which) { return sizeof(struct transcribe_session_limits); case TRANSCRIBE_ABI_EXT: return sizeof(struct transcribe_ext); - case TRANSCRIBE_ABI_BACKEND_DEVICE: - return sizeof(struct transcribe_backend_device); + case TRANSCRIBE_ABI_DEVICE_INFO: + return sizeof(struct transcribe_device_info); case TRANSCRIBE_ABI_SPEAKER_SEGMENT: return sizeof(struct transcribe_speaker_segment); } @@ -262,8 +262,8 @@ extern "C" size_t transcribe_abi_struct_align(transcribe_abi_struct which) { return alignof(struct transcribe_session_limits); case TRANSCRIBE_ABI_EXT: return alignof(struct transcribe_ext); - case TRANSCRIBE_ABI_BACKEND_DEVICE: - return alignof(struct transcribe_backend_device); + case TRANSCRIBE_ABI_DEVICE_INFO: + return alignof(struct transcribe_device_info); case TRANSCRIBE_ABI_SPEAKER_SEGMENT: return alignof(struct transcribe_speaker_segment); } @@ -649,7 +649,7 @@ extern "C" void transcribe_segment_init(struct transcribe_segment * p) { p->struct_size = sizeof(*p); } -extern "C" void transcribe_backend_device_init(struct transcribe_backend_device * p) { +extern "C" void transcribe_device_info_init(struct transcribe_device_info * p) { if (p == nullptr) { return; } @@ -735,7 +735,7 @@ namespace { // library-side prefix do NOT raise this value. #define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) -constexpr size_t k_min_model_params_size = TRANSCRIBE_FIELD_END(transcribe_model_load_params, gpu_device); +constexpr size_t k_min_model_params_size = TRANSCRIBE_FIELD_END(transcribe_model_load_params, device); constexpr size_t k_min_context_params_size = TRANSCRIBE_FIELD_END(transcribe_session_params, kv_type); // run_params is the one 0.2.0 exception to the append-only rule: `diarize` // was inserted mid-struct, shifting every field from `language` on by 8 @@ -762,7 +762,7 @@ constexpr size_t k_min_word_size = TRANSCRIBE_FIELD_END(transcribe_wo constexpr size_t k_min_token_size = TRANSCRIBE_FIELD_END(transcribe_token, text); constexpr size_t k_min_speaker_segment_size = TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p); constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); -constexpr size_t k_min_backend_device_size = TRANSCRIBE_FIELD_END(transcribe_backend_device, kind); +constexpr size_t k_min_device_info_size = TRANSCRIBE_FIELD_END(transcribe_device_info, kind); // k_min_whisper_chunk_trace_size lives in arch/whisper/public.cpp with // the chunk-trace accessor that uses it. @@ -980,10 +980,10 @@ transcribe_device_type to_device_type(enum ggml_backend_dev_type t) { } } -// Fill a caller-owned transcribe_backend_device from a ggml device, honoring +// Fill a caller-owned transcribe_device_info from a ggml device, honoring // the caller's declared struct_size via copy_out_prefix. ggml_backend_dev_get_props // queries memory live, so every call observes a fresh memory_free snapshot. -void fill_backend_device(ggml_backend_dev_t dev, uint64_t caller_size, struct transcribe_backend_device * out) { +void fill_device_info(ggml_backend_dev_t dev, uint64_t caller_size, struct transcribe_device_info * out) { ggml_backend_dev_props props{}; ggml_backend_dev_get_props(dev, &props); @@ -998,7 +998,7 @@ void fill_backend_device(ggml_backend_dev_t dev, uint64_t caller_size, struct tr description = props.description != nullptr ? props.description : ""; } - struct transcribe_backend_device staged{}; + struct transcribe_device_info staged{}; staged.struct_size = caller_size; staged.name = name; staged.description = description; @@ -1012,22 +1012,43 @@ void fill_backend_device(ggml_backend_dev_t dev, uint64_t caller_size, struct tr } // namespace -static int transcribe_backend_device_count_impl(void) { +static int transcribe_device_count_impl(void) { return static_cast(ggml_backend_dev_count()); } -static transcribe_status transcribe_get_backend_device_impl(int index, struct transcribe_backend_device * out) { +static transcribe_device_t transcribe_device_get_impl(int index) { + if (index < 0 || index >= static_cast(ggml_backend_dev_count())) { + return nullptr; + } + return reinterpret_cast(ggml_backend_dev_get(static_cast(index))); +} + +static ggml_backend_dev_t device_from_handle(transcribe_device_t device) { + if (device == nullptr) { + return nullptr; + } + ggml_backend_dev_t candidate = reinterpret_cast(device); + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + if (ggml_backend_dev_get(i) == candidate) { + return candidate; + } + } + return nullptr; +} + +static transcribe_status transcribe_device_get_info_impl(transcribe_device_t device, + struct transcribe_device_info * out) { if (out == nullptr) { return TRANSCRIBE_ERR_INVALID_ARG; } - if (const auto st = check_struct_size(out->struct_size, k_min_backend_device_size); st != TRANSCRIBE_OK) { + if (const auto st = check_struct_size(out->struct_size, k_min_device_info_size); st != TRANSCRIBE_OK) { return st; } - if (index < 0 || index >= static_cast(ggml_backend_dev_count())) { + ggml_backend_dev_t dev = device_from_handle(device); + if (dev == nullptr) { return TRANSCRIBE_ERR_INVALID_ARG; } - ggml_backend_dev_t dev = ggml_backend_dev_get(static_cast(index)); - fill_backend_device(dev, out->struct_size, out); + fill_device_info(dev, out->struct_size, out); return TRANSCRIBE_OK; } @@ -1433,11 +1454,9 @@ static transcribe_status transcribe_model_load_file_impl(const char * return st; } - // gpu_device is validated where the device registry is available — in - // load_common::init_backends, which each family calls. 0 means auto/first - // of kind; a positive index selects a specific GPU; negative / out of - // range / kind-mismatched values return TRANSCRIBE_ERR_INVALID_ARG from - // there. See the public header and transcribe-load-common.h. + // device is validated where the registry is available, in + // load_common::init_backends. NULL means automatic selection; a non-NULL + // handle selects that exact registered device or fails. // Raw-validate the backend request before the families' first // enum-typed load of it (see enum_field_raw). init_backends re-checks @@ -2538,25 +2557,15 @@ extern "C" const char * transcribe_model_backend(const struct transcribe_model * return model->backend.c_str(); } -static transcribe_status transcribe_model_get_device_impl(const struct transcribe_model * model, - struct transcribe_backend_device * out) { - if (model == nullptr || out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_backend_device_size); st != TRANSCRIBE_OK) { - return st; - } - // The model's primary backend is bound by per-family load(); a model - // that never resolved one has none. - if (model->primary_backend == nullptr) { - return TRANSCRIBE_ERR_BACKEND; +static transcribe_device_t transcribe_model_device_impl(const struct transcribe_model * model) { + if (model == nullptr || model->primary_backend == nullptr) { + return nullptr; } ggml_backend_dev_t dev = ggml_backend_get_device(model->primary_backend); - if (dev == nullptr) { - return TRANSCRIBE_ERR_BACKEND; + if (dev == nullptr || device_from_handle(reinterpret_cast(dev)) == nullptr) { + return nullptr; } - fill_backend_device(dev, out->struct_size, out); - return TRANSCRIBE_OK; + return reinterpret_cast(dev); } // Timings @@ -3121,14 +3130,18 @@ extern "C" transcribe_status transcribe_init_backends_default(void) { [&] { return transcribe_init_backends_default_impl(); }); } -extern "C" int transcribe_backend_device_count(void) { - return api_guard_value("transcribe_backend_device_count", 0, - [&] { return transcribe_backend_device_count_impl(); }); +extern "C" int transcribe_device_count(void) { + return api_guard_value("transcribe_device_count", 0, [&] { return transcribe_device_count_impl(); }); +} + +extern "C" transcribe_device_t transcribe_device_get(int index) { + return api_guard_value("transcribe_device_get", static_cast(nullptr), + [&] { return transcribe_device_get_impl(index); }); } -extern "C" transcribe_status transcribe_get_backend_device(int index, struct transcribe_backend_device * out) { - return api_guard_status("transcribe_get_backend_device", - [&] { return transcribe_get_backend_device_impl(index, out); }); +extern "C" transcribe_status transcribe_device_get_info(transcribe_device_t device, + struct transcribe_device_info * out) { + return api_guard_status("transcribe_device_get_info", [&] { return transcribe_device_get_info_impl(device, out); }); } extern "C" bool transcribe_backend_available(transcribe_backend_request kind) { @@ -3228,10 +3241,9 @@ extern "C" void transcribe_stream_reset(struct transcribe_session * session) { api_guard_void("transcribe_stream_reset", [&] { transcribe_stream_reset_impl(session); }); } -extern "C" transcribe_status transcribe_model_get_device(const struct transcribe_model * model, - struct transcribe_backend_device * out) { - return api_guard_status("transcribe_model_get_device", - [&] { return transcribe_model_get_device_impl(model, out); }); +extern "C" transcribe_device_t transcribe_model_device(const struct transcribe_model * model) { + return api_guard_value("transcribe_model_device", static_cast(nullptr), + [&] { return transcribe_model_device_impl(model); }); } extern "C" int transcribe_tokenize(const struct transcribe_model * model, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index db1f0d56..92e9a0d2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1295,4 +1295,10 @@ if(TRANSCRIBE_BUILD_EXAMPLES) -DWAV=${CMAKE_SOURCE_DIR}/samples/jfk.wav -DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/cli-output-smoke -P ${CMAKE_CURRENT_SOURCE_DIR}/cli_output_smoke.cmake) + + add_test(NAME transcribe_cli_device_arg_smoke + COMMAND ${CMAKE_COMMAND} + -DCLI=$ + -DWAV=${CMAKE_SOURCE_DIR}/samples/jfk.wav + -P ${CMAKE_CURRENT_SOURCE_DIR}/cli_device_arg_smoke.cmake) endif() diff --git a/tests/api_smoke.c b/tests/api_smoke.c index 2e1ad6e0..ce776b72 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -146,7 +146,7 @@ static void test_backend_devices(void) { * compiled-in builds register at startup; dynamic-backend builds * register via transcribe_init_backends. This smoke runs in both * configurations. */ - const int n_before = transcribe_backend_device_count(); + const int n_before = transcribe_device_count(); CHECK(n_before >= 0); /* init_backends argument contract. */ @@ -160,16 +160,16 @@ static void test_backend_devices(void) { * never re-registers (device count stable). */ const int st1 = transcribe_init_backends("."); CHECK(st1 == TRANSCRIBE_OK || st1 == TRANSCRIBE_ERR_BACKEND); - const int n_after = transcribe_backend_device_count(); + const int n_after = transcribe_device_count(); const int st2 = transcribe_init_backends("."); CHECK(st2 == st1); - CHECK(transcribe_backend_device_count() == n_after); + CHECK(transcribe_device_count() == n_after); if (n_after > 0) { - struct transcribe_backend_device dev; - transcribe_backend_device_init(&dev); + struct transcribe_device_info dev; + transcribe_device_info_init(&dev); CHECK(dev.struct_size == sizeof(dev)); - CHECK(transcribe_get_backend_device(0, &dev) == TRANSCRIBE_OK); + CHECK(transcribe_device_get_info(transcribe_device_get(0), &dev) == TRANSCRIBE_OK); CHECK(dev.name != NULL && dev.name[0] != '\0'); CHECK(dev.description != NULL); CHECK(dev.kind != NULL && dev.kind[0] != '\0'); @@ -180,16 +180,18 @@ static void test_backend_devices(void) { } /* Out-of-range probes answer cleanly: error status or false, never UB. */ - struct transcribe_backend_device dev2; - transcribe_backend_device_init(&dev2); - CHECK(transcribe_get_backend_device(-1, &dev2) == TRANSCRIBE_ERR_INVALID_ARG); - CHECK(transcribe_get_backend_device(1 << 20, &dev2) == TRANSCRIBE_ERR_INVALID_ARG); - CHECK(transcribe_get_backend_device(0, NULL) == TRANSCRIBE_ERR_INVALID_ARG); + struct transcribe_device_info dev2; + transcribe_device_info_init(&dev2); + CHECK(transcribe_device_get_info(transcribe_device_get(-1), &dev2) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_device_get_info(transcribe_device_get(1 << 20), &dev2) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_device_get_info(transcribe_device_get(0), NULL) == TRANSCRIBE_ERR_INVALID_ARG); + struct transcribe_device_info dev_bad = { 0 }; + CHECK(transcribe_device_get_info(transcribe_device_get(0), &dev_bad) == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); CHECK(!transcribe_backend_available((transcribe_backend_request) 999)); /* ABI accessors must know the new struct. */ - CHECK(transcribe_abi_struct_size(TRANSCRIBE_ABI_BACKEND_DEVICE) == sizeof(struct transcribe_backend_device)); - CHECK(transcribe_abi_struct_align(TRANSCRIBE_ABI_BACKEND_DEVICE) == _Alignof(struct transcribe_backend_device)); + CHECK(transcribe_abi_struct_size(TRANSCRIBE_ABI_DEVICE_INFO) == sizeof(struct transcribe_device_info)); + CHECK(transcribe_abi_struct_align(TRANSCRIBE_ABI_DEVICE_INFO) == _Alignof(struct transcribe_device_info)); } static void test_log_level_values(void) { @@ -217,7 +219,7 @@ static void test_init_macros(void) { transcribe_model_load_params_init(&mp_macro); CHECK(mp_macro.struct_size == sizeof(struct transcribe_model_load_params)); CHECK(mp_macro.backend == TRANSCRIBE_BACKEND_AUTO); - CHECK(mp_macro.gpu_device == 0); + CHECK(mp_macro.device == NULL); struct transcribe_session_params cp_macro; transcribe_session_params_init(&cp_macro); @@ -425,15 +427,12 @@ static void test_load_invalid(void) { TRANSCRIBE_ERR_BAD_STRUCT_SIZE); CHECK(m == NULL); - /* gpu_device selection is validated during load against the live device - * registry (in load_common::init_backends), not as an upfront reserved- - * field check — so a nonzero gpu_device no longer short-circuits to - * INVALID_ARG before the file is even opened. A missing file still - * surfaces as FILE_NOT_FOUND regardless of gpu_device. */ + /* Exact selection is validated by load_common after opening the model. A + * missing file therefore still reports FILE_NOT_FOUND first. */ struct transcribe_model_load_params mp_dev; transcribe_model_load_params_init(&mp_dev); - mp_dev.gpu_device = 1; - m = (struct transcribe_model *) 0xdeadbeef; + mp_dev.device = transcribe_device_get(0); + m = (struct transcribe_model *) 0xdeadbeef; CHECK(transcribe_model_load_file("/__transcribe_smoke_does_not_exist__.gguf", &mp_dev, &m) == TRANSCRIBE_ERR_FILE_NOT_FOUND); CHECK(m == NULL); diff --git a/tests/backend_init_throw_unit.cpp b/tests/backend_init_throw_unit.cpp index f3facaa2..078e66b5 100644 --- a/tests/backend_init_throw_unit.cpp +++ b/tests/backend_init_throw_unit.cpp @@ -55,7 +55,7 @@ void free_plan(transcribe::BackendPlan & plan) { void test_baseline_no_hook() { unset_env("TRANSCRIBE_TEST_DEV_INIT_THROW"); transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", plan); + const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", plan); CHECK(st == TRANSCRIBE_OK); CHECK(plan.primary != nullptr); CHECK(!plan.scheduler_list.empty()); @@ -65,7 +65,7 @@ void test_baseline_no_hook() { void test_nonmatching_hook_is_inert() { set_env("TRANSCRIBE_TEST_DEV_INIT_THROW", "no-such-device-name-xyzzy"); transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", plan); + const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", plan); CHECK(st == TRANSCRIBE_OK); CHECK(plan.primary != nullptr); free_plan(plan); @@ -77,11 +77,11 @@ void test_empty_hook_value_is_inert() { // POSIX-only: Windows deletes the variable when setting an empty value. unset_env("TRANSCRIBE_TEST_DEV_INIT_THROW"); transcribe::BackendPlan baseline; - CHECK(transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", baseline) == TRANSCRIBE_OK); + CHECK(transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", baseline) == TRANSCRIBE_OK); set_env("TRANSCRIBE_TEST_DEV_INIT_THROW", ""); transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", plan); + const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", plan); CHECK(st == TRANSCRIBE_OK); CHECK(plan.primary != nullptr); CHECK(plan.primary_kind == baseline.primary_kind); @@ -95,7 +95,7 @@ void test_empty_hook_value_is_inert() { void test_auto_falls_back_to_cpu_when_every_device_throws() { set_env("TRANSCRIBE_TEST_DEV_INIT_THROW", "*"); transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", plan); + const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", plan); CHECK(st == TRANSCRIBE_OK); CHECK(plan.primary != nullptr); CHECK(plan.primary_kind == transcribe::BackendKind::Cpu); @@ -113,7 +113,7 @@ void test_specific_gpu_request_fails_cleanly_when_every_device_throws() { }; for (const auto kind : kinds) { transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(kind, 0, "test", plan); + const transcribe_status st = transcribe::load_common::init_backends(kind, nullptr, "test", plan); CHECK(st == TRANSCRIBE_ERR_BACKEND); CHECK(plan.primary == nullptr); } @@ -124,7 +124,7 @@ void test_cpu_request_unaffected_by_hook() { set_env("TRANSCRIBE_TEST_DEV_INIT_THROW", "*"); for (const auto kind : { TRANSCRIBE_BACKEND_CPU, TRANSCRIBE_BACKEND_CPU_ACCEL }) { transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(kind, 0, "test", plan); + const transcribe_status st = transcribe::load_common::init_backends(kind, nullptr, "test", plan); CHECK(st == TRANSCRIBE_OK); CHECK(plan.primary_kind == transcribe::BackendKind::Cpu); free_plan(plan); @@ -132,14 +132,14 @@ void test_cpu_request_unaffected_by_hook() { unset_env("TRANSCRIBE_TEST_DEV_INIT_THROW"); } -void test_explicit_gpu_device_fails_cleanly_when_it_throws() { - // Only meaningful when a GPU/IGPU device sits at index > 0. +void test_explicit_device_fails_cleanly_when_it_throws() { + // Only meaningful when a GPU/IGPU device is registered. int gpu_index = -1; - const int n = transcribe_backend_device_count(); - for (int i = 1; i < n; ++i) { - struct transcribe_backend_device dev; - transcribe_backend_device_init(&dev); - if (transcribe_get_backend_device(i, &dev) != TRANSCRIBE_OK) { + const int n = transcribe_device_count(); + for (int i = 0; i < n; ++i) { + struct transcribe_device_info dev; + transcribe_device_info_init(&dev); + if (transcribe_device_get_info(transcribe_device_get(i), &dev) != TRANSCRIBE_OK) { continue; } if (dev.device_type == TRANSCRIBE_DEVICE_TYPE_GPU || dev.device_type == TRANSCRIBE_DEVICE_TYPE_IGPU) { @@ -153,7 +153,7 @@ void test_explicit_gpu_device_fails_cleanly_when_it_throws() { set_env("TRANSCRIBE_TEST_DEV_INIT_THROW", "*"); transcribe::BackendPlan plan; const transcribe_status st = - transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, gpu_index, "test", plan); + transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, transcribe_device_get(gpu_index), "test", plan); CHECK(st == TRANSCRIBE_ERR_BACKEND); CHECK(plan.primary == nullptr); unset_env("TRANSCRIBE_TEST_DEV_INIT_THROW"); @@ -174,7 +174,7 @@ int main() { test_auto_falls_back_to_cpu_when_every_device_throws(); test_specific_gpu_request_fails_cleanly_when_every_device_throws(); test_cpu_request_unaffected_by_hook(); - test_explicit_gpu_device_fails_cleanly_when_it_throws(); + test_explicit_device_fails_cleanly_when_it_throws(); if (g_failures != 0) { std::fprintf(stderr, "%d check(s) failed\n", g_failures); diff --git a/tests/backend_init_unit.cpp b/tests/backend_init_unit.cpp index 20f7084a..613a9f1e 100644 --- a/tests/backend_init_unit.cpp +++ b/tests/backend_init_unit.cpp @@ -10,7 +10,7 @@ // actually returns, not on registry probes — a device can be // registered but fail initialization. // - AUTO: always succeeds; asserts based on the returned primary_kind. -// - Explicit gpu_device: rejects invalid selectors and, when a +// - Exact device: rejects invalid selectors and, when a // nonzero GPU index exists, binds that exact registry device. // - Invalid enum: returns TRANSCRIBE_ERR_INVALID_ARG. @@ -83,7 +83,7 @@ int main() { // --------------------------------------------------------------- { BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CPU, 0, "test-cpu", plan); + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CPU, nullptr, "test-cpu", plan); REQUIRE(st == TRANSCRIBE_OK); CHECK_EQ(plan.primary_kind, BackendKind::Cpu); REQUIRE(plan.primary != nullptr); @@ -104,7 +104,7 @@ int main() { // --------------------------------------------------------------- { BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CPU_ACCEL, 0, "test-cpu-accel", plan); + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CPU_ACCEL, nullptr, "test-cpu-accel", plan); REQUIRE(st == TRANSCRIBE_OK); CHECK_EQ(plan.primary_kind, BackendKind::Cpu); REQUIRE(plan.primary != nullptr); @@ -129,7 +129,7 @@ int main() { // call init_backends() and assert based on what it returns. { BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_METAL, 0, "test-metal", plan); + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_METAL, nullptr, "test-metal", plan); if (st == TRANSCRIBE_OK) { CHECK_EQ(plan.primary_kind, BackendKind::Metal); @@ -146,7 +146,7 @@ int main() { // --------------------------------------------------------------- { BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_VULKAN, 0, "test-vulkan", plan); + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_VULKAN, nullptr, "test-vulkan", plan); if (st == TRANSCRIBE_OK) { CHECK_EQ(plan.primary_kind, BackendKind::Vulkan); @@ -163,7 +163,7 @@ int main() { // --------------------------------------------------------------- { BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CUDA, 0, "test-cuda", plan); + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CUDA, nullptr, "test-cuda", plan); if (st == TRANSCRIBE_OK) { CHECK_EQ(plan.primary_kind, BackendKind::Cuda); @@ -180,7 +180,7 @@ int main() { // --------------------------------------------------------------- { BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_ROCM, 0, "test-rocm", plan); + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_ROCM, nullptr, "test-rocm", plan); if (st == TRANSCRIBE_OK) { CHECK_EQ(plan.primary_kind, BackendKind::Rocm); @@ -202,7 +202,7 @@ int main() { // primary_kind rather than pre-judging from the registry. { BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test-auto", plan); + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test-auto", plan); REQUIRE(st == TRANSCRIBE_OK); CHECK(plan.primary != nullptr); CHECK(plan.primary_kind != BackendKind::Unknown); @@ -224,59 +224,52 @@ int main() { // --------------------------------------------------------------- { BackendPlan plan; - transcribe_status st = init_backends(static_cast(999), 0, "test-invalid", plan); + transcribe_status st = + init_backends(static_cast(999), nullptr, "test-invalid", plan); CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); } // --------------------------------------------------------------- - // 8. Explicit gpu_device validation + // 8. Exact device selection, including registry index zero // --------------------------------------------------------------- - { - BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_AUTO, -1, "test-gpu-negative", plan); - CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); - } - { - BackendPlan plan; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CPU, 1, "test-gpu-cpu-request", plan); - CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); - } - { - BackendPlan plan; - const int out_of_range = static_cast(ggml_backend_dev_count()) + 1; - transcribe_status st = init_backends(TRANSCRIBE_BACKEND_AUTO, out_of_range, "test-gpu-out-of-range", plan); - CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); - } - const size_t n_dev = ggml_backend_dev_count(); - for (size_t i = 1; i < n_dev; ++i) { + for (size_t i = 0; i < n_dev; ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); - if (dev != nullptr && !is_gpu_device(dev)) { - BackendPlan plan; - transcribe_status st = - init_backends(TRANSCRIBE_BACKEND_AUTO, static_cast(i), "test-gpu-non-gpu", plan); - CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); - break; + if (dev == nullptr || ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) { + continue; + } + BackendPlan plan; + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_AUTO, reinterpret_cast(dev), + "test-device-explicit", plan); + if (st == TRANSCRIBE_OK) { + CHECK(plan.primary != nullptr); + CHECK(ggml_backend_get_device(plan.primary) == dev); + free_plan(plan); + } else { + CHECK_EQ(st, TRANSCRIBE_ERR_BACKEND); } } - for (size_t i = 1; i < n_dev; ++i) { + for (size_t i = 0; i < n_dev; ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); if (dev != nullptr && is_gpu_device(dev)) { BackendPlan plan; - transcribe_status st = - init_backends(TRANSCRIBE_BACKEND_AUTO, static_cast(i), "test-gpu-explicit", plan); - if (st == TRANSCRIBE_OK) { - CHECK(plan.primary != nullptr); - CHECK(ggml_backend_get_device(plan.primary) == dev); - free_plan(plan); - } else { - CHECK_EQ(st, TRANSCRIBE_ERR_BACKEND); - } + transcribe_status st = init_backends(TRANSCRIBE_BACKEND_CPU, reinterpret_cast(dev), + "test-device-kind-mismatch", plan); + CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); break; } } + { + int foreign_storage = 0; + BackendPlan plan; + transcribe_status st = + init_backends(TRANSCRIBE_BACKEND_AUTO, reinterpret_cast(&foreign_storage), + "test-device-foreign", plan); + CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); + } + // --------------------------------------------------------------- // Summary // --------------------------------------------------------------- diff --git a/tests/backend_metal_simdgroup_gate_unit.cpp b/tests/backend_metal_simdgroup_gate_unit.cpp index b3485ba2..e35642d2 100644 --- a/tests/backend_metal_simdgroup_gate_unit.cpp +++ b/tests/backend_metal_simdgroup_gate_unit.cpp @@ -93,7 +93,7 @@ void test_hook_unset_matches_hardware_verdict() { } unset_env("TRANSCRIBE_TEST_METAL_NO_SIMDGROUP_MM"); transcribe::BackendPlan plan; - CHECK(transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", plan) == TRANSCRIBE_OK); + CHECK(transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", plan) == TRANSCRIBE_OK); CHECK(plan.primary_kind == expected_auto_kind()); free_plan(plan); } @@ -105,7 +105,7 @@ void test_nonmatching_hook_is_inert() { } set_env("TRANSCRIBE_TEST_METAL_NO_SIMDGROUP_MM", "no-such-device-name-xyzzy"); transcribe::BackendPlan plan; - CHECK(transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", plan) == TRANSCRIBE_OK); + CHECK(transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", plan) == TRANSCRIBE_OK); CHECK(plan.primary_kind == expected_auto_kind()); free_plan(plan); unset_env("TRANSCRIBE_TEST_METAL_NO_SIMDGROUP_MM"); @@ -118,7 +118,7 @@ void test_auto_skips_gated_metal_and_falls_back_to_cpu() { } set_env("TRANSCRIBE_TEST_METAL_NO_SIMDGROUP_MM", "*"); transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, 0, "test", plan); + const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_AUTO, nullptr, "test", plan); CHECK(st == TRANSCRIBE_OK); CHECK(plan.primary != nullptr); CHECK(plan.primary_kind == transcribe::BackendKind::Cpu); @@ -134,7 +134,8 @@ void test_explicit_metal_is_honored_despite_gate() { } set_env("TRANSCRIBE_TEST_METAL_NO_SIMDGROUP_MM", "*"); transcribe::BackendPlan plan; - const transcribe_status st = transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_METAL, 0, "test", plan); + const transcribe_status st = + transcribe::load_common::init_backends(TRANSCRIBE_BACKEND_METAL, nullptr, "test", plan); CHECK(st == TRANSCRIBE_OK); CHECK(plan.primary_kind == transcribe::BackendKind::Metal); free_plan(plan); diff --git a/tests/cli_device_arg_smoke.cmake b/tests/cli_device_arg_smoke.cmake new file mode 100644 index 00000000..ef5ef5f5 --- /dev/null +++ b/tests/cli_device_arg_smoke.cmake @@ -0,0 +1,22 @@ +# Verify --device rejects malformed, negative, trailing-junk, and overflowing +# values instead of silently treating them as exact device 0. + +if(NOT DEFINED CLI OR NOT DEFINED WAV) + message(FATAL_ERROR "CLI and WAV are required") +endif() + +foreach(value IN ITEMS abc 0x1 1tail -1 2147483648) + execute_process( + COMMAND "${CLI}" --device "${value}" "${WAV}" + RESULT_VARIABLE result + OUTPUT_VARIABLE stdout + ERROR_VARIABLE stderr) + if(result EQUAL 0) + message(FATAL_ERROR + "--device ${value} unexpectedly succeeded\nstdout:\n${stdout}\nstderr:\n${stderr}") + endif() + if(NOT stderr MATCHES "--device must be an integer index >= 0") + message(FATAL_ERROR + "--device ${value} returned the wrong diagnostic\nstderr:\n${stderr}") + endif() +endforeach() diff --git a/tools/transcribe-bench/main.cpp b/tools/transcribe-bench/main.cpp index 2516f863..d4194c4b 100644 --- a/tools/transcribe-bench/main.cpp +++ b/tools/transcribe-bench/main.cpp @@ -13,6 +13,7 @@ #include "transcribe.h" #include "wav.h" +#include #include #include #include @@ -23,6 +24,20 @@ namespace { +bool parse_device_index(const char * text, int & out) { + if (text == nullptr || text[0] == '\0') { + return false; + } + const char * end = text + std::strlen(text); + int parsed = 0; + const auto result = std::from_chars(text, end, parsed); + if (result.ec != std::errc{} || result.ptr != end || parsed < 0) { + return false; + } + out = parsed; + return true; +} + struct bench_args { std::string model_path; std::string sample_path; @@ -32,7 +47,7 @@ struct bench_args { int n_threads = 0; bool quiet = false; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int gpu_device = 0; // --device N: 0 = auto, >0 = index + int device_index = -1; // --device N: -1 = auto, >=0 = exact device // Passed through to transcribe_run_params::spec_k_drafts. -1 = family // default, 0 = spec decode off, > 0 = explicit draft length. Silently // ignored by families without supports_spec_decode. Set by @@ -56,8 +71,8 @@ void print_usage(const char * argv0) { " cpu is strict CPU (no GPU, no BLAS/AMX).\n" " cpu_accel is CPU + host-memory accelerators\n" " (BLAS/AMX) when the build includes them.\n" - " --device N GPU device index: 0 = auto (first of kind),\n" - " >0 selects that ggml registry index\n" + " --device N exact device index from transcribe-cli --list-devices,\n" + " including 0 (default: automatic selection)\n" " --spec-k-drafts N speculative-decode draft length on the offline\n" " path: -1 = family default, 0 = off, > 0 = K.\n" " Ignored by families without spec support.\n" @@ -190,9 +205,8 @@ bool parse_args(int argc, char ** argv, bench_args & out) { if (!v) { return false; } - out.gpu_device = std::atoi(v); - if (out.gpu_device < 0) { - std::fprintf(stderr, "error: --device must be >= 0 (0 = auto)\n"); + if (!parse_device_index(v, out.device_index)) { + std::fprintf(stderr, "error: --device must be an integer index >= 0\n"); return false; } } else { @@ -316,8 +330,12 @@ int main(int argc, char ** argv) { } struct transcribe_model_load_params mp; transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.gpu_device = args.gpu_device; + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; + } struct transcribe_model * model = nullptr; if (const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); st != TRANSCRIBE_OK) {