From 28a6d9523c1eb56d1856f0a7edca21f0c485ea71 Mon Sep 17 00:00:00 2001 From: tejaswinp Date: Thu, 20 Aug 2026 23:39:39 -0700 Subject: [PATCH 1/4] fix: engines default to no runtime cache A TRTEngine built without a module came up with the default RuntimeSettings, whose runtime_cache is a path string. Nothing owned the wrapper that string implies, so attaching it handed the live IRuntimeCache to the engine's IRuntimeConfig and then let it be collected -- a use-after-free at the next createExecutionContext. Engines now default to runtime_cache=None, matching the cpp side, where RuntimeSettings::runtime_cache is an intrusive_ptr defaulting to nullptr and no string form exists. The implicit cache belongs to the module, which resolves its path string to a RuntimeCache and pushes it down in setup_engine; that path is unchanged. TorchTensorRTModule resets its own RuntimeSettings on the post-load paths (set_extra_state, __setstate__) and those resets move to runtime_cache=None too, so the module and the engine it rebuilds agree. Leaving them at the string default would let a later runtime_config(...) block resolve the stale path and install an autosaving handle at the shared default location on exit -- switching caching on via a call that never mentioned it. An engine reached without a module -- built from packed engine info, or loaded as a graph constant from a saved ExportedProgram -- now runs with no cache instead of a dangling one, and a caller can attach a RuntimeCache explicitly. No configuration loses working behaviour: on the Python runtime this path raised, on the cpp runtime it already attached nothing, and on standard TensorRT the runtime config is never initialized. --- .../dynamo/runtime/_TRTEngine.py | 26 ++++++++++++------- .../dynamo/runtime/_TorchTensorRTModule.py | 10 ++++--- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py b/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py index 3b28641c35..509846ca5e 100644 --- a/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py +++ b/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py @@ -282,11 +282,18 @@ def __init__( # engines compiled with native multi-device collective layers. self._nccl_comm: Optional[Any] = None - # Owns RuntimeSettings + the live trt.IRuntimeConfig + the - # engine-implicit RuntimeCache. Hides all RTX feature gates. - # ``RuntimeSettings`` default; callers wanting non-defaults assign via - # the module's ``runtime_settings`` setter after compile. - self._trt_runtime_config: TRTRuntimeConfig = TRTRuntimeConfig(RuntimeSettings()) + # Owns RuntimeSettings + the live trt.IRuntimeConfig. Hides all RTX + # feature gates. + # + # ``runtime_cache=None``: an engine never owns a runtime cache. The + # module owns the implicit one and pushes it down via + # ``setup_engine``; an engine with no module (a packed-engine-info + # build, or a constant in an AOT-loaded ExportedProgram) runs without + # one until a caller attaches a ``RuntimeCache`` explicitly. Mirrors + # the cpp default (``RuntimeSettings::runtime_cache = nullptr``). + self._trt_runtime_config: TRTRuntimeConfig = TRTRuntimeConfig( + RuntimeSettings(runtime_cache=None) + ) # Multiple optimization profiles. Manual selection by default: # ``_active_profile_index`` is the profile currently loaded in the TRT # context (default 0, reused across calls). ``_auto_select_profiles`` @@ -399,10 +406,11 @@ def __setstate__(self, state: Any) -> None: # NCCL communicators cannot be pickled; rebind lazily on the next # forward pass via setup_nccl_comm(). self._nccl_comm = None - # RuntimeSettings are NOT serialized -- restore defaults. Callers - # who want runtime-mode overrides must reapply them post-load via - # ``mod.runtime_settings = ...`` (per ``TorchTensorRTModule``) or a runtime CM. - self._trt_runtime_config = TRTRuntimeConfig(RuntimeSettings()) + # RuntimeSettings are NOT serialized -- restore defaults, runtime cache + # included (see ``__init__``). Callers who want runtime-mode overrides + # must reapply them post-load via ``mod.runtime_settings = ...`` (per + # ``TorchTensorRTModule``) or a runtime CM. + self._trt_runtime_config = TRTRuntimeConfig(RuntimeSettings(runtime_cache=None)) self._active_profile_index = 0 self._auto_select_profiles = False diff --git a/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py b/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py index 6028359130..6f7dc5cf65 100644 --- a/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py +++ b/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py @@ -566,9 +566,11 @@ def set_extra_state(self, state: SerializedTorchTensorRTModuleFmt) -> None: self.settings = metadata["settings"] self.symbolic_shape_expressions = metadata["inout_symexprs"] - # RuntimeSettings are NOT serialized; restore defaults. Caller can - # reapply via ``mod.runtime_settings = ...`` (per submodule) or a CM after load. - self._runtime_settings = RuntimeSettings() + # RuntimeSettings are NOT serialized; the reset leaves no runtime + # cache, matching the freshly-built engine below. A caller who wants + # one reapplies via ``mod.runtime_settings = ...`` (per submodule) or + # a CM after load. + self._runtime_settings = RuntimeSettings(runtime_cache=None) # Mirror the settings reset on the implicit cache handle so a # stale wrapper from prior use doesn't survive load_state_dict and # silently write the fresh engine's cache bytes to the old path. @@ -682,7 +684,7 @@ def __getstate__(self) -> dict[str, Any]: return state def __setstate__(self, state: dict[str, Any]) -> None: - state.setdefault("_runtime_settings", RuntimeSettings()) + state.setdefault("_runtime_settings", RuntimeSettings(runtime_cache=None)) state.setdefault("_implicit_cache_handle", None) set_state = getattr(super(), "__setstate__", None) if set_state is not None: From 8c5aa83e2f03434b6b95214f31d24bf368aca2ff Mon Sep 17 00:00:00 2001 From: tejaswinp Date: Thu, 20 Aug 2026 23:40:44 -0700 Subject: [PATCH 2/4] refactor: TRTRuntimeConfig takes only None or a RuntimeCache _apply_settings had three arms, and the str one built a RuntimeCache it did not outlive. Its own docstring already claimed raw strings were not accepted here; the code twenty lines below accepted them. Engines now take only something that owns what it points at -- the Python equivalent of the cpp intrusive_ptr. A str raises TypeError naming the module as the place path strings are resolved. The class's own default follows: a TRTRuntimeConfig built with no settings would otherwise start from the string form this commit exists to abolish. TorchTensorRTModule._resolve_runtime_cache normalizes an empty-string runtime_cache to None rather than passing it through, so no str can reach an engine from the module. Also corrects the RuntimeSettings.runtime_cache docstring, which promised the engine owned the implicit handle and saved it on __del__, and a reference to a method renamed some time ago. --- .../dynamo/runtime/_TorchTensorRTModule.py | 8 +- py/torch_tensorrt/runtime/_runtime_config.py | 76 ++++++++----------- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py b/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py index 6f7dc5cf65..b52bb8d360 100644 --- a/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py +++ b/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py @@ -357,6 +357,10 @@ def runtime_settings(self, rs: RuntimeSettings) -> None: def _resolve_runtime_cache(self, rs: RuntimeSettings) -> RuntimeSettings: """Normalize ``rs.runtime_cache`` to ``None`` | ``RuntimeCache`` (never a path str). + This module is the only place a path string is resolved; engines reject + one (see ``TRTRuntimeConfig._apply_settings``), so the returned settings + must never carry a ``str``. + Manages the ``_implicit_cache_handle`` slot as a side effect: builds a fresh wrapper for a new path, reuses the existing one for the same path, releases it (with save-on-swap) for non-path inputs. @@ -372,7 +376,9 @@ def _resolve_runtime_cache(self, rs: RuntimeSettings) -> RuntimeSettings: if not (isinstance(rc, str) and rc): if rc is not self._implicit_cache_handle: self._set_managed_handle(None) - return rs + # An empty string means "no cache", but engines take only None or a + # RuntimeCache -- normalize so no str ever reaches one. + return rs.merge(runtime_cache=None) if isinstance(rc, str) else rs # Branch 2: same path + wrapper still usable -> reuse. Keeps the CM # enter/exit cycle cheap (no teardown/rebuild loses in-memory kernels). diff --git a/py/torch_tensorrt/runtime/_runtime_config.py b/py/torch_tensorrt/runtime/_runtime_config.py index 1b2cbee643..d76d437261 100644 --- a/py/torch_tensorrt/runtime/_runtime_config.py +++ b/py/torch_tensorrt/runtime/_runtime_config.py @@ -68,13 +68,13 @@ class RuntimeSettings: TRT-RTX-only; no-op on standard TensorRT. cuda_graph_strategy: ``"disabled" | "whole_graph_capture"``. TRT-RTX-only. runtime_cache: ``None``, a disk path string, or a - :class:`RuntimeCache`. ``None`` ⇒ no cache attached. A - string is honored at engine construction time and primes a - per-engine disk-backed cache (engine owns the implicit handle and - it saves on ``__del__``). A handle is the shared-cache form, - typically obtained from :func:`torch_tensorrt.runtime.runtime_cache` - -- multiple engines attaching the same handle share one - ``IRuntimeCache``. + :class:`RuntimeCache`. ``None`` ⇒ no cache attached. A string is + resolved by :class:`TorchTensorRTModule`, which builds the + engine-implicit :class:`RuntimeCache`, owns it, and saves it on + ``__del__``; engines themselves only ever receive ``None`` or a + handle. A handle is the shared-cache form, typically obtained from + :func:`torch_tensorrt.runtime.runtime_cache` -- multiple engines + attaching the same handle share one ``IRuntimeCache``. Equality compares all fields; for ``runtime_cache``, handle equality is by identity (same handle ⇒ same cache). @@ -149,7 +149,9 @@ class TRTRuntimeConfig: """ def __init__(self, settings: Optional[RuntimeSettings] = None) -> None: - self._settings: RuntimeSettings = settings or RuntimeSettings() + self._settings: RuntimeSettings = settings or RuntimeSettings( + runtime_cache=None + ) # Live trt.IRuntimeConfig (RTX) or None (non-RTX / pre-init). self._live: Any = None @@ -164,8 +166,8 @@ def set_settings(self, new: RuntimeSettings) -> bool: On change, invalidates the live ``IRuntimeConfig`` and signals callers to recreate the ``IExecutionContext``. Disk persistence of any prior implicit cache handle is the module's responsibility (see - ``TorchTensorRTModule._materialize_implicit_handle``); this method is - a pure-execution swap. + ``TorchTensorRTModule._set_managed_handle``); this method is a + pure-execution swap. """ if new == self._settings: return False @@ -248,15 +250,18 @@ def is_monolithic_capturable( def _apply_settings(self) -> None: """Apply ``self._settings`` to the live ``trt.IRuntimeConfig``. - Resolves ``runtime_cache``: - - ``None`` ⇒ no cache attached. - - ``RuntimeCache`` ⇒ caller owns lifecycle. ``ensure_cache`` - materializes the inner ``IRuntimeCache`` on first use and drains - any pending warm bytes loaded into the handle's pending buffer at - construction time (by ``_TorchTensorRTModule._resolve_runtime_cache`` - for engine-implicit handles, or by the ``runtime_cache`` CM for - shared ones). String paths are pre-wrapped into handles upstream; - raw strings are not accepted here. + Resolves ``runtime_cache``, which must be ``None`` or a + :class:`RuntimeCache` -- something that *owns* what it points at. A + ``RuntimeCache``'s ``ensure_cache`` materializes the inner + ``IRuntimeCache`` on first use and drains any pending warm bytes loaded + into the handle's pending buffer at construction time (by + ``_TorchTensorRTModule._resolve_runtime_cache`` for engine-implicit + handles, or by the ``runtime_cache`` CM for shared ones). + + Path strings are resolved upstream by ``TorchTensorRTModule`` and are + rejected here: a string owns nothing, so honoring one would mean + building a handle this method does not outlive, handing its + ``IRuntimeCache`` to ``_live``, and letting it be collected on return. """ # Deferred imports: trt is import-aliased to tensorrt_rtx on RTX builds, # and _runtime_cache imports this module's RuntimeSettings. @@ -275,36 +280,15 @@ def _apply_settings(self) -> None: rc = self._settings.runtime_cache if rc is None: - logger.debug("Runtime cache disabled (no RuntimeCache / path provided).") + logger.debug("Runtime cache disabled (no RuntimeCache provided).") elif isinstance(rc, RuntimeCache): - cache = rc.ensure_cache(self._live) - self._live.set_runtime_cache(cache) - elif isinstance(rc, str): - # ``TorchTensorRTModule._resolve_runtime_cache`` pre-wraps path - # strings on the compile / configure path, but engines created - # directly (e.g. the Python ``TRTEngine`` constructed from a - # cross-runtime ``.pt2`` load — see - # ``test_cross_runtime_serde::test_save_python_load_python``) - # get a default ``RuntimeSettings(runtime_cache=RUNTIME_CACHE_PATH)`` - # that's never seen by the module's resolver. Wrap defensively - # here so the load path doesn't crash; this also keeps the - # documented contract that callers MAY pass a path string. - # - # ``RuntimeSettings`` is a frozen dataclass, so we can't store the - # wrapper back onto ``self._settings``; just use it locally. The - # wrapper is GC'd after this call, which is fine: ensure_cache has - # already materialized the underlying IRuntimeCache on ``_live``. - wrapped = RuntimeCache(path=rc, autosave_on_del=True) - try: - wrapped.load() - except Exception as e: - logger.warning(f"Failed to warm-load runtime cache from {rc!r}: {e}") - cache = wrapped.ensure_cache(self._live) - self._live.set_runtime_cache(cache) + self._live.set_runtime_cache(rc.ensure_cache(self._live)) else: raise TypeError( - f"runtime_cache must be None, str, or RuntimeCache by the " - f"time it reaches TRTRuntimeConfig; got {type(rc).__name__}." + f"runtime_cache must be None or a RuntimeCache by the time it " + f"reaches TRTRuntimeConfig; got {type(rc).__name__}. Path " + f"strings are resolved by TorchTensorRTModule -- an engine " + f"used without one must be given a RuntimeCache explicitly." ) logger.info("TensorRT-RTX runtime config configured") From 7fffe7c7b058bc69e0f21725a76b7204ce8b4b58 Mon Sep 17 00:00:00 2001 From: tejaswinp Date: Thu, 20 Aug 2026 23:41:51 -0700 Subject: [PATCH 3/4] test: cover runtime cache ownership across module and module-less paths TestEngineOwnsNoCache pins the contract at the engine: a module-less engine defaults to no cache, executes without one, accepts an explicitly attached RuntimeCache, and raises TypeError on a path string. TestModuleStillOwnsImplicitCache guards the other direction -- the compile path must keep building, attaching and persisting its implicit cache -- and covers the empty-string normalization. TestPostLoadOwnsNoCache pins the same contract on the reset paths, where module and engine could otherwise disagree: the config's own default, and what torch.load / load_state_dict leave behind. Its last test is the one that matters -- a cuda-graph-only context manager over a loaded module must not install a cache on enter or leave one installed on exit, because re-applying a path string through the setter creates a handle rather than restoring one. --- .../dynamo/runtime/test_000_runtime_cache.py | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/tests/py/dynamo/runtime/test_000_runtime_cache.py b/tests/py/dynamo/runtime/test_000_runtime_cache.py index 5616a234c8..dd17dc15f0 100644 --- a/tests/py/dynamo/runtime/test_000_runtime_cache.py +++ b/tests/py/dynamo/runtime/test_000_runtime_cache.py @@ -757,5 +757,221 @@ def test_pending_warm_bytes_populated_at_construction(self): self.assertTrue(found, "No TorchTensorRTModule with implicit handle found") +@unittest.skipIf( + not ENABLED_FEATURES.tensorrt_rtx, + "Runtime cache is only available with TensorRT-RTX", +) +@unittest.skipIf( + ENABLED_FEATURES.torch_tensorrt_runtime, + "Module-less TRTEngine construction requires the Python TRTEngine path", +) +class TestEngineOwnsNoCache(TestCase): + """An engine never owns a runtime cache; the module does. + + A module-less engine -- built from packed engine info, or loaded as a + graph constant from a saved ``ExportedProgram`` -- comes up with + ``runtime_cache=None`` and must run without one, rather than attaching a + handle nothing outlives. + """ + + def _bare_engine(self, compiled): + from torch_tensorrt.dynamo.runtime._TRTEngine import TRTEngine + + mod = _find_python_trt_module(compiled) + self.assertIsNotNone(mod, "expected a TorchTensorRTModule after compile") + return TRTEngine(mod._pack_engine_info()) + + @staticmethod + def _first(out): + """``execute`` returns a bare Tensor for single-output engines.""" + return out if isinstance(out, torch.Tensor) else out[0] + + def test_bare_engine_defaults_to_no_cache(self): + model, inputs = _fresh_conv_model_and_inputs() + engine = self._bare_engine(_compile(model, inputs)) + self.assertIsNone(engine.runtime_settings.runtime_cache) + + def test_bare_engine_executes_without_a_cache(self): + model, inputs = _fresh_conv_model_and_inputs() + compiled = _compile(model, inputs) + ref = compiled(*inputs) + engine = self._bare_engine(compiled) + out = self._first(engine.execute(list(inputs))) + self.assertGreater(cosine_similarity(ref, out), COSINE_THRESHOLD) + + def test_bare_engine_takes_an_explicit_cache(self): + """The opt-in an AOT deployment is expected to make.""" + model, inputs = _fresh_conv_model_and_inputs() + compiled = _compile(model, inputs) + ref = compiled(*inputs) + engine = self._bare_engine(compiled) + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "rc.bin") + handle = torchtrt.runtime.RuntimeCache(path=path) + engine.update_runtime_settings(RuntimeSettings(runtime_cache=handle)) + out = self._first(engine.execute(list(inputs))) + self.assertGreater(cosine_similarity(ref, out), COSINE_THRESHOLD) + handle.save() + self.assertTrue(os.path.exists(path)) + self.assertGreater(os.path.getsize(path), 0) + + def test_path_string_on_an_engine_raises(self): + """Strings are the module's business; an engine rejects them.""" + model, inputs = _fresh_conv_model_and_inputs() + engine = self._bare_engine(_compile(model, inputs)) + with tempfile.TemporaryDirectory() as tmp: + engine.update_runtime_settings( + RuntimeSettings(runtime_cache=os.path.join(tmp, "rc.bin")) + ) + with self.assertRaisesRegex(TypeError, "must be None or a RuntimeCache"): + engine.execute(list(inputs)) + + +@unittest.skipIf( + not ENABLED_FEATURES.tensorrt_rtx, + "Runtime cache is only available with TensorRT-RTX", +) +class TestModuleStillOwnsImplicitCache(TestCase): + """Guard against over-correction: the compile path must keep caching.""" + + def test_compiled_module_persists_its_implicit_cache(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "rc.bin") + model, inputs = _fresh_conv_model_and_inputs() + compiled = _compile(model, inputs) + _apply_runtime_settings(compiled, RuntimeSettings(runtime_cache=path)) + compiled(*inputs) + del compiled + gc.collect() + self.assertTrue(os.path.exists(path), "implicit cache was not saved") + self.assertGreater(os.path.getsize(path), 0) + + def test_empty_path_string_is_normalized_to_none(self): + """An empty string means "no cache" and must not reach the engine.""" + from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import ( + TorchTensorRTModule, + ) + + model, inputs = _fresh_conv_model_and_inputs() + compiled = _compile(model, inputs) + _apply_runtime_settings(compiled, RuntimeSettings(runtime_cache="")) + + # Engine-flavor agnostic: the module's resolved settings are what gets + # dispatched, on both the Python and cpp runtimes. + mods = [ + m for _, m in compiled.named_modules() if isinstance(m, TorchTensorRTModule) + ] + self.assertTrue(mods, "expected a TorchTensorRTModule after compile") + for m in mods: + self.assertIsNone(m.runtime_settings.runtime_cache) + compiled(*inputs) + + +@unittest.skipIf( + not ENABLED_FEATURES.tensorrt_rtx, + "Runtime cache is only available with TensorRT-RTX", +) +class TestPostLoadOwnsNoCache(TestCase): + """The reset paths must leave no cache, matching the engine they rebuild. + + ``set_extra_state`` / ``__setstate__`` restore ``RuntimeSettings`` defaults + after a load. If that reset kept the default path *string*, the module would + disagree with the engine it just built (which comes up ``None``), and a later + ``runtime_config(...)`` block -- a call that need not mention caching at all -- + would resolve the string and leave an autosaving handle installed on exit. + """ + + def _round_tripped(self, tmp): + model, inputs = _fresh_conv_model_and_inputs() + compiled = _compile(model, inputs) + path = os.path.join(tmp, "mod.pt") + torch.save(compiled, path) + return torch.load(path, weights_only=False), inputs + + def test_config_default_has_no_cache(self): + """A default-constructed config must not start from a path string.""" + from torch_tensorrt.runtime._runtime_config import TRTRuntimeConfig + + self.assertIsNone(TRTRuntimeConfig().settings.runtime_cache) + + def test_torch_load_leaves_no_cache(self): + from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import ( + TorchTensorRTModule, + ) + + with tempfile.TemporaryDirectory() as tmp: + loaded, _ = self._round_tripped(tmp) + mods = [ + m + for _, m in loaded.named_modules() + if isinstance(m, TorchTensorRTModule) + ] + self.assertTrue(mods, "expected a TorchTensorRTModule after load") + for m in mods: + self.assertIsNone(m.runtime_settings.runtime_cache) + + def test_load_state_dict_leaves_no_cache(self): + from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import ( + TorchTensorRTModule, + ) + + model, inputs = _fresh_conv_model_and_inputs() + compiled = _compile(model, inputs) + fresh, _ = _fresh_conv_model_and_inputs() + target = _compile(fresh, inputs) + target.load_state_dict(compiled.state_dict()) + + mods = [ + m for _, m in target.named_modules() if isinstance(m, TorchTensorRTModule) + ] + self.assertTrue(mods, "expected a TorchTensorRTModule after load_state_dict") + for m in mods: + self.assertIsNone(m.runtime_settings.runtime_cache) + + def test_unrelated_context_manager_does_not_install_a_cache(self): + """A cuda-graph-only CM must not switch caching on, on enter or on exit. + + The CM snapshots the module's pre-resolution view; re-applying a path + string through the setter *creates* a handle rather than restoring one, + so a stale string here would survive ``__exit__`` pointed at the shared + default path with ``autosave_on_del=True``. + """ + from torch_tensorrt.dynamo._defaults import RUNTIME_CACHE_PATH + from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import ( + TorchTensorRTModule, + ) + from torch_tensorrt.runtime import runtime_config + + default_existed = os.path.exists(RUNTIME_CACHE_PATH) + with tempfile.TemporaryDirectory() as tmp: + loaded, inputs = self._round_tripped(tmp) + mods = [ + m + for _, m in loaded.named_modules() + if isinstance(m, TorchTensorRTModule) + ] + self.assertTrue(mods, "expected a TorchTensorRTModule after load") + + with runtime_config(loaded, cuda_graph_strategy="whole_graph_capture"): + for m in mods: + self.assertIsNone( + m._implicit_cache_handle, "CM installed a cache on enter" + ) + loaded(*inputs) + + for m in mods: + self.assertIsNone( + m._implicit_cache_handle, "CM left a cache installed on exit" + ) + self.assertIsNone(m.runtime_settings.runtime_cache) + + if not default_existed: + self.assertFalse( + os.path.exists(RUNTIME_CACHE_PATH), + "an unrelated CM wrote the shared default cache file", + ) + + if __name__ == "__main__": run_tests() From cb95af3caa765ac93605a442493a5bbac7456c55 Mon Sep 17 00:00:00 2001 From: tejaswinp Date: Tue, 25 Aug 2026 15:24:48 -0700 Subject: [PATCH 4/4] fix: drop the live IRuntimeConfig when applying settings fails ensure_initialized assigns self._live before _apply_settings runs, so an exception out of the apply left a half-configured IRuntimeConfig in place -- strategies set, runtime cache never attached. The early-return guard at the top then made the next call a no-op, so a caller who caught the error and retried proceeded silently against those partial settings; the error was only ever raised once. Reset self._live and re-raise instead, so a retry re-attempts initialization and fails the same way. Measured before the change: first execute raised TypeError, second returned normally with _live still populated. --- py/torch_tensorrt/runtime/_runtime_config.py | 8 +++++- .../dynamo/runtime/test_000_runtime_cache.py | 25 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/py/torch_tensorrt/runtime/_runtime_config.py b/py/torch_tensorrt/runtime/_runtime_config.py index d76d437261..4fd16bc641 100644 --- a/py/torch_tensorrt/runtime/_runtime_config.py +++ b/py/torch_tensorrt/runtime/_runtime_config.py @@ -186,7 +186,13 @@ def ensure_initialized(self, cuda_engine: Any) -> None: if self._live is not None: return self._live = cuda_engine.create_runtime_config() - self._apply_settings() + try: + self._apply_settings() + except Exception: + # Reset the live config so a later ensure_initialized rebuilds it + # instead of short-circuiting on the guard above. + self._live = None + raise def reset(self) -> None: """Drop the live ``IRuntimeConfig``; the next ``ensure_initialized`` rebuilds.""" diff --git a/tests/py/dynamo/runtime/test_000_runtime_cache.py b/tests/py/dynamo/runtime/test_000_runtime_cache.py index dd17dc15f0..1c43e63a9b 100644 --- a/tests/py/dynamo/runtime/test_000_runtime_cache.py +++ b/tests/py/dynamo/runtime/test_000_runtime_cache.py @@ -816,6 +816,31 @@ def test_bare_engine_takes_an_explicit_cache(self): self.assertTrue(os.path.exists(path)) self.assertGreater(os.path.getsize(path), 0) + def test_failed_settings_do_not_leave_a_half_configured_config(self): + """A failed apply must raise every time, not just the first. + + ``ensure_initialized`` early-returns when the live ``IRuntimeConfig`` + exists, so a config left behind by a failed apply would make a retry + silently succeed against settings that were never fully applied. + """ + model, inputs = _fresh_conv_model_and_inputs() + engine = self._bare_engine(_compile(model, inputs)) + with tempfile.TemporaryDirectory() as tmp: + engine.update_runtime_settings( + RuntimeSettings(runtime_cache=os.path.join(tmp, "rc.bin")) + ) + for attempt in range(2): + with self.assertRaisesRegex( + TypeError, + "must be None or a RuntimeCache", + msg=f"attempt {attempt + 1} did not raise", + ): + engine.execute(list(inputs)) + self.assertIsNone( + engine._trt_runtime_config._live, + "a half-configured IRuntimeConfig survived the failure", + ) + def test_path_string_on_an_engine_raises(self): """Strings are the module's business; an engine rejects them.""" model, inputs = _fresh_conv_model_and_inputs()