fix: engines default to no runtime cache; the module owns the implicit one

py/torch_tensorrt/dynamo/runtime/_TRTEngine.py

@@ -282,11 +282,18 @@ def __init__(
         # engines compiled with native multi-device collective layers.
         self._nccl_comm: Optional[Any] = None
 
-        # Owns RuntimeSettings + the live trt.IRuntimeConfig + the
-        # engine-implicit RuntimeCache. Hides all RTX feature gates.
-        # ``RuntimeSettings`` default; callers wanting non-defaults assign via
-        # the module's ``runtime_settings`` setter after compile.
-        self._trt_runtime_config: TRTRuntimeConfig = TRTRuntimeConfig(RuntimeSettings())
+        # Owns RuntimeSettings + the live trt.IRuntimeConfig. Hides all RTX
+        # feature gates.
+        #
+        # ``runtime_cache=None``: an engine never owns a runtime cache. The
+        # module owns the implicit one and pushes it down via
+        # ``setup_engine``; an engine with no module (a packed-engine-info
+        # build, or a constant in an AOT-loaded ExportedProgram) runs without
+        # one until a caller attaches a ``RuntimeCache`` explicitly. Mirrors
+        # the cpp default (``RuntimeSettings::runtime_cache = nullptr``).
+        self._trt_runtime_config: TRTRuntimeConfig = TRTRuntimeConfig(
+            RuntimeSettings(runtime_cache=None)
+        )
         # Multiple optimization profiles. Manual selection by default:
         # ``_active_profile_index`` is the profile currently loaded in the TRT
         # context (default 0, reused across calls). ``_auto_select_profiles``
@@ -399,10 +406,11 @@ def __setstate__(self, state: Any) -> None:
         # NCCL communicators cannot be pickled; rebind lazily on the next
         # forward pass via setup_nccl_comm().
         self._nccl_comm = None
-        # RuntimeSettings are NOT serialized -- restore defaults. Callers
-        # who want runtime-mode overrides must reapply them post-load via
-        # ``mod.runtime_settings = ...`` (per ``TorchTensorRTModule``) or a runtime CM.
-        self._trt_runtime_config = TRTRuntimeConfig(RuntimeSettings())
+        # RuntimeSettings are NOT serialized -- restore defaults, runtime cache
+        # included (see ``__init__``). Callers who want runtime-mode overrides
+        # must reapply them post-load via ``mod.runtime_settings = ...`` (per
+        # ``TorchTensorRTModule``) or a runtime CM.
+        self._trt_runtime_config = TRTRuntimeConfig(RuntimeSettings(runtime_cache=None))
 
         self._active_profile_index = 0
         self._auto_select_profiles = False

py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py

@@ -357,6 +357,10 @@ def runtime_settings(self, rs: RuntimeSettings) -> None:
     def _resolve_runtime_cache(self, rs: RuntimeSettings) -> RuntimeSettings:
         """Normalize ``rs.runtime_cache`` to ``None`` | ``RuntimeCache`` (never a path str).
 
+        This module is the only place a path string is resolved; engines reject
+        one (see ``TRTRuntimeConfig._apply_settings``), so the returned settings
+        must never carry a ``str``.
+
         Manages the ``_implicit_cache_handle`` slot as a side effect: builds a
         fresh wrapper for a new path, reuses the existing one for the same
         path, releases it (with save-on-swap) for non-path inputs.
@@ -372,7 +376,9 @@ def _resolve_runtime_cache(self, rs: RuntimeSettings) -> RuntimeSettings:
         if not (isinstance(rc, str) and rc):
             if rc is not self._implicit_cache_handle:
                 self._set_managed_handle(None)
-            return rs
+            # An empty string means "no cache", but engines take only None or a
+            # RuntimeCache -- normalize so no str ever reaches one.
+            return rs.merge(runtime_cache=None) if isinstance(rc, str) else rs
 
         # Branch 2: same path + wrapper still usable -> reuse. Keeps the CM
         # enter/exit cycle cheap (no teardown/rebuild loses in-memory kernels).
@@ -566,9 +572,11 @@ def set_extra_state(self, state: SerializedTorchTensorRTModuleFmt) -> None:
             self.settings = metadata["settings"]
             self.symbolic_shape_expressions = metadata["inout_symexprs"]
 
-            # RuntimeSettings are NOT serialized; restore defaults. Caller can
-            # reapply via ``mod.runtime_settings = ...`` (per submodule) or a CM after load.
-            self._runtime_settings = RuntimeSettings()
+            # RuntimeSettings are NOT serialized; the reset leaves no runtime
+            # cache, matching the freshly-built engine below. A caller who wants
+            # one reapplies via ``mod.runtime_settings = ...`` (per submodule) or
+            # a CM after load.
+            self._runtime_settings = RuntimeSettings(runtime_cache=None)
             # Mirror the settings reset on the implicit cache handle so a
             # stale wrapper from prior use doesn't survive load_state_dict and
             # silently write the fresh engine's cache bytes to the old path.
@@ -682,7 +690,7 @@ def __getstate__(self) -> dict[str, Any]:
         return state
 
     def __setstate__(self, state: dict[str, Any]) -> None:
-        state.setdefault("_runtime_settings", RuntimeSettings())
+        state.setdefault("_runtime_settings", RuntimeSettings(runtime_cache=None))
         state.setdefault("_implicit_cache_handle", None)
         set_state = getattr(super(), "__setstate__", None)
         if set_state is not None:

py/torch_tensorrt/runtime/_runtime_config.py

@@ -68,13 +68,13 @@ class RuntimeSettings:
             TRT-RTX-only; no-op on standard TensorRT.
         cuda_graph_strategy: ``"disabled" | "whole_graph_capture"``. TRT-RTX-only.
         runtime_cache: ``None``, a disk path string, or a
-            :class:`RuntimeCache`. ``None`` ⇒ no cache attached. A
-            string is honored at engine construction time and primes a
-            per-engine disk-backed cache (engine owns the implicit handle and
-            it saves on ``__del__``). A handle is the shared-cache form,
-            typically obtained from :func:`torch_tensorrt.runtime.runtime_cache`
-            -- multiple engines attaching the same handle share one
-            ``IRuntimeCache``.
+            :class:`RuntimeCache`. ``None`` ⇒ no cache attached. A string is
+            resolved by :class:`TorchTensorRTModule`, which builds the
+            engine-implicit :class:`RuntimeCache`, owns it, and saves it on
+            ``__del__``; engines themselves only ever receive ``None`` or a
+            handle. A handle is the shared-cache form, typically obtained from
+            :func:`torch_tensorrt.runtime.runtime_cache` -- multiple engines
+            attaching the same handle share one ``IRuntimeCache``.
 
     Equality compares all fields; for ``runtime_cache``, handle equality is
     by identity (same handle ⇒ same cache).
@@ -149,7 +149,9 @@ class TRTRuntimeConfig:
     """
 
     def __init__(self, settings: Optional[RuntimeSettings] = None) -> None:
-        self._settings: RuntimeSettings = settings or RuntimeSettings()
+        self._settings: RuntimeSettings = settings or RuntimeSettings(
+            runtime_cache=None
+        )
         # Live trt.IRuntimeConfig (RTX) or None (non-RTX / pre-init).
         self._live: Any = None
 
@@ -164,8 +166,8 @@ def set_settings(self, new: RuntimeSettings) -> bool:
         On change, invalidates the live ``IRuntimeConfig`` and signals callers
         to recreate the ``IExecutionContext``. Disk persistence of any prior
         implicit cache handle is the module's responsibility (see
-        ``TorchTensorRTModule._materialize_implicit_handle``); this method is
-        a pure-execution swap.
+        ``TorchTensorRTModule._set_managed_handle``); this method is a
+        pure-execution swap.
         """
         if new == self._settings:
             return False
@@ -184,7 +186,13 @@ def ensure_initialized(self, cuda_engine: Any) -> None:
         if self._live is not None:
             return
         self._live = cuda_engine.create_runtime_config()
-        self._apply_settings()
+        try:
+            self._apply_settings()
+        except Exception:
+            # Reset the live config so a later ensure_initialized rebuilds it
+            # instead of short-circuiting on the guard above.
+            self._live = None
+            raise
 
     def reset(self) -> None:
         """Drop the live ``IRuntimeConfig``; the next ``ensure_initialized`` rebuilds."""
@@ -248,15 +256,18 @@ def is_monolithic_capturable(
     def _apply_settings(self) -> None:
         """Apply ``self._settings`` to the live ``trt.IRuntimeConfig``.
 
-        Resolves ``runtime_cache``:
-        - ``None`` ⇒ no cache attached.
-        - ``RuntimeCache`` ⇒ caller owns lifecycle. ``ensure_cache``
-          materializes the inner ``IRuntimeCache`` on first use and drains
-          any pending warm bytes loaded into the handle's pending buffer at
-          construction time (by ``_TorchTensorRTModule._resolve_runtime_cache``
-          for engine-implicit handles, or by the ``runtime_cache`` CM for
-          shared ones). String paths are pre-wrapped into handles upstream;
-          raw strings are not accepted here.
+        Resolves ``runtime_cache``, which must be ``None`` or a
+        :class:`RuntimeCache` -- something that *owns* what it points at. A
+        ``RuntimeCache``'s ``ensure_cache`` materializes the inner
+        ``IRuntimeCache`` on first use and drains any pending warm bytes loaded
+        into the handle's pending buffer at construction time (by
+        ``_TorchTensorRTModule._resolve_runtime_cache`` for engine-implicit
+        handles, or by the ``runtime_cache`` CM for shared ones).
+
+        Path strings are resolved upstream by ``TorchTensorRTModule`` and are
+        rejected here: a string owns nothing, so honoring one would mean
+        building a handle this method does not outlive, handing its
+        ``IRuntimeCache`` to ``_live``, and letting it be collected on return.
         """
         # Deferred imports: trt is import-aliased to tensorrt_rtx on RTX builds,
         # and _runtime_cache imports this module's RuntimeSettings.
@@ -275,36 +286,15 @@ def _apply_settings(self) -> None:
 
         rc = self._settings.runtime_cache
         if rc is None:
-            logger.debug("Runtime cache disabled (no RuntimeCache / path provided).")
+            logger.debug("Runtime cache disabled (no RuntimeCache provided).")
         elif isinstance(rc, RuntimeCache):
-            cache = rc.ensure_cache(self._live)
-            self._live.set_runtime_cache(cache)
-        elif isinstance(rc, str):
-            # ``TorchTensorRTModule._resolve_runtime_cache`` pre-wraps path
-            # strings on the compile / configure path, but engines created
-            # directly (e.g. the Python ``TRTEngine`` constructed from a
-            # cross-runtime ``.pt2`` load — see
-            # ``test_cross_runtime_serde::test_save_python_load_python``)
-            # get a default ``RuntimeSettings(runtime_cache=RUNTIME_CACHE_PATH)``
-            # that's never seen by the module's resolver. Wrap defensively
-            # here so the load path doesn't crash; this also keeps the
-            # documented contract that callers MAY pass a path string.
-            #
-            # ``RuntimeSettings`` is a frozen dataclass, so we can't store the
-            # wrapper back onto ``self._settings``; just use it locally. The
-            # wrapper is GC'd after this call, which is fine: ensure_cache has
-            # already materialized the underlying IRuntimeCache on ``_live``.
-            wrapped = RuntimeCache(path=rc, autosave_on_del=True)
-            try:
-                wrapped.load()
-            except Exception as e:
-                logger.warning(f"Failed to warm-load runtime cache from {rc!r}: {e}")
-            cache = wrapped.ensure_cache(self._live)
-            self._live.set_runtime_cache(cache)
+            self._live.set_runtime_cache(rc.ensure_cache(self._live))
         else:
             raise TypeError(
-                f"runtime_cache must be None, str, or RuntimeCache by the "
-                f"time it reaches TRTRuntimeConfig; got {type(rc).__name__}."
+                f"runtime_cache must be None or a RuntimeCache by the time it "
+                f"reaches TRTRuntimeConfig; got {type(rc).__name__}. Path "
+                f"strings are resolved by TorchTensorRTModule -- an engine "
+                f"used without one must be given a RuntimeCache explicitly."
             )
         logger.info("TensorRT-RTX runtime config configured")