feat(pathfinder): add CTK-root canary fallback for NVIDIA binaries

cuda_pathfinder/cuda/pathfinder/_binaries/find_nvidia_binary_utility.py

@@ -3,9 +3,9 @@
 
 import functools
 import os
-import shutil
 
 from cuda.pathfinder._binaries import supported_nvidia_binaries
+from cuda.pathfinder._utils.ctk_root_canary import CTK_ROOT_CANARY_ANCHOR_LIBNAMES
 from cuda.pathfinder._utils.env_vars import get_cuda_path_or_home
 from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages
 from cuda.pathfinder._utils.platform_aware import IS_WINDOWS
@@ -28,6 +28,45 @@ def _normalize_utility_name(utility_name: str) -> str:
     return utility_name
 
 
+def _is_executable_candidate(path: str) -> bool:
+    if not os.path.isfile(path):
+        return False
+    if IS_WINDOWS:
+        return True
+    return os.access(path, os.X_OK)
+
+
+def _ctk_bin_subdirs(root: str) -> list[str]:
+    if IS_WINDOWS:
+        return [
+            os.path.join(root, "bin", "x64"),
+            os.path.join(root, "bin", "x86_64"),
+            os.path.join(root, "bin"),
+        ]
+    return [os.path.join(root, "bin")]
+
+
+def _resolve_ctk_root_via_canary() -> str | None:
+    from cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib import resolve_ctk_root_via_canary
+
+    ctk_root: str | None = resolve_ctk_root_via_canary(CTK_ROOT_CANARY_ANCHOR_LIBNAMES[0])
+    return ctk_root
+
+
+def _resolve_in_trusted_dirs(normalized_name: str, dirs: list[str]) -> str | None:
+    """Resolve ``normalized_name`` against ``dirs`` in order."""
+    seen: set[str] = set()
+    for directory in dirs:
+        if directory in seen:
+            continue
+        assert directory
+        seen.add(directory)
+        candidate = os.path.join(directory, normalized_name)
+        if _is_executable_candidate(candidate):
+            return candidate
+    return None
+
+
 @functools.cache
 def find_nvidia_binary_utility(utility_name: str) -> str | None:
     """Locate a CUDA binary utility executable.
@@ -65,6 +104,12 @@ def find_nvidia_binary_utility(utility_name: str) -> str | None:
              ``bin/x64``, ``bin/x86_64``, and ``bin`` subdirectories on Windows,
              or just ``bin`` on Linux.
 
+        4. **CTK-root canary fallback**
+
+           - Only when steps 1-3 miss: resolve the ``cudart`` library through the
+             OS dynamic loader, derive the CUDA Toolkit root from it, and search
+             that root's bin layout.
+
     Note:
         Results are cached using ``@functools.cache`` for performance. The cache
         persists for the lifetime of the process.
@@ -73,6 +118,9 @@ def find_nvidia_binary_utility(utility_name: str) -> str | None:
         (``.exe``, ``.bat``, ``.cmd``). On Unix-like systems, executables
         are identified by the ``X_OK`` (execute) permission bit.
 
+        Lookup is restricted to the trusted directories and the canary-derived
+        CTK root listed above.
+
     Example:
         >>> from cuda.pathfinder import find_nvidia_binary_utility
         >>> nvdisasm = find_nvidia_binary_utility("nvdisasm")
@@ -98,10 +146,15 @@ def find_nvidia_binary_utility(utility_name: str) -> str | None:
 
     # 3. Search in CUDA Toolkit (CUDA_HOME/CUDA_PATH)
     if (cuda_home := get_cuda_path_or_home()) is not None:
-        if IS_WINDOWS:
-            dirs.append(os.path.join(cuda_home, "bin", "x64"))
-            dirs.append(os.path.join(cuda_home, "bin", "x86_64"))
-        dirs.append(os.path.join(cuda_home, "bin"))
+        dirs.extend(_ctk_bin_subdirs(cuda_home))
 
     normalized_name = _normalize_utility_name(utility_name)
-    return shutil.which(normalized_name, path=os.pathsep.join(dirs))
+    found = _resolve_in_trusted_dirs(normalized_name, dirs)
+    if found is not None:
+        return found
+
+    # 4. CTK-root canary fallback.
+    ctk_root = _resolve_ctk_root_via_canary()
+    if ctk_root is not None:
+        return _resolve_in_trusted_dirs(normalized_name, _ctk_bin_subdirs(ctk_root))
+    return None

cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py

@@ -8,6 +8,8 @@
 from dataclasses import dataclass
 from typing import Literal
 
+from cuda.pathfinder._utils.ctk_root_canary import CTK_ROOT_CANARY_ANCHOR_LIBNAMES
+
 PackagedWith = Literal["ctk", "other", "driver"]
 
 
@@ -73,7 +75,7 @@ class DescriptorSpec:
         site_packages_windows=("nvidia/cu13/bin/x86_64", "nvidia/cuda_nvcc/nvvm/bin"),
         anchor_rel_dirs_linux=("nvvm/lib64",),
         anchor_rel_dirs_windows=("nvvm/bin/*", "nvvm/bin"),
-        ctk_root_canary_anchor_libnames=("cudart",),
+        ctk_root_canary_anchor_libnames=CTK_ROOT_CANARY_ANCHOR_LIBNAMES,
     ),
     DescriptorSpec(
         name="cublas",
@@ -292,7 +294,7 @@ class DescriptorSpec:
         site_packages_windows=("nvidia/cu13/bin/x86_64", "nvidia/cuda_cupti/bin"),
         anchor_rel_dirs_linux=("extras/CUPTI/lib64", "lib"),
         anchor_rel_dirs_windows=("extras/CUPTI/lib64", "bin"),
-        ctk_root_canary_anchor_libnames=("cudart",),
+        ctk_root_canary_anchor_libnames=CTK_ROOT_CANARY_ANCHOR_LIBNAMES,
     ),
     DescriptorSpec(
         name="cudla",

cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py

@@ -136,13 +136,26 @@ def _loadable_via_canary_subprocess(libname: str, *, timeout: float = _CANARY_PR
     return _resolve_system_loaded_abs_path_in_subprocess(libname, timeout=timeout) is not None
 
 
+def resolve_ctk_root_via_canary(canary_libname: str) -> str | None:
+    """Resolve the CUDA Toolkit root from a system-loadable canary library.
+
+    The canary library's absolute path is resolved by the OS dynamic loader in
+    an isolated subprocess, which honors ``LD_LIBRARY_PATH`` on Linux and the
+    native DLL search on Windows. The toolkit root is then derived from that
+    path. Returns ``None`` if the canary cannot be resolved or no root can be
+    derived.
+    """
+    canary_abs_path = _resolve_system_loaded_abs_path_in_subprocess(canary_libname)
+    if canary_abs_path is None:
+        return None
+    ctk_root: str | None = derive_ctk_root(canary_abs_path)
+    return ctk_root
+
+
 def _try_ctk_root_canary(ctx: SearchContext) -> str | None:
     """Try CTK-root canary fallback for descriptor-configured libraries."""
     for canary_libname in ctx.desc.ctk_root_canary_anchor_libnames:
-        canary_abs_path = _resolve_system_loaded_abs_path_in_subprocess(canary_libname)
-        if canary_abs_path is None:
-            continue
-        ctk_root = derive_ctk_root(canary_abs_path)
+        ctk_root = resolve_ctk_root_via_canary(canary_libname)
         if ctk_root is None:
             continue
         find = find_via_ctk_root(ctx, ctk_root)

cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py

@@ -19,6 +19,7 @@
     platform_include_subdirs,
     resolve_conda_anchor,
 )
+from cuda.pathfinder._utils.ctk_root_canary import CTK_ROOT_CANARY_ANCHOR_LIBNAMES
 from cuda.pathfinder._utils.env_vars import get_cuda_path_or_home
 from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages
 
@@ -121,7 +122,7 @@ def find_via_ctk_root_canary(desc: HeaderDescriptor) -> LocatedHeaderDir | None:
     """
     if not desc.use_ctk_root_canary:
         return None
-    canary_abs_path = _resolve_system_loaded_abs_path_in_subprocess("cudart")
+    canary_abs_path = _resolve_system_loaded_abs_path_in_subprocess(CTK_ROOT_CANARY_ANCHOR_LIBNAMES[0])
     if canary_abs_path is None:
         return None
     ctk_root = derive_ctk_root(canary_abs_path)

cuda_pathfinder/cuda/pathfinder/_utils/ctk_root_canary.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+CTK_ROOT_CANARY_ANCHOR_LIBNAMES = ("cudart",)

cuda_pathfinder/docs/nv-versions.json

@@ -3,6 +3,10 @@
         "version": "latest",
         "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/latest/"
     },
+    {
+        "version": "1.6.0",
+        "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/1.6.0/"
+    },
     {
         "version": "1.5.6",
         "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/1.5.6/"

cuda_pathfinder/docs/source/release/1.6.0-notes.rst

@@ -0,0 +1,40 @@
+.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+.. SPDX-License-Identifier: Apache-2.0
+
+.. py:currentmodule:: cuda.pathfinder
+
+``cuda-pathfinder`` 1.6.0 Release notes
+=======================================
+
+Highlights
+----------
+
+* :func:`find_nvidia_binary_utility` now probes configured binary directories
+  directly instead of delegating to ``shutil.which``, avoiding implicit
+  current-working-directory lookup on Windows. If those locations miss, a new
+  CTK-root canary fallback resolves ``cudart`` through the OS dynamic loader,
+  derives its CUDA Toolkit root, and searches that root's ``bin`` layout. With
+  multiple visible toolkits, the result follows the ``cudart`` selected by the
+  dynamic loader and may differ from the executable selected by a shell.
+  (`PR #2196 <https://github.com/NVIDIA/cuda-python/pull/2196>`_)
+
+* Add dynamic-library loading and header discovery for the cuQuantum libraries
+  ``cudensitymat``, ``cupauliprop``, ``cutensornet``, ``custabilizer``, and
+  ``custatevec`` on Linux.
+  (`PR #2376 <https://github.com/NVIDIA/cuda-python/pull/2376>`_)
+
+* Add dynamic-library loading support for ``cutensorMp`` on Linux, including
+  preloading the packaged CUDA runtime in CUDA 12 wheel-only environments.
+  (`PR #2374 <https://github.com/NVIDIA/cuda-python/pull/2374>`_)
+
+* Add Windows Arm64 discovery for ``cudla.dll``, enabling
+  ``load_nvidia_dynamic_lib("cudla")`` to find cuDLA in the CUDA Toolkit's
+  architecture-specific ``bin`` directories.
+  (`PR #2274 <https://github.com/NVIDIA/cuda-python/pull/2274>`_)
+
+Internal maintenance
+--------------------
+
+* Clean up dead and misleading error-handling code in the Linux and Windows
+  dynamic-library loaders; runtime behavior is unchanged.
+  (`PR #2239 <https://github.com/NVIDIA/cuda-python/pull/2239>`_)

cuda_pathfinder/tests/test_ctk_root_discovery.py

@@ -17,6 +17,7 @@
     _load_lib_no_cache,
     _resolve_system_loaded_abs_path_in_subprocess,
     _try_ctk_root_canary,
+    resolve_ctk_root_via_canary,
 )
 from cuda.pathfinder._dynamic_libs.search_steps import (
     SearchContext,
@@ -369,6 +370,35 @@ def test_canary_skips_when_abs_path_none(mocker):
     assert _try_ctk_root_canary(_ctx("nvvm")) is None
 
 
+# ---------------------------------------------------------------------------
+# resolve_ctk_root_via_canary (shared by lib and binary discovery)
+# ---------------------------------------------------------------------------
+
+
+def test_resolve_ctk_root_via_canary_returns_root(tmp_path, mocker):
+    ctk_root = tmp_path / "cuda-13"
+    _create_cudart_in_ctk(ctk_root)
+    probe = mocker.patch(
+        f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess",
+        return_value=_fake_canary_path(ctk_root),
+    )
+    assert resolve_ctk_root_via_canary("cudart") == str(ctk_root)
+    probe.assert_called_once_with("cudart")
+
+
+def test_resolve_ctk_root_via_canary_none_when_probe_fails(mocker):
+    mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None)
+    assert resolve_ctk_root_via_canary("cudart") is None
+
+
+def test_resolve_ctk_root_via_canary_none_when_unrecognized(mocker):
+    mocker.patch(
+        f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess",
+        return_value=os.path.join(os.sep, "weird", "path", "libcudart.so.13"),
+    )
+    assert resolve_ctk_root_via_canary("cudart") is None
+
+
 # ---------------------------------------------------------------------------
 # _load_lib_no_cache search-order
 # ---------------------------------------------------------------------------