From 1de15692e5fae9bb8545564a727a760ab5f27020 Mon Sep 17 00:00:00 2001 From: abetlen Date: Sun, 26 Jul 2026 14:04:05 -0700 Subject: [PATCH] feat: update llama.cpp to d2a818231 --- CHANGELOG.md | 2 + .../low_level_api/low_level_api_chat_cpp.py | 8 +++- examples/server/server.py | 16 +++++-- llama_cpp/llama.py | 15 ++++-- llama_cpp/llama_chat_format.py | 8 +++- llama_cpp/llama_cpp.py | 46 +++++++++++++------ llama_cpp/mtmd_cpp.py | 7 +++ tests/test_llama.py | 8 +++- vendor/llama.cpp | 2 +- 9 files changed, 82 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 629b645a5a..c2108eecf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@d2a818231 + ## [0.3.34] - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 20f7a158ac..18d7309a2a 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,8 +76,12 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap + if self.params.use_mlock: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif self.params.use_mmap: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE self.model = llama_cpp.llama_model_load_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/examples/server/server.py b/examples/server/server.py index 72adc79059..18f8327b23 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -11030,7 +11030,9 @@ def _build_prompt_plan_locked( "multiple videos require MTMD to report frame counts" ) input_text = mtmd_cpp.mtmd_input_text() - input_text.text = prompt.encode("utf-8") + input_text_bytes = prompt.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = False input_text.parse_special = True chunks = mtmd_cpp.mtmd_input_chunks_init() @@ -11609,10 +11611,14 @@ def build_model_params( model_params.tensor_split = tensor_split_ref if vocab_only is not None: model_params.vocab_only = vocab_only - if use_mmap is not None: - model_params.use_mmap = use_mmap - if use_mlock is not None: - model_params.use_mlock = use_mlock + if use_mlock: + model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap is not None: + model_params.load_mode = ( + llama_cpp.LLAMA_LOAD_MODE_MMAP + if use_mmap + else llama_cpp.LLAMA_LOAD_MODE_NONE + ) kv_overrides_ref = None if kv_overrides is not None: diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b5bffd46b5..e27d90dc86 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -244,8 +244,14 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap if lora_path is None else False - self.model_params.use_mlock = use_mlock + if lora_path is not None: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE + elif use_mlock: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE # kv_overrides is the original python dict self.kv_overrides = kv_overrides @@ -2142,8 +2148,9 @@ def __getstate__(self): main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_mlock=self.model_params.use_mlock, + use_mmap=self.model_params.load_mode + in (llama_cpp.LLAMA_LOAD_MODE_MMAP, llama_cpp.LLAMA_LOAD_MODE_MLOCK), + use_mlock=self.model_params.load_mode == llama_cpp.LLAMA_LOAD_MODE_MLOCK, kv_overrides=self.kv_overrides, # Context Params seed=self._seed, diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0034bdae98..4f41c2eb75 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2936,7 +2936,9 @@ def __call__( # Create input text structure input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True @@ -3485,7 +3487,9 @@ def raise_exception(message: str): bitmap_cleanup.append(bitmap) input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 64399fe317..eabde545e3 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -520,6 +520,18 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available +# }; +LLAMA_LOAD_MODE_NONE = 0 +LLAMA_LOAD_MODE_MMAP = 1 +LLAMA_LOAD_MODE_MLOCK = 2 +LLAMA_LOAD_MODE_DIRECT_IO = 3 + + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -789,8 +801,9 @@ class llama_model_imatrix_data(ctypes.Structure): # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# int32_t n_gpu_layers; // number of layers to store in VRAM +# int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model # // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; @@ -812,9 +825,6 @@ class llama_model_imatrix_data(ctypes.Structure): # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used @@ -826,17 +836,15 @@ class llama_model_params(ctypes.Structure): Attributes: devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern - n_gpu_layers (int): number of layers to store in VRAM + n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io (bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used @@ -849,15 +857,13 @@ class llama_model_params(ctypes.Structure): ] # NOTE: unused n_gpu_layers: int split_mode: int + load_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool @@ -868,15 +874,13 @@ class llama_model_params(ctypes.Structure): ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), @@ -1276,6 +1280,20 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: ... +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int, /) -> Optional[bytes]: + """Get the model load mode name.""" + ... + + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(value: bytes, /) -> int: + """Get the model load mode from a string.""" + ... + + # // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" # LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); @ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 35357a3279..6d8ecbd6f6 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -133,8 +133,15 @@ class mtmd_context_params(Structure): class mtmd_input_text(Structure): """Text input passed to `mtmd_tokenize`.""" + if TYPE_CHECKING: + text: Optional[bytes] + text_len: int + add_special: bool + parse_special: bool + _fields_ = [ ("text", c_char_p), + ("text_len", c_size_t), ("add_special", c_bool), ("parse_special", c_bool), ] diff --git a/tests/test_llama.py b/tests/test_llama.py index 70fce12d8e..fd110bb574 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -103,8 +103,12 @@ def test_real_model(llama_cpp_model_path): assert os.path.exists(llama_cpp_model_path) params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_mlock = llama_cpp.llama_supports_mlock() + if llama_cpp.llama_supports_mlock(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif llama_cpp.llama_supports_mmap(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE params.check_tensors = False model = internals.LlamaModel(path_model=llama_cpp_model_path, params=params) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3546c7948..d2a818231e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 +Subproject commit d2a818231effb12b7b20b80b3b8c7756a9a33a04